[shader] Go down to just a single input shader.
This is in preparation for adding child shaders.
Once we have child shaders we can have multiple
images by adding child shaders and changing the
input image.
Also:
- Moves all the image loading code into ShaderNode.
- Input image dimensions are now determined by the image
loaded, instead of being hard-coded.
Bug: skia:11272
Change-Id: I411ff67aa76d381cdce667a268dbc4282d7e8a16
Reviewed-on: https://skia-review.googlesource.com/c/buildbot/+/373819
Commit-Queue: Joe Gregorio <jcgregorio@google.com>
Reviewed-by: Kevin Lubick <kjlubick@google.com>
diff --git a/particles/modules/json/index.ts b/particles/modules/json/index.ts
index ed2d5cb..e3298bf 100644
--- a/particles/modules/json/index.ts
+++ b/particles/modules/json/index.ts
@@ -10,6 +10,7 @@
export interface SKSLMetaData {
Uniforms: number[] | null;
+ ImageURL: string;
Children: ChildShader[] | null;
}
diff --git a/scrap/go/scrap/scrap.go b/scrap/go/scrap/scrap.go
index ff32ab1..3bf802b 100644
--- a/scrap/go/scrap/scrap.go
+++ b/scrap/go/scrap/scrap.go
@@ -137,6 +137,9 @@
// Uniforms are all the inputs to the shader.
Uniforms []float32
+ // ImageURL is the URL of an image to load as an input shader.
+ ImageURL string
+
// Child shaders. A slice because order is important when mapping uniform
// names in code to child shaders passed to makeShaderWithChildren.
Children []ChildShader
diff --git a/shaders/modules/json/index.ts b/shaders/modules/json/index.ts
index ed2d5cb..e3298bf 100644
--- a/shaders/modules/json/index.ts
+++ b/shaders/modules/json/index.ts
@@ -10,6 +10,7 @@
export interface SKSLMetaData {
Uniforms: number[] | null;
+ ImageURL: string;
Children: ChildShader[] | null;
}
diff --git a/shaders/modules/shadernode/index.ts b/shaders/modules/shadernode/index.ts
index c84ac25..b511b06 100644
--- a/shaders/modules/shadernode/index.ts
+++ b/shaders/modules/shadernode/index.ts
@@ -7,19 +7,22 @@
* const shader = node.getShader(predefinedUniformValues);
*/
import { jsonOrThrow } from 'common-sk/modules/jsonOrThrow';
+import { errorMessage } from 'elements-sk/errorMessage';
import { Uniform } from '../../../infra-sk/modules/uniform/uniform';
import {
CanvasKit,
+ Image,
MallocObj, RuntimeEffect, Shader,
} from '../../build/canvaskit/canvaskit';
import { ScrapBody, ScrapID } from '../json';
+const DEFAULT_SIZE = 512;
+
export const predefinedUniforms = `uniform float3 iResolution; // Viewport resolution (pixels)
uniform float iTime; // Shader playback time (s)
uniform float4 iMouse; // Mouse drag pos=.xy Click pos=.zw (pixels)
-uniform float3 iImageResolution; // iImage1 and iImage2 resolution (pixels)
-uniform shader iImage1; // An input image (Mandrill).
-uniform shader iImage2; // An input image (Soccer ball).`;
+uniform float3 iImageResolution; // iImage1 resolution (pixels)
+uniform shader iImage1; // An input image (Mandrill).`;
/** How many of the uniforms listed in predefinedUniforms are of type 'shader'? */
export const numPredefinedShaderUniforms = predefinedUniforms.match(/^uniform shader/gm)!.length;
@@ -55,15 +58,28 @@
return vec4(1, 0, 0, 1);
}`;
+export type callback = ()=> void;
+
+const defaultImageURL = '/dist/mandrill.png';
+
const defaultBody: ScrapBody = {
Type: 'sksl',
Body: defaultShader,
SKSLMetaData: {
Uniforms: [],
+ ImageURL: '',
Children: [],
},
};
+/** Describes an image used as a shader. */
+interface InputImage {
+ width: number;
+ height: number;
+ image: HTMLImageElement;
+ shader: Shader;
+}
+
/**
* Called ShaderNode because once we support child shaders this will be just one
* node in a tree of shaders.
@@ -78,7 +94,7 @@
/** The shader code compiled. */
private effect: RuntimeEffect | null = null;
- private inputImageShaders: Shader[] = [];
+ private inputImageShader: InputImage | null = null;
private canvasKit: CanvasKit;
@@ -116,33 +132,55 @@
private _numPredefinedUniformValues: number = 0;
- constructor(canvasKit: CanvasKit, inputImageShaders: Shader[]) {
+ constructor(canvasKit: CanvasKit) {
this.canvasKit = canvasKit;
- if (inputImageShaders.length !== numPredefinedShaderUniforms) {
- throw new Error(`ShaderNode requires exactly two predefined image shaders, got ${inputImageShaders.length}`);
- }
- this.inputImageShaders = inputImageShaders;
- this.body = defaultBody;
+ this.inputImageShaderFromCanvasImageSource(new Image(DEFAULT_SIZE, DEFAULT_SIZE));
+ this.setScrap(defaultBody);
}
- /** Loads a scrap from the backend for the given scrap id. */
- async loadScrap(scrapID: string): Promise<void> {
+ /**
+ * Loads a scrap from the backend for the given scrap id.
+ *
+ * The imageLoadedCallback is called once the image has fully loaded.
+ */
+ async loadScrap(scrapID: string, imageLoadedCallback: callback | null = null): Promise<void> {
this.scrapID = scrapID;
const resp = await fetch(`/_/load/${scrapID}`, {
credentials: 'include',
});
const scrapBody = (await jsonOrThrow(resp)) as ScrapBody;
- this.setScrap(scrapBody);
+ this.setScrap(scrapBody, imageLoadedCallback);
}
- /** Sets the code and uniforms of a shader to run. */
- setScrap(scrapBody: ScrapBody): void {
+ /**
+ * Sets the code and uniforms of a shader to run.
+ *
+ * The imageLoadedCallback is called once the image has fully loaded.
+ */
+ setScrap(scrapBody: ScrapBody, imageLoadedCallback: callback | null = null): void {
this.body = scrapBody;
this._shaderCode = this.body.Body;
this.currentUserUniformValues = this.body.SKSLMetaData?.Uniforms || [];
+
+ const imageURL = this.body.SKSLMetaData?.ImageURL || defaultImageURL;
+ this.promiseOnImageLoaded(imageURL).then((imageElement) => {
+ this.inputImageShaderFromCanvasImageSource(imageElement);
+ if (imageLoadedCallback) {
+ imageLoadedCallback();
+ }
+ }).catch(errorMessage);
this.compile();
}
+ /** Returns a copy of the current ScrapBody for the shader. */
+ getScrap(): ScrapBody {
+ return JSON.parse(JSON.stringify(this.body));
+ }
+
+ get inputImageElement(): HTMLImageElement {
+ return this.inputImageShader!.image;
+ }
+
/**
* Saves the scrap to the backend returning a Promise that resolves to the
* scrap id that it was stored at, or reject on an error.
@@ -153,6 +191,10 @@
Type: 'sksl',
SKSLMetaData: {
Uniforms: this._currentUserUniformValues,
+ // TODO(jcgregorio) Remember once we start saving ImageURLs that we
+ // need to to remove ImageURLs that aren't relative or that don't
+ // start with http[s]://, e.g. don't store blob:// or file:// urls.
+ ImageURL: '',
Children: [],
},
};
@@ -282,7 +324,14 @@
// Copy in our local edited uniform values to the right spots.
this.currentUserUniformValues.forEach((val, index) => { uniformsFloat32Array[index + this._numPredefinedUniformValues] = val; });
- return this.effect!.makeShaderWithChildren(uniformsFloat32Array, false, this.inputImageShaders);
+ // Write in the iImageResolution uniform values.
+ const imageResolution = this.findUniform('iImageResolution');
+ if (imageResolution) {
+ uniformsFloat32Array[imageResolution.slot] = this.inputImageShader!.width;
+ uniformsFloat32Array[imageResolution.slot + 1] = this.inputImageShader!.height;
+ }
+
+ return this.effect!.makeShaderWithChildren(uniformsFloat32Array, false, [this.inputImageShader!.shader]);
}
get numPredefinedUniformValues(): number {
@@ -338,4 +387,40 @@
}
return false;
}
+
+ private findUniform(name: string): Uniform | null {
+ for (let i = 0; i < this.uniforms.length; i++) {
+ if (name === this.uniforms[i].name) {
+ return this.uniforms[i];
+ }
+ }
+ return null;
+ }
+
+ private promiseOnImageLoaded(url: string): Promise<HTMLImageElement> {
+ return new Promise<HTMLImageElement>((resolve, reject) => {
+ const ele = new Image();
+ ele.src = url;
+ if (ele.complete) {
+ resolve(ele);
+ } else {
+ ele.addEventListener('load', () => resolve(ele));
+ ele.addEventListener('error', (e) => reject(e));
+ }
+ });
+ }
+
+ private inputImageShaderFromCanvasImageSource(imageElement: HTMLImageElement): void {
+ if (this.inputImageShader) {
+ this.inputImageShader.shader.delete();
+ }
+ const image = this.canvasKit!.MakeImageFromCanvasImageSource(imageElement);
+ this.inputImageShader = {
+ width: imageElement.naturalWidth,
+ height: imageElement.naturalHeight,
+ image: imageElement,
+ shader: image.makeShaderOptions(this.canvasKit!.TileMode.Clamp, this.canvasKit!.TileMode.Clamp, this.canvasKit!.FilterMode.Linear, this.canvasKit!.MipmapMode.None),
+ };
+ image.delete();
+ }
}
diff --git a/shaders/modules/shadernode/index_test.ts b/shaders/modules/shadernode/index_test.ts
index e911f4f..86d14f2 100644
--- a/shaders/modules/shadernode/index_test.ts
+++ b/shaders/modules/shadernode/index_test.ts
@@ -19,20 +19,10 @@
const createShaderNode = async (): Promise<ShaderNode> => {
const ck = await getCanvasKit();
- const image = ck.MakeImageFromCanvasImageSource(new Image(512, 512));
- const shader1 = image.makeShaderOptions(ck.TileMode.Clamp, ck.TileMode.Clamp, ck.FilterMode.Linear, ck.MipmapMode.None);
- const shader2 = image.makeShaderOptions(ck.TileMode.Clamp, ck.TileMode.Clamp, ck.FilterMode.Linear, ck.MipmapMode.None);
-
- return new ShaderNode(ck, [shader1, shader2]);
+ return new ShaderNode(ck);
};
describe('ShaderNode', async () => {
- it('constructor throws when not passed in the correct number of image shaders', async () => {
- const ck = await getCanvasKit();
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
- assert.throws(() => { const node = new ShaderNode(ck, []); });
- });
-
it('constructor builds with a default shader', async () => {
const node = await createShaderNode();
node.compile();
@@ -68,6 +58,7 @@
`,
SKSLMetaData: {
Children: [],
+ ImageURL: '',
Uniforms: [1, 0, 1, 0],
},
});
@@ -113,6 +104,7 @@
`,
SKSLMetaData: {
Children: [],
+ ImageURL: '',
Uniforms: startingUniformValues,
},
});
@@ -146,6 +138,7 @@
`,
SKSLMetaData: {
Children: [],
+ ImageURL: '',
Uniforms: [1, 0, 1, 0],
},
});
@@ -154,4 +147,18 @@
assert.deepEqual(node.compileErrorLineNumbers, [4]);
node.compileErrorMessage.startsWith('error: 4:');
});
+
+ it('makes a copy of the ScrapBody', async () => {
+ const node = await createShaderNode();
+ const startScrap = node.getScrap();
+ assert.isNotEmpty(startScrap.Body);
+ startScrap.Body = '';
+ // Confirm we haven't changed the original scrap.
+ assert.isNotEmpty(node['body']!.Body);
+ });
+
+ it('always starts with non-null input image', async () => {
+ const node = await createShaderNode();
+ assert.isNotNull(node.inputImageElement);
+ });
});
diff --git a/shaders/modules/shaders-app-sk/shaders-app-sk.ts b/shaders/modules/shaders-app-sk/shaders-app-sk.ts
index e99df36..fe6d403 100644
--- a/shaders/modules/shaders-app-sk/shaders-app-sk.ts
+++ b/shaders/modules/shaders-app-sk/shaders-app-sk.ts
@@ -146,8 +146,6 @@
private paint: Paint | null = null;
- private inputImageShaders: Shader[] = [];
-
private shaderNode: ShaderNode | null = null;
// Records the lines that have been marked as having errors. We keep these
@@ -196,7 +194,9 @@
></uniform-dimensions-sk>`);
break;
case 'iImageResolution':
- ret.push(html`<uniform-imageresolution-sk .uniform=${uniform}></uniform-imageresolution-sk>`);
+ // No-op. This is no longer handled via uniform control, the
+ // dimensions are handed directly into the ShaderNode from the image
+ // measurements.
break;
default:
if (uniform.name.toLowerCase().indexOf('color') !== -1) {
@@ -242,13 +242,9 @@
<textarea rows=${numPredefinedUniformLines} cols=75 readonly id="predefinedShaderInputs">${predefinedUniforms}</textarea>
<div id=imageSources>
<figure>
- <img id=iImage1 loading="eager" src="/dist/mandrill.png">
+ ${ele.shaderNode?.inputImageElement}
<figcaption>iImage1</figcaption>
</figure>
- <figure>
- <img id=iImage2 loading="eager" src="/dist/soccer.png">
- <figcaption>iImage2</figcaption>
- </figure>
</div>
</details>
<div id="codeEditor"></div>
@@ -309,29 +305,13 @@
// Continue the setup once CanvasKit WASM has loaded.
kitReady.then(async (ck: CanvasKit) => {
this.kit = ck;
-
- try {
- this.inputImageShaders = [];
- // Wait until all the images are loaded.
- // Note: All shader images MUST be 512 x 512 to agree with iImageResolution.
- const elements = await Promise.all<HTMLImageElement>([this.promiseOnImageLoaded('#iImage1'), this.promiseOnImageLoaded('#iImage2')]);
- // Convert them into shaders.
- elements.forEach((ele) => {
- const image = this.kit!.MakeImageFromCanvasImageSource(ele);
- const shader = image.makeShaderOptions(this.kit!.TileMode.Clamp, this.kit!.TileMode.Clamp, this.kit!.FilterMode.Linear, this.kit!.MipmapMode.None);
- this.inputImageShaders.push(shader);
- });
- } catch (error) {
- errorMessage(error);
- }
-
this.paint = new this.kit.Paint();
try {
this.stateChanged = stateReflector(
/* getState */ () => (this.state as unknown) as HintableObject,
/* setState */ (newState: HintableObject) => {
this.state = (newState as unknown) as State;
- this.shaderNode = new ShaderNode(this.kit!, this.inputImageShaders);
+ this.shaderNode = new ShaderNode(this.kit!);
if (!this.state.id) {
this.run();
} else {
@@ -345,22 +325,6 @@
});
}
- /**
- * Returns a Promise that resolves when in image loads in an <img> element
- * with the given id.
- */
- private promiseOnImageLoaded(id: string): Promise<HTMLImageElement> {
- return new Promise<HTMLImageElement>((resolve, reject) => {
- const ele = $$<HTMLImageElement>(id, this)!;
- if (ele.complete) {
- resolve(ele);
- } else {
- ele.addEventListener('load', () => resolve(ele));
- ele.addEventListener('error', (e) => reject(e));
- }
- });
- }
-
private dimensionsChanged(e: Event) {
const newDims = (e as CustomEvent<DimensionsChangedEventDetail>).detail;
this.width = newDims.width;
@@ -380,7 +344,10 @@
return;
}
try {
- await this.shaderNode!.loadScrap(this.state.id);
+ await this.shaderNode!.loadScrap(this.state.id, () => {
+ // Re-render once the input image has loaded.
+ this._render();
+ });
this._render();
const predefinedUniformValues = new Array(this.shaderNode!.numPredefinedUniformValues).fill(0);
@@ -492,7 +459,6 @@
this.kit!.setCurrentContext(this.canvasKitContext);
const shader = this.shaderNode!.getShader(this.getPredefinedUniformValuesFromControls());
if (!shader) {
- errorMessage('Failed to get shader.', 0);
return;
}
diff --git a/shaders/modules/shaders-app-sk/shaders-app-sk_puppeteer_test.ts b/shaders/modules/shaders-app-sk/shaders-app-sk_puppeteer_test.ts
index b8eeacc..96b46fe 100644
--- a/shaders/modules/shaders-app-sk/shaders-app-sk_puppeteer_test.ts
+++ b/shaders/modules/shaders-app-sk/shaders-app-sk_puppeteer_test.ts
@@ -10,7 +10,7 @@
let testBed: TestBed;
before(async () => {
testBed = await loadCachedTestBed(
- path.join(__dirname, '..', '..', 'webpack.config.ts')
+ path.join(__dirname, '..', '..', 'webpack.config.ts'),
);
});
diff --git a/shaders/tsconfig.json b/shaders/tsconfig.json
index 826931f..b81dac7 100644
--- a/shaders/tsconfig.json
+++ b/shaders/tsconfig.json
@@ -10,5 +10,6 @@
"target": "es2017",
"types": ["mocha", "chai", "node"]
},
- "include": ["./modules/**/*.ts", "./pages/**/*.ts"]
+ "include": ["./modules/**/*.ts", "./pages/**/*.ts"],
+ "exclude": ["./node_modules/"]
}
