[skottie] add audio layer support to skottie web player
Change-Id: I18d059cd24c1d6447adbdafbf65a22f59bca1a35
Reviewed-on: https://skia-review.googlesource.com/c/buildbot/+/369434
Commit-Queue: Jorge Betancourt <jmbetancourt@google.com>
Reviewed-by: Kevin Lubick <kjlubick@google.com>
Reviewed-by: Florin Malita <fmalita@chromium.org>
diff --git a/.gitignore b/.gitignore
index 6e8b531..6b1d47b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
+# MacOS file system attributes
+*.DS_Store
# Compiled python files.
*.pyc
# Files open in vi.
@@ -111,4 +113,4 @@
# Ignore all bazel-* symlinks. There is no full list since this can change
# based on the name of the directory bazel is cloned into.
-/bazel-*
\ No newline at end of file
+/bazel-*
diff --git a/skottie/modules/audio.js b/skottie/modules/audio.js
new file mode 100644
index 0000000..fa8f343
--- /dev/null
+++ b/skottie/modules/audio.js
@@ -0,0 +1,58 @@
+import {Howl, Howler} from 'howler';
+
+// seek tolerance in seconds, keeps the Howl player from seeking unnecessarily
+// if the number is too small, Howl.seek() is called too often and creates a popping noise
+// too large and audio layers may be skipped over
+const kTolerance = 0.75;
+
+// SoundMaps have string : player pairs
+export function SoundMap() {
+ this.map = new Map();
+ this.setPlayer = function(name, player) {
+ if (typeof name == 'string' && player.hasOwnProperty('seek')) {
+ this.map.set(name, player);
+ }
+ };
+ this.getPlayer = function(name) {
+ return this.map.get(name);
+ };
+ this.pause = function() {
+ for(const player of this.map.values()) {
+ player.seek(-1);
+ }
+ }
+}
+
+/**
+ * AudioPlayers wrap a howl and control playback through seek calls
+ *
+ * @param source - URL or base64 data URI pointing to audio data
+ * @param format - only needed if extension is not provided by source (inline URI)
+ *
+ */
+export function AudioPlayer(source) {
+ this.playing = false;
+ this.howl = new Howl({
+ src: [source],
+ preload: true
+ });
+ this.seek = function(t) {
+ if (!this.playing && t >=0) {
+ this.howl.play();
+ this.playing = true;
+ }
+
+ if (this.playing) {
+ if (t < 0) {
+ this.howl.pause();
+ this.playing = false;
+ } else {
+ const playerPos = this.howl.seek();
+
+ if (Math.abs(playerPos - t) > kTolerance) {
+ this.howl.seek(t);
+ }
+ }
+ }
+ };
+}
diff --git a/skottie/modules/skottie-player-sk/skottie-player-sk.js b/skottie/modules/skottie-player-sk/skottie-player-sk.js
index 5409864..051e788 100644
--- a/skottie/modules/skottie-player-sk/skottie-player-sk.js
+++ b/skottie/modules/skottie-player-sk/skottie-player-sk.js
@@ -183,7 +183,7 @@
ck.setDecodeCacheLimitBytes(CACHE_SIZE);
this._engine.kit = ck;
- this._initializeSkottie(config.lottie, config.assets);
+ this._initializeSkottie(config.lottie, config.assets, config.soundMap);
this._render();
});
}
@@ -223,7 +223,7 @@
}
}
- _initializeSkottie(lottieJSON, assets) {
+ _initializeSkottie(lottieJSON, assets, soundMap) {
this._state.loading = false;
// Rebuild the surface only if needed.
@@ -252,7 +252,7 @@
}
this._engine.animation = this._engine.kit.MakeManagedAnimation(
- JSON.stringify(lottieJSON), assets,
+ JSON.stringify(lottieJSON), assets, null, soundMap
);
if (!this._engine.animation) {
throw new Error('Could not parse Lottie JSON.');
diff --git a/skottie/modules/skottie-sk/skottie-sk.js b/skottie/modules/skottie-sk/skottie-sk.js
index f64d596..6989d84 100644
--- a/skottie/modules/skottie-sk/skottie-sk.js
+++ b/skottie/modules/skottie-sk/skottie-sk.js
@@ -21,6 +21,7 @@
import { setupListeners, onUserEdit, reannotate} from '../lottie-annotations'
import { stateReflector } from 'common-sk/modules/stateReflector'
import '../skottie-text-editor'
+import { SoundMap, AudioPlayer } from '../audio'
const JSONEditor = require('jsoneditor/dist/jsoneditor-minimalist.js');
const bodymovin = require('lottie-web/build/player/lottie.min.js');
@@ -408,11 +409,12 @@
_initializePlayer() {
this._skottiePlayer.initialize({
- width: this._width,
- height: this._height,
- lottie: this._state.lottie,
- assets: this._state.assets,
- fps: this._fps,
+ width: this._width,
+ height: this._height,
+ lottie: this._state.lottie,
+ assets: this._state.assets,
+ soundMap: this._state.soundMap,
+ fps: this._fps,
}).then(() => {
this._duration = this._skottiePlayer.duration();
// If the user has specified a value for FPS, we want to lock the
@@ -446,9 +448,12 @@
Promise.all(toLoad).then((externalAssets) => {
const assets = {};
+ const sounds = new SoundMap();
for (const asset of externalAssets) {
- if (asset) {
+ if (asset && asset.bytes) {
assets[asset.name] = asset.bytes;
+ } else if (asset && asset.player) {
+ sounds.setPlayer(asset.name, asset.player);
}
}
@@ -460,6 +465,7 @@
});
this._state.assets = assets;
+ this._state.soundMap = sounds;
this.render();
this._initializePlayer();
// Re-sync all players
@@ -517,25 +523,43 @@
_loadAssets(assets) {
const promises = [];
for (const asset of assets) {
- // asset.p is the filename, if it's an image.
- // Don't try to load inline/dataURI images.
- const should_load = asset.p && asset.p.startsWith && !asset.p.startsWith('data:');
- if (should_load) {
- promises.push(fetch(`${this._assetsPath}/${this._hash}/${asset.p}`)
- .then((resp) => {
- // fetch does not reject on 404
- if (!resp.ok) {
- console.error(`Could not load ${asset.p}: status ${resp.status}`)
- return null;
- }
- return resp.arrayBuffer().then((buffer) => {
- return {
- 'name': asset.p,
- 'bytes': buffer
- };
- });
- })
- );
+ if (asset.id.startsWith('audio_')) {
+ // Howler handles our audio assets, they don't provide a promise when making a new Howl.
+ // We push the audio asset as is and hope that it loads before playback starts.
+ const inline = asset.p && asset.p.startsWith && asset.p.startsWith('data:');
+ if (inline) {
+ promises.push({
+ 'name': asset.id,
+ 'player': new AudioPlayer(asset.p)
+ });
+ } else {
+ promises.push({
+ 'name': asset.id,
+ 'player': new AudioPlayer(`${this._assetsPath}/${this._hash}/${asset.p}`)
+ });
+ }
+ }
+ else {
+ // asset.p is the filename, if it's an image.
+ // Don't try to load inline/dataURI images.
+ const should_load = asset.p && asset.p.startsWith && !asset.p.startsWith('data:');
+ if (should_load) {
+ promises.push(fetch(`${this._assetsPath}/${this._hash}/${asset.p}`)
+ .then((resp) => {
+ // fetch does not reject on 404
+ if (!resp.ok) {
+ console.error(`Could not load ${asset.p}: status ${resp.status}`)
+ return null;
+ }
+ return resp.arrayBuffer().then((buffer) => {
+ return {
+ 'name': asset.p,
+ 'bytes': buffer
+ };
+ });
+ })
+ );
+ }
}
}
return promises;
@@ -546,11 +570,14 @@
this._wasmTimePassed = Date.now() - this._firstFrameTime;
this._lottie && this._lottie.pause();
this._live && this._live.pause();
+ this._state.soundMap && this._state.soundMap.pause();
$$('#playpause').textContent = 'Play';
} else {
this._lottie && this._lottie.play();
this._live && this._live.play();
this._firstFrameTime = Date.now() - (this._wasmTimePassed || 0);
+ // There is no need call a soundMap.play() function here.
+ // Skottie invokes the play by calling seek on the needed audio track.
$$('#playpause').textContent = 'Pause';
}
this._playing = !this._playing;
diff --git a/skottie/package-lock.json b/skottie/package-lock.json
index 142e879..c951bd1 100644
--- a/skottie/package-lock.json
+++ b/skottie/package-lock.json
@@ -3456,6 +3456,11 @@
"integrity": "sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==",
"dev": true
},
+ "howler": {
+ "version": "2.2.1",
+ "resolved": "https://registry.npmjs.org/howler/-/howler-2.2.1.tgz",
+ "integrity": "sha512-0iIXvuBO/81CcrQ/HSSweYmbT50fT2mIc9XMFb+kxIfk2pW/iKzDbX1n3fZmDXMEIpYvyyfrB+gXwPYSDqUxIQ=="
+ },
"hpack.js": {
"version": "2.1.6",
"resolved": "https://registry.npmjs.org/hpack.js/-/hpack.js-2.1.6.tgz",
diff --git a/skottie/package.json b/skottie/package.json
index 514d2e5..8e5f779 100644
--- a/skottie/package.json
+++ b/skottie/package.json
@@ -9,6 +9,7 @@
"canvaskit-wasm": "0.5.0",
"common-sk": "~3.1.0",
"elements-sk": "^4.0.0",
+ "howler": "^2.2.1",
"jsoneditor": "~5.24.3",
"lit-html": "~1.0.0",
"lottie-web": "^5.4.3"
