blob: 5eadad33ea85ec9e1f867b4e31ea2227d860300a [file] [log] [blame]
/**
* @module skottie-audio-sk
* @description <h2><code>skottie-audio-sk</code></h2>
*
* <p>
* A skottie audio sync manager.
* </p>
* <p>
* This module lets the user upload a song track and synchronize
* it with the animation by speeding up or slowing down
* the animation to fit the music.
* </p>
* <p>
* To make this easier, the module can do some analysis of the song
* to suggest the tempo of the song (in beats per minute or BPMs).
* Once a song is uploaded, we identify the top 5 likely bpms
* for the song and suggest those options or read the bpm encoded in the file name.
* If none of those options work, BPMs can be set manually.
* </p>
* <p>
* Once the BPM is established, a second "Beat duration" value is required.
* This Beat duration reflects how many frames of the animation
* represent a single beat. This value will be used to calculate the final speed
* of the animation itself.
* </p>
*
*
* @evt apply - This event is triggered when the audio settings are set.
*
*
*/
import { define } from 'elements-sk/define';
import { html } from 'lit-html';
import { Howl } from 'howler';
import { ElementSk } from '../../../infra-sk/modules/ElementSk';
import { LottieAnimation } from '../types';
const INPUT_FILE_ID = 'fileInput';
const BPM_ID = 'bpm';
const BEAT_DURATION_ID = 'beatDuration';
const START_BUTTON_ID = 'startButton';
type LoadingState = 'idle' | 'loading' | 'loaded' | 'submitted';
export interface AudioStartEventDetail {
speed: number;
}
interface animationMarker {
cm: string;
}
interface markerData {
beat?: number;
type: string;
}
interface tempoInterval {
count: number;
tempo: number;
}
interface tempoPeak {
volume: number;
position: number;
}
export class SkottieAudioSk extends ElementSk {
private static template = (ele: SkottieAudioSk) => html`
<div>
<header class="header">
Audio
</header>
<section class=section>
<div class=inputs>
<label class=input-label>
<input
type=file
name=file
id=${INPUT_FILE_ID}
/> Choose audio file
</label>
<checkbox-sk label="Loop"
?checked=${ele.shouldLoop}
@click=${ele.toggleLoop}>
</checkbox-sk>
<label class=input-label>
<input
type=number
id=${BPM_ID}
.value=${ele.bpm}
required
/> BPM
</label>
${ele.bpmListTemplate()}
<label class=input-label>
<input
type=number
id=${BEAT_DURATION_ID}
.value=${ele.beatDuration}
required
/> Beat Duration (in frames)
</label>
</div>
${ele.startButtonTemplate()}
<section>
</div>
`;
private startButtonTemplate = () => {
if (this.loadingState === 'idle' || this.loadingState === 'submitted'
) {
return null;
}
if (this.loadingState === 'loading' || this.bpmCalculationState === 'loading') {
return html`<div>Loading...</div>`;
}
return html`
<button
class=start
id=${START_BUTTON_ID}
@click=${this.start}
>Start</button>
`;
};
private static bpmListOptionTemplate = (option: tempoInterval, onClick: (t: number)=> void) => html`
<li class=bpm-options-item>
<button class=bpm-options-item-button
@click=${() => onClick(option.tempo)}
>
${option.tempo}
</button>
</li>
`;
private bpmListTemplate = () => html`
<ul class=bpm-options>
${this.bmpList.map(((b: tempoInterval) => SkottieAudioSk.bpmListOptionTemplate(
b,
(option: number) => this.onBpmSelected(option),
)))}
</ul>
`;
private _animation: LottieAnimation | null = null;
private beatDuration: number = 0;
private bmpList: tempoInterval[] = [];
private bpm: number = 0;
private bpmCalculationState: LoadingState = 'idle';
private file: File | null = null;
private loadingState: LoadingState = 'idle';
private shouldLoop: boolean = true;
private sound: Howl | null = null;
constructor() {
super(SkottieAudioSk.template);
}
set animation(val: LottieAnimation) {
if (this._animation !== val) {
this._animation = val;
this.updateAnimation(val);
}
}
private updateAnimation(animation: LottieAnimation): void {
const markers = (animation.markers || []) as animationMarker[];
const marker = markers.find((markerItem: animationMarker) => {
try {
const md = JSON.parse(markerItem.cm) as markerData;
if (md.type === 'beat') {
return true;
}
} catch (err) {
// Marker does not have beat information
}
return false;
});
if (marker) {
const payload = JSON.parse(marker.cm) as markerData;
if (payload.beat) {
this.beatDuration = payload.beat;
this._render();
}
}
}
private static getPeaks(data: Float32Array[]): tempoPeak[] {
// What we're going to do here, is to divide up our audio into parts.
// We will then identify, for each part, what the loudest sample is in that
// part.
// It's implied that that sample would represent the most likely 'beat'
// within that part.
// Each part is 0.5 seconds long - or 22,050 samples.
// This will give us 60 'beats' - we will only take the loudest half of
// those.
// This will allow us to ignore breaks, and allow us to address tracks with
// a BPM below 120.
const partSize = 22050;
const parts = data[0].length / partSize;
let peaks: tempoPeak[] = [];
for (let i = 0; i < parts; i++) {
let max: tempoPeak | null = null;
for (let j = i * partSize; j < (i + 1) * partSize; j++) {
const volume = Math.max(Math.abs(data[0][j]), Math.abs(data[1][j]));
if (!max || (volume > max.volume)) {
max = {
position: j,
volume: volume,
};
}
}
peaks.push(max!);
}
// We then sort the peaks according to volume...
peaks.sort((a: tempoPeak, b: tempoPeak) => b.volume - a.volume);
// ...take the loundest half of those...
peaks = peaks.splice(0, peaks.length * 0.5);
// ...and re-sort it back based on position.
peaks.sort((a: tempoPeak, b: tempoPeak) => a.position - b.position);
return peaks;
}
private getIntervals(peaks: tempoPeak[]) {
// What we now do is get all of our peaks, and then measure the distance to
// other peaks, to create intervals. Then based on the distance between
// those peaks (the distance of the intervals) we can calculate the BPM of
// that particular interval.
// The interval that is seen the most should have the BPM that corresponds
// to the track itself.
const groups: tempoInterval[] = [];
peaks.forEach((peak: tempoPeak, index: number) => {
for (let i = 1; (index + i) < peaks.length && i < 10; i++) {
const group = {
tempo: (60 * 44100) / (peaks[index + i].position - peak.position),
count: 1,
};
while (group.tempo < 90) {
group.tempo *= 2;
}
while (group.tempo > 180) {
group.tempo /= 2;
}
group.tempo = Math.round(group.tempo);
const groupTempo = groups.find((interval: tempoInterval) => interval.tempo === group.tempo);
if (!groupTempo) {
groups.push(group);
} else {
groupTempo.count += 1;
}
}
});
return groups;
}
private onOfflineRenderComplete(e: OfflineAudioCompletionEvent): void {
const buffer = e.renderedBuffer;
const peaks = SkottieAudioSk.getPeaks([buffer.getChannelData(0), buffer.getChannelData(1)]);
const groups = this.getIntervals(peaks);
const top = groups.sort((intA: tempoInterval, intB: tempoInterval) => intB.count - intA.count).splice(0, 5);
this.bmpList = top;
if (!this.bpm) {
this.bpm = top[0].tempo;
}
this.bpmCalculationState = 'loaded';
this._render();
}
// It looks for peaks on the audio to estimate best guesses of the bpm
// more info about it https://jmperezperez.com/bpm-detection-javascript/
private detectBPMs(ev: ProgressEvent<FileReader>): void {
const track = ev.target!.result as ArrayBuffer;
// Create offline context
const offlineContext = new OfflineAudioContext(2, 30 * 44100, 44100);
offlineContext.decodeAudioData(track, (buffer: AudioBuffer) => {
// Create buffer source
const source = offlineContext.createBufferSource();
source.buffer = buffer;
// Beats, or kicks, generally occur around the 100 to 150 hz range.
// Below this is often the bassline. So let's focus just on that.
// First a lowpass to remove most of the song.
const lowpass = offlineContext.createBiquadFilter();
lowpass.type = 'lowpass';
lowpass.frequency.value = 150;
lowpass.Q.value = 1;
// Run the output of the source through the low pass.
source.connect(lowpass);
// Now a highpass to remove the bassline.
const highpass = offlineContext.createBiquadFilter();
highpass.type = 'highpass';
highpass.frequency.value = 100;
highpass.Q.value = 1;
// Run the output of the lowpass through the highpass.
lowpass.connect(highpass);
// Run the output of the highpass through our offline context.
highpass.connect(offlineContext.destination);
// Start the source, and render the output into the offline conext.
source.start(0);
offlineContext.startRendering();
});
offlineContext.oncomplete = (e: OfflineAudioCompletionEvent) => this.onOfflineRenderComplete(e);
}
private onBpmSelected(option: number): void {
this.bpm = option;
if (this.loadingState === 'submitted') {
this.start();
} else {
this._render();
}
}
private onFileDataLoaded(ev: ProgressEvent<FileReader>): void {
const result = ev.target!.result as string;
if (this.sound) {
this.sound.unload();
}
this.sound = new Howl({
src: [result],
loop: this.shouldLoop,
});
this.sound.on('load', () => this.onAudioLoaded());
}
private onBpmChange(ev: Event): void {
ev.preventDefault();
const value = parseInt((ev.target as HTMLInputElement).value, 10);
if (this.bpm !== value) {
this.bpm = value;
if (this.loadingState === 'submitted') {
this.start();
} else {
this._render();
}
}
}
private onBeatDurationChange(ev: Event): void {
ev.preventDefault();
const value = +(ev.target as HTMLInputElement).value;
if (this.beatDuration !== value) {
this.beatDuration = value;
}
this._render();
}
private toggleLoop(ev: Event): void {
ev.preventDefault();
this.shouldLoop = !this.shouldLoop;
if (this.sound) {
this.sound.loop(this.shouldLoop);
}
this._render();
}
private onAudioLoaded(): void {
this.loadingState = 'loaded';
this._render();
}
// It looks for a bpm as part of the file name.
// The format of the name should be [name]bpm_[number]
private static searchBpmOnName(name: string): number {
const regex = /bpm_([0-9]*)/i;
const found = name.match(regex);
if (found) {
return +found[1];
}
return 0;
}
private start(): void {
const animBeat = this.beatDuration;
const animFps: number = this._animation?.fr || 0;
const songBpm = this.bpm;
const songBps = songBpm / 60;
const animBps = animFps / animBeat;
let animSpeed = songBps / animBps;
if (animSpeed > 1.5) {
animSpeed /= 2;
}
this.dispatchEvent(new CustomEvent<AudioStartEventDetail>('apply', {
detail: {
speed: animSpeed,
},
}));
if (this.sound) {
this.sound.seek(0);
if (!this.sound.playing()) {
this.sound.play();
}
}
this.loadingState = 'submitted';
this._render();
}
pause(): void {
if (this.sound) {
this.sound.pause();
}
}
resume(): void {
if (this.sound) {
this.sound.play();
}
}
rewind(): void {
if (this.sound) {
this.sound.seek(0);
}
}
private onFileChange(ev: Event): void {
this.file = (ev.target as HTMLInputElement).files![0];
this.bpm = SkottieAudioSk.searchBpmOnName(this.file.name);
this.loadingState = 'loading';
this.bpmCalculationState = 'loading';
const reader = new FileReader();
reader.readAsDataURL(this.file);
reader.addEventListener('load', (e: ProgressEvent<FileReader>) => this.onFileDataLoaded(e), false);
const arrayBufferReader = new FileReader();
arrayBufferReader.readAsArrayBuffer(this.file);
arrayBufferReader.addEventListener('load', (e: ProgressEvent<FileReader>) => this.detectBPMs(e), false);
this._render();
}
async inputEvent(ev: Event): Promise<void> {
const target = ev.target as HTMLInputElement;
if (target.id === INPUT_FILE_ID) {
this.onFileChange(ev);
} else if (target.id === BPM_ID) {
this.onBpmChange(ev);
} else if (target.id === BEAT_DURATION_ID) {
this.onBeatDurationChange(ev);
}
this._render();
}
connectedCallback(): void {
super.connectedCallback();
this._render();
this.addEventListener('input', this.inputEvent);
}
disconnectedCallback(): void {
super.disconnectedCallback();
this.removeEventListener('input', this.inputEvent);
}
}
define('skottie-audio-sk', SkottieAudioSk);