feat: audio system, demo scene updates, docs, and engine improvements

engine/core/Engine.ts

@@ -4,6 +4,7 @@ import { VideoManager } from './VideoManager'
 import { StateManager } from './StateManager'
 import { ChoiceSystem } from '../systems/ChoiceSystem'
 import { QTESystem } from '../systems/QTESystem'
+import { AudioSystem } from '../systems/AudioSystem'
 
 type EventHandler = (...args: any[]) => void
 
@@ -13,6 +14,7 @@ export class Engine {
   stateManager: StateManager
   choiceSystem: ChoiceSystem
   qteSystem: QTESystem
+  audioSystem: AudioSystem
 
   private currentScene: SceneNode | null = null
   private events: Map<EngineEvent, Set<EventHandler>> = new Map()
@@ -29,6 +31,7 @@ export class Engine {
     this.stateManager = new StateManager()
     this.choiceSystem = new ChoiceSystem()
     this.qteSystem = new QTESystem()
+    this.audioSystem = new AudioSystem()
 
     this.videoManager.onTimeUpdate(this.onTimeUpdate)
   }
@@ -63,6 +66,25 @@ export class Engine {
       this.stateManager.apply(scene.onEnter)
     }
 
+    if (scene.videoMuted) {
+      this.videoManager.setMuted(true)
+    } else {
+      this.videoManager.setMuted(false)
+    }
+
+    const bgmUrl = scene.bgmUrl || null
+    if (bgmUrl) {
+      this.audioSystem.play(
+        bgmUrl,
+        scene.bgmVolume ?? 0.8,
+        scene.bgmCrossFade ?? 2.0,
+        scene.bgmDuckLevel,
+        scene.bgmDuckFade,
+      )
+    } else {
+      this.audioSystem.stop(scene.bgmCrossFade ?? 2.0)
+    }
+
     if (scene.type === 'image') {
       this.justCameFromImage = true
       this.isInitialScene = false
@@ -123,6 +145,8 @@ export class Engine {
 
       this.emit('qteTrigger', qte)
 
+      this.audioSystem.duckOn('qte')
+
       this.qteSystem.trigger(
         qte,
         (remaining, total) => {
@@ -130,6 +154,7 @@ export class Engine {
         },
         (success) => {
           this.qteResolved = true
+          this.audioSystem.duckOff('qte')
           if (success) {
             if (qte.effects?.success) {
               this.stateManager.apply(qte.effects.success)
@@ -167,6 +192,7 @@ export class Engine {
       const validChoices = this.getValidChoices(scene)
       if (validChoices.length > 0) {
         this.emit('choiceRequest', validChoices)
+        this.audioSystem.duckOn('choice')
         this.choiceSystem.start(
           validChoices,
           (timerState) => {
@@ -196,6 +222,12 @@ export class Engine {
     })
 
     this.emit('hotspotUpdate', visible)
+
+    if (visible.length > 0) {
+      this.audioSystem.duckOn('hotspot')
+    } else {
+      this.audioSystem.duckOff('hotspot')
+    }
   }
 
   getVisibleHotspots(scene: SceneNode): Hotspot[] {
@@ -209,6 +241,8 @@ export class Engine {
   clickHotspot(hotspot: Hotspot) {
     if (!this.currentScene) return
 
+    this.audioSystem.duckOff('hotspot')
+
     if (hotspot.effects) {
       this.stateManager.apply(hotspot.effects)
     }
@@ -234,6 +268,7 @@ export class Engine {
 
     if (validChoices.length > 0) {
       this.emit('choiceRequest', validChoices)
+      this.audioSystem.duckOn('choice')
 
       this.choiceSystem.start(
         validChoices,
@@ -241,6 +276,7 @@ export class Engine {
           this.emit('choiceTimer', timerState)
         },
         (defaultChoice) => {
+          this.audioSystem.duckOff('choice')
           this.emit('choiceTimeout', defaultChoice)
           this.makeChoice(defaultChoice)
         }
@@ -269,6 +305,8 @@ export class Engine {
   makeChoice(choice: Choice) {
     if (!this.currentScene) return
 
+    this.audioSystem.duckOff('choice')
+
     if (choice.effects) {
       this.stateManager.apply(choice.effects)
     }
@@ -291,6 +329,7 @@ export class Engine {
     this.ended = true
     this.loopActive = false
     this.qteSystem.cancel()
+    this.audioSystem.stop(2.0)
     this.emit('gameEnd')
   }
 
@@ -338,6 +377,7 @@ export class Engine {
 
   destroy() {
     this.qteSystem.destroy()
+    this.audioSystem.destroy()
     this.videoManager.detach()
     this.events.clear()
   }

engine/core/VideoManager.ts

@@ -154,6 +154,11 @@ export class VideoManager {
     this.active.currentTime = time
   }
 
+  setMuted(muted: boolean) {
+    if (this.elA) this.elA.muted = muted
+    if (this.elB) this.elB.muted = muted
+  }
+
   onEnd(cb: VideoEndCallback) {
     this.onEndCallback = cb
   }

engine/systems/AudioSystem.ts

@@ -0,0 +1,181 @@
+type DuckReason = 'qte' | 'choice' | 'hotspot'
+
+export class AudioSystem {
+  private ctx: AudioContext | null = null
+  private gainNode: GainNode | null = null
+  private currentSource: AudioBufferSourceNode | null = null
+  private currentUrl = ''
+  private currentVolume = 0.8
+  private bufferCache: Map<string, AudioBuffer> = new Map()
+  private maxCache = 3
+  private duckCount: Map<DuckReason, number> = new Map()
+  private duckLevel = 0.35
+  private duckFade = 0.5
+
+  private getCtx(): AudioContext {
+    if (!this.ctx) {
+      this.ctx = new AudioContext()
+      this.gainNode = this.ctx.createGain()
+      this.gainNode.connect(this.ctx.destination)
+    }
+    if (this.ctx.state === 'suspended') {
+      this.ctx.resume()
+    }
+    return this.ctx
+  }
+
+  async play(
+    url: string,
+    volume: number,
+    crossFade: number,
+    duckLvl?: number,
+    duckFd?: number,
+  ) {
+    if (duckLvl !== undefined) this.duckLevel = duckLvl
+    if (duckFd !== undefined) this.duckFade = duckFd
+    this.currentVolume = volume
+
+    if (url === this.currentUrl) {
+      const target = this.duckTarget()
+      if (this.gainNode) {
+        this.gainNode.gain.cancelScheduledValues(this.getCtx().currentTime)
+        this.gainNode.gain.setValueAtTime(this.gainNode.gain.value, this.getCtx().currentTime)
+        this.gainNode.gain.exponentialRampToValueAtTime(
+          Math.max(0.001, target),
+          this.getCtx().currentTime + 0.3,
+        )
+      }
+      return
+    }
+
+    const prevSource = this.currentSource
+
+    this.currentUrl = url
+    const buffer = await this.loadBuffer(url)
+    if (!buffer) return
+
+    const ctx = this.getCtx()
+    const gain = this.gainNode!
+
+    const target = this.duckTarget()
+
+    if (prevSource) {
+      gain.gain.cancelScheduledValues(ctx.currentTime)
+      gain.gain.setValueAtTime(gain.gain.value, ctx.currentTime)
+      gain.gain.exponentialRampToValueAtTime(0.001, ctx.currentTime + crossFade)
+      try { prevSource.stop(ctx.currentTime + crossFade + 0.1) } catch {}
+    }
+
+    const source = ctx.createBufferSource()
+    source.buffer = buffer
+    source.loop = true
+    source.connect(gain)
+
+    gain.gain.cancelScheduledValues(ctx.currentTime)
+    gain.gain.setValueAtTime(0.001, ctx.currentTime)
+    gain.gain.exponentialRampToValueAtTime(
+      Math.max(0.001, target),
+      ctx.currentTime + crossFade,
+    )
+
+    source.start(0)
+    this.currentSource = source
+  }
+
+  async stop(fadeOut: number) {
+    if (!this.currentSource || !this.gainNode) return
+    const ctx = this.getCtx()
+    this.currentUrl = ''
+    const gain = this.gainNode
+    try {
+      gain.gain.cancelScheduledValues(ctx.currentTime)
+      gain.gain.setValueAtTime(gain.gain.value, ctx.currentTime)
+      gain.gain.exponentialRampToValueAtTime(0.001, ctx.currentTime + fadeOut)
+      const src = this.currentSource
+      setTimeout(() => {
+        try { src.stop() } catch {}
+      }, (fadeOut + 0.1) * 1000)
+    } catch {}
+    this.currentSource = null
+  }
+
+  duckOn(reason: DuckReason) {
+    this.duckCount.set(reason, (this.duckCount.get(reason) ?? 0) + 1)
+    this.applyDuck()
+  }
+
+  duckOff(reason: DuckReason) {
+    const cur = this.duckCount.get(reason) ?? 0
+    if (cur <= 1) {
+      this.duckCount.delete(reason)
+    } else {
+      this.duckCount.set(reason, cur - 1)
+    }
+    this.applyDuck()
+  }
+
+  private applyDuck() {
+    if (!this.gainNode || !this.currentSource) return
+    const target = this.duckTarget()
+    const ctx = this.getCtx()
+    const dt = this.duckCount.size > 0 ? this.duckFade : this.duckFade
+    this.gainNode.gain.cancelScheduledValues(ctx.currentTime)
+    this.gainNode.gain.setValueAtTime(this.gainNode.gain.value, ctx.currentTime)
+    this.gainNode.gain.exponentialRampToValueAtTime(
+      Math.max(0.001, target),
+      ctx.currentTime + dt,
+    )
+  }
+
+  private duckTarget(): number {
+    if (this.duckCount.size > 0) {
+      return this.currentVolume * this.duckLevel
+    }
+    return this.currentVolume
+  }
+
+  setVolumeRaw(v: number) {
+    if (!this.gainNode) return
+    const ctx = this.getCtx()
+    this.gainNode.gain.cancelScheduledValues(ctx.currentTime)
+    this.gainNode.gain.setValueAtTime(v, ctx.currentTime)
+  }
+
+  private async loadBuffer(url: string): Promise<AudioBuffer | null> {
+    const cached = this.bufferCache.get(url)
+    if (cached) return cached
+
+    try {
+      const resp = await fetch(url)
+      const arrayBuf = await resp.arrayBuffer()
+      const ctx = this.getCtx()
+      const buffer = await ctx.decodeAudioData(arrayBuf)
+      this.bufferCache.set(url, buffer)
+
+      // LRU eviction
+      if (this.bufferCache.size > this.maxCache) {
+        const firstKey = this.bufferCache.keys().next().value
+        if (firstKey) this.bufferCache.delete(firstKey)
+      }
+
+      return buffer
+    } catch {
+      return null
+    }
+  }
+
+  destroy() {
+    if (this.currentSource) {
+      try { this.currentSource.stop() } catch {}
+      this.currentSource = null
+    }
+    this.bufferCache.clear()
+    this.duckCount.clear()
+    this.currentUrl = ''
+    if (this.ctx) {
+      this.ctx.close()
+      this.ctx = null
+      this.gainNode = null
+    }
+  }
+}

engine/types.ts

@@ -11,6 +11,12 @@ export interface SceneNode {
   onEnter?: Effect[]
   loopStart?: number
   loopEnd?: number
+  bgmUrl?: string
+  bgmVolume?: number
+  bgmCrossFade?: number
+  bgmDuckLevel?: number
+  bgmDuckFade?: number
+  videoMuted?: boolean
 }
 
 export interface Choice {