feat: phase1-2 baseline echo scaffold

.gitignore

@@ -0,0 +1,23 @@
+pycache/
+*.py[cod]
+*.pyo
+*.pyd
+*.so
+*.egg-info/
+dist/
+build/
+.env
+.venv
+venv/
+env/
+node_modules/
+*.log
+.vscode/
+.idea/
+.DS_Store
+Thumbs.db
+pip-wheel-metadata/
+.cache/
+coverage/
+models/
+checkpoints/

Dockerfile

@@ -0,0 +1,8 @@
+FROM python:3.11-slim
+ENV PYTHONUNBUFFERED=1
+WORKDIR /app
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+COPY . /app
+EXPOSE 7860
+CMD ["uvicorn","app:app","--host","0.0.0.0","--port","7860"]

README.md

@@ -1,12 +1,25 @@
----
-title: Mirage
-emoji: 👀
-colorFrom: indigo
-colorTo: indigo
-sdk: docker
-pinned: false
-license: apache-2.0
-short_description: seemingly real
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Mirage
+
+Phase 1–2 FastAPI + WebSocket echo scaffold (no ML models yet).
+
+## Current Status
+- CPU baseline
+- `/audio` & `/video` echo
+- Static client
+
+## Planned Phases
+- GPU switch
+- Metrics
+- Voice skeleton
+- Video skeleton
+- Adaptation
+- Security
+
+## Local Run
+```bash
+pip install -r requirements.txt
+uvicorn app:app --port 7860
+```
+
+## License
+MIT

SPEC.md

@@ -0,0 +1,93 @@
+## Goals
+- End-to-end audio latency < 250 ms (capture -> inference -> playback)
+- Video pipeline: 512x512 @ ≥20 FPS target under load
+
+## Hardware Target
+- Phase 1–2: CPU basic (development + echo scaffolds)
+- Later phases: Single NVIDIA A10G (24GB) for combined audio + video low-latency inference
+
+## Voice Pipeline
+| Item | Planned |
+|------|---------|
+| Framework | TODO |
+| Content Encoder | TODO |
+| F0 Extractor | RMVPE |
+| Chunk Size | 192 ms |
+| Sample Rate | 16 kHz |
+| Precision | FP16 mixed |
+| Overlap-Add | disabled |
+| Accept Threshold | < 0.65 * chunk_ms ratio runtime/real |
+| Fail Condition | > 0.80 * chunk_ms for 40 consecutive chunks |
+
+## Video Pipeline
+| Item | Planned |
+|------|---------|
+| Model | TODO |
+| Detector | SCRFD |
+| Detect Interval | 5 frames |
+| Resolution | 512x512 |
+| FPS Target | 20 |
+| Confidence Threshold (stable) | ≥0.85 |
+| Re-detect Threshold | <0.70 confidence triggers re-detect next frame |
+| Quality Degrade Order | quality → fps → resolution |
+
+## Transport
+- WebSockets (bi-directional control + media chunks)
+- Audio: PCM16, 16 kHz, mono frames (chunked ~192 ms) 
+- Video: JPEG compressed frames (progressive baseline) initially
+
+## Sync Strategy
+- Audio clock is master timeline
+- Drop late video frames if video timestamp >150 ms behind audio head
+- Never let audio lead video by more than 150 ms (else request video degrade)
+- If sustained drift (>120 frames) re-align by soft skip (video) not audio stretch
+
+## Metrics
+Planned collection (no aggregation service yet):
+- Audio chunks processed (count, accept ratio)
+- Per-stage timings: encode / f0 / convert / post
+- Video frames processed & dropped
+- Detector invoke interval stats
+- GPU memory (allocated / reserved / fragmentation indicator)
+- Queue backlog lengths (audio, video, outbound)
+
+## Adaptation Rules
+- Increase audio chunk size if runtime/real ratio >0.75 for last 40 chunks
+- Decrease audio chunk size if runtime/real ratio <0.55 for last 120 chunks
+- Video degrade order:
+  1. Lower quality (JPEG quality step down)
+  2. Reduce FPS toward 15
+  3. Reduce resolution (512 → 384 → 256)
+- Restore in reverse order after 300 stable frames (<0.60 ratio)
+
+## Security
+- Planned: JWT token required for WebSocket upgrade
+- Rate limiting (connection + message frequency)
+- Frame size guard (reject > configured max bytes)
+- Basic anomaly detection: abandon session if >30% frames invalid over 200 frame window
+
+## Licensing
+- Current: MIT
+- Future: Add `LICENSES.md` enumerating third-party components and model licenses
+
+## Phases
+| Phase | Status |
+|-------|--------|
+| 1 | Pending |
+| 2 | Pending |
+| 3 | Pending |
+| 4 | Pending |
+| 5 | Pending |
+| 6 | Pending |
+| 7 | Pending |
+| 8 | Pending |
+| 9 | Pending |
+| 10 | Pending |
+
+## Open Questions
+- RVC fork URL to adopt / baseline? (Need candidate repo)
+- Reenact / face animation model repo selection?
+- Alignment expectation: do we need phoneme-level alignment or chunk-level only?
+
+## Face Detector Standardization
+Adopt SCRFD as unified face detector. Run detection every 5 frames; reuse boxes otherwise. Force immediate re-detect if last confidence <0.70. Consider face track stable when confidence ≥0.85 for 3 consecutive detections. This reduces detector load while maintaining robustness against drift and occlusion.

app.py

@@ -0,0 +1,62 @@
+from fastapi import FastAPI, WebSocket, WebSocketDisconnect
+from fastapi.responses import HTMLResponse
+from fastapi.staticfiles import StaticFiles
+from pathlib import Path
+import traceback
+
+app = FastAPI(title="Mirage Phase 1+2 Scaffold")
+
+# Mount the static directory
+static_dir = Path(__file__).parent / "static"
+app.mount("/static", StaticFiles(directory=str(static_dir)), name="static")
+
+
+@app.get("/", response_class=HTMLResponse)
+async def root():
+    """Serve the static/index.html file contents as HTML."""
+    index_path = static_dir / "index.html"
+    try:
+        content = index_path.read_text(encoding="utf-8")
+    except FileNotFoundError:
+        # Minimal fallback to satisfy route even if file not yet present.
+        content = "<html><body><h1>Mirage Scaffold</h1><p>Place an index.html in /static.</p></body></html>"
+    return HTMLResponse(content)
+
+
+@app.get("/health")
+async def health():
+    return {"status": "ok", "phase": "baseline"}
+
+
+async def _echo_websocket(websocket: WebSocket, kind: str):
+    await websocket.accept()
+    while True:
+        try:
+            data = await websocket.receive_bytes()
+            # Echo straight back
+            await websocket.send_bytes(data)
+        except WebSocketDisconnect:
+            # Silent disconnect
+            break
+        except Exception:  # noqa: BLE001
+            # Print traceback for unexpected errors, then break loop
+            print(f"[{kind} ws] Unexpected error:")
+            traceback.print_exc()
+            break
+
+
+@app.websocket("/audio")
+async def audio_ws(websocket: WebSocket):
+    await _echo_websocket(websocket, "audio")
+
+
+@app.websocket("/video")
+async def video_ws(websocket: WebSocket):
+    await _echo_websocket(websocket, "video")
+
+
+# Note: The Dockerfile / README launch with: uvicorn app:app --port 7860
+if __name__ == "__main__":  # Optional direct run helper
+    import uvicorn  # type: ignore
+
+    uvicorn.run("app:app", host="0.0.0.0", port=7860, reload=False)

requirements.txt

@@ -0,0 +1,4 @@
+fastapi
+uvicorn[standard]
+websockets
+jinja2

static/app.js

@@ -0,0 +1,224 @@
+/* Mirage Echo Baseline Client */
+
+// Globals (scoped to this module)
+let audioWs = null;
+let videoWs = null;
+let audioContext = null;
+let processorNode = null; // AudioWorkletNode for capturing (pcm-chunker)
+let playerNode = null; // AudioWorkletNode for playback (pcm-player)
+let lastVideoSentTs = 0;
+let remoteImageURL = null;
+
+const LOG_EL = document.getElementById('log');
+const START_BTN = document.getElementById('startBtn');
+const LOCAL_VID = document.getElementById('localVid');
+const REMOTE_VID_IMG = document.getElementById('remoteVid');
+const REMOTE_AUDIO = document.getElementById('remoteAudio');
+
+function log(msg) {
+  const ts = new Date().toISOString().split('T')[1].replace('Z','');
+  LOG_EL.textContent += `[${ts}] ${msg}\n`;
+  LOG_EL.scrollTop = LOG_EL.scrollHeight;
+}
+
+function wsURL(path) {
+  const proto = (location.protocol === 'https:') ? 'wss:' : 'ws:';
+  return `${proto}//${location.host}${path}`;
+}
+
+async function setupAudio(stream) {
+  audioContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 16000 });
+  if (audioContext.state === 'suspended') {
+    try { await audioContext.resume(); } catch (e) { log('AudioContext resume failed'); }
+  }
+
+  // Worklet loading
+  try {
+    await audioContext.audioWorklet.addModule('/static/worklet.js');
+  } catch (e) {
+    log('Failed to load worklet.js (pcm-chunker) - audio sending disabled.');
+    console.error(e);
+    return;
+  }
+
+  const samplesPerChunk = Math.round(audioContext.sampleRate * 0.25); // 250 ms -> spec said 0.25 (192ms earlier spec but instruction uses 0.25 here)
+  log(`AudioContext sampleRate=${audioContext.sampleRate}, samplesPerChunk=${samplesPerChunk}`);
+  processorNode = new AudioWorkletNode(audioContext, 'pcm-chunker', {
+    processorOptions: { samplesPerChunk }
+  });
+  playerNode = new AudioWorkletNode(audioContext, 'pcm-player');
+
+  // Capture mic
+  const source = audioContext.createMediaStreamSource(stream);
+  source.connect(processorNode);
+  // Keep worklet active via silent gain path (0 gain) to destination (some browsers optimize away otherwise)
+  const gain = audioContext.createGain();
+  gain.gain.value = 0;
+  processorNode.connect(gain).connect(audioContext.destination);
+  // Do NOT connect processorNode to destination to avoid local direct monitor; playback handled by pcm-player.
+
+  processorNode.port.onmessage = (event) => {
+    if (!audioWs || audioWs.readyState !== WebSocket.OPEN) return;
+    const ab = event.data;
+    if (ab instanceof ArrayBuffer) audioWs.send(ab);
+  };
+
+  // Connect playback node
+  playerNode.connect(audioContext.destination);
+  log('Audio nodes ready (pcm-chunker + pcm-player)');
+}
+
+let _rxChunks = 0;
+let _loopback = false;
+function setupAudioWebSocket() {
+  audioWs = new WebSocket(wsURL('/audio'));
+  audioWs.binaryType = 'arraybuffer';
+  audioWs.onopen = () => log('Audio WS open');
+  audioWs.onclose = () => log('Audio WS closed');
+  audioWs.onerror = (e) => log('Audio WS error');
+  audioWs.onmessage = (evt) => {
+    if (!(evt.data instanceof ArrayBuffer)) return;
+    // Clone buffer BEFORE transferring to avoid ArrayBuffer detachment errors when reusing
+    const src = evt.data;
+    const copyBuf = src.slice(0); // shallow copy; original remains intact for stats
+    // Amplitude stats (compute on copy or original before transfer)
+    const view = new Int16Array(src);
+    let min = 32767, max = -32768;
+    for (let i=0;i<view.length;i++) { const v=view[i]; if (v<min) min=v; if (v>max) max=v; }
+    // Forward copy to player (transfer copy to avoid overhead next GC cycle)
+    if (playerNode) playerNode.port.postMessage(copyBuf, [copyBuf]);
+    _rxChunks++;
+    if ((_rxChunks % 20) === 0) {
+      log(`Audio chunks received: ${_rxChunks} amp:[${min},${max}]`);
+    }
+    if (_loopback && audioWs && audioWs.readyState === WebSocket.OPEN) {
+      // echo back again (will double) purely for test; guard to prevent infinite recursion (already from server)
+    }
+  };
+}
+
+async function setupVideo(stream) {
+  const track = stream.getVideoTracks()[0];
+  if (!track) {
+    log('No video track found');
+    return;
+  }
+  const processor = new MediaStreamTrackProcessor({ track });
+  const reader = processor.readable.getReader();
+
+  const canvas = document.createElement('canvas');
+  canvas.width = 256;
+  canvas.height = 256;
+  const ctx = canvas.getContext('2d');
+
+  async function readLoop() {
+    try {
+      const { value: frame, done } = await reader.read();
+      if (done) return;
+
+      const now = performance.now();
+      const needSend = (now - lastVideoSentTs) >= 100; // ~10 fps
+
+      if (needSend && frame) {
+        try {
+          // Draw frame
+          if ('displayWidth' in frame && 'displayHeight' in frame) {
+            ctx.drawImage(frame, 0, 0, canvas.width, canvas.height);
+          } else {
+            // Fallback path: createImageBitmap then draw
+            const bmp = await createImageBitmap(frame);
+            ctx.drawImage(bmp, 0, 0, canvas.width, canvas.height);
+            bmp.close && bmp.close();
+          }
+
+          await new Promise((res, rej) => {
+            canvas.toBlob((blob) => {
+              if (!blob) return res();
+              blob.arrayBuffer().then((ab) => {
+                if (videoWs && videoWs.readyState === WebSocket.OPEN) {
+                  videoWs.send(ab);
+                }
+                res();
+              }).catch(rej);
+            }, 'image/jpeg', 0.65);
+          });
+          lastVideoSentTs = now;
+        } catch (err) {
+          log('Video frame send error');
+          console.error(err);
+        }
+      }
+
+      frame.close && frame.close();
+      readLoop();
+    } catch (err) {
+      log('Video read loop error');
+      console.error(err);
+    }
+  }
+  readLoop();
+}
+
+function setupVideoWebSocket() {
+  videoWs = new WebSocket(wsURL('/video'));
+  videoWs.binaryType = 'arraybuffer';
+  videoWs.onopen = () => log('Video WS open');
+  videoWs.onclose = () => log('Video WS closed');
+  videoWs.onerror = () => log('Video WS error');
+  videoWs.onmessage = (evt) => {
+    if (!(evt.data instanceof ArrayBuffer)) return;
+    const blob = new Blob([evt.data], { type: 'image/jpeg' });
+    if (remoteImageURL) URL.revokeObjectURL(remoteImageURL);
+    remoteImageURL = URL.createObjectURL(blob);
+    REMOTE_VID_IMG.src = remoteImageURL;
+  };
+}
+
+async function start() {
+  START_BTN.disabled = true;
+  log('Requesting media...');
+  let stream;
+  try {
+    stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: true });
+  } catch (e) {
+    log('getUserMedia failed');
+    console.error(e);
+    START_BTN.disabled = false;
+    return;
+  }
+  LOCAL_VID.srcObject = stream;
+  log('Media acquired');
+
+  setupAudioWebSocket();
+  setupVideoWebSocket();
+  await setupAudio(stream);
+  await setupVideo(stream);
+}
+
+START_BTN.addEventListener('click', start);
+
+// Expose for debugging
+function testTone(seconds = 1, freq = 440) {
+  if (!audioContext || !playerNode) { log('testTone: audio not ready'); return; }
+  const sampleRate = audioContext.sampleRate;
+  const total = Math.floor(sampleRate * seconds);
+  const int16 = new Int16Array(total);
+  for (let i=0;i<total;i++) {
+    const s = Math.sin(2 * Math.PI * freq * (i / sampleRate));
+    int16[i] = s * 32767;
+  }
+  // slice into chunk-sized buffers similar to inbound network flow
+  const chunk = Math.floor(sampleRate * 0.25);
+  for (let off = 0; off < int16.length; off += chunk) {
+    const view = int16.subarray(off, Math.min(off + chunk, int16.length));
+    // copy to standalone buffer for transfer
+    const copy = new Int16Array(view.length);
+    copy.set(view);
+    playerNode.port.postMessage(copy.buffer, [copy.buffer]);
+  }
+  log(`Injected test tone ${freq}Hz for ${seconds}s`);
+}
+
+window.__mirage = { start, audioWs: () => audioWs, videoWs: () => videoWs, testTone };
+// Diagnostics helpers
+window.__mirage.toggleLoopback = function(on){ _loopback = on !== undefined ? !!on : !_loopback; log('Local loopback=' + _loopback); };

static/index.html

@@ -0,0 +1,23 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8" />
+  <title>Mirage Echo Baseline</title>
+  <meta name="viewport" content="width=device-width,initial-scale=1" />
+  <style>
+    video, img { width: 300px; }
+    #log { font: 11px/1.3 -apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Helvetica,Arial,sans-serif,monospace; white-space: pre-line; }
+  </style>
+</head>
+<body>
+  <h1>Mirage Echo Baseline</h1>
+  <button id="startBtn">Start</button>
+  <div>
+    <video id="localVid" autoplay muted playsinline></video>
+    <img id="remoteVid" alt="remote video frame" />
+  </div>
+  <audio id="remoteAudio" autoplay></audio>
+  <div id="log"></div>
+  <script src="/static/app.js"></script>
+</body>
+</html>

static/worklet.js

@@ -0,0 +1,86 @@
+class PCMChunker extends AudioWorkletProcessor {
+  constructor(options) {
+    super();
+    this.samplesPerChunk = (options && options.processorOptions && options.processorOptions.samplesPerChunk) || 16000;
+    this.buffer = new Float32Array(this.samplesPerChunk);
+    this.offset = 0;
+  }
+
+  process(inputs) {
+    const input = inputs[0];
+    if (input && input[0]) {
+      const data = input[0];
+      let i = 0;
+      while (i < data.length) {
+        const space = this.samplesPerChunk - this.offset;
+        const toCopy = Math.min(space, data.length - i);
+        this.buffer.set(data.subarray(i, i + toCopy), this.offset);
+        this.offset += toCopy;
+        i += toCopy;
+        if (this.offset >= this.samplesPerChunk) {
+          const out = new Int16Array(this.samplesPerChunk);
+          for (let j = 0; j < this.samplesPerChunk; j++) {
+            let s = this.buffer[j];
+            if (s > 1) s = 1; else if (s < -1) s = -1;
+            out[j] = s < 0 ? s * 32768 : s * 32767;
+          }
+          const buf = out.buffer;
+          this.port.postMessage(buf, [buf]);
+          this.offset = 0;
+        }
+      }
+    }
+    return true;
+  }
+}
+
+registerProcessor('pcm-chunker', PCMChunker);
+
+// PCM player pulls Int16 buffers from a queue pushed via port messages and outputs Float32 samples.
+class PCMPlayer extends AudioWorkletProcessor {
+  constructor() {
+    super();
+    this.queue = [];
+    this.current = null;
+    this.offset = 0;
+    this.samplesPerBuffer = 0;
+    this.port.onmessage = (e) => {
+      const d = e.data;
+      if (d instanceof ArrayBuffer) {
+        this.queue.push(new Int16Array(d));
+      } else if (d instanceof Int16Array) {
+        this.queue.push(d);
+      }
+    };
+  }
+  process(_inputs, outputs) {
+    const output = outputs[0][0];
+    if (!output) return true;
+    let i = 0;
+    while (i < output.length) {
+      if (!this.current) {
+        this.current = this.queue.shift();
+        this.offset = 0;
+        if (!this.current) {
+          // Fill rest with silence
+          while (i < output.length) output[i++] = 0;
+          break;
+        }
+      }
+      const remain = this.current.length - this.offset;
+      const needed = output.length - i;
+      const toCopy = Math.min(remain, needed);
+      for (let j = 0; j < toCopy; j++) {
+        output[i + j] = this.current[this.offset + j] / 32768;
+      }
+      i += toCopy;
+      this.offset += toCopy;
+      if (this.offset >= this.current.length) {
+        this.current = null;
+      }
+    }
+    return true;
+  }
+}
+
+registerProcessor('pcm-player', PCMPlayer);