chore: ignore local worktrees

feat: add scene asset store
fix: validate scene asset edge cases
docs: record task 1 review fix
feat: expose scene asset api
test api composition deletion
feat: add scene assets to asset library
Fix scene asset library integration
Update asset tab test expectations
feat: render realtime stage with scene assets
feat: add immersive conversation mode
Gate immersive mode during startup
feat: surface avatar matting readiness
Fix scene asset selection and upload validation
Fix scene avatar anchor rendering
Fix unified scene asset routes
Stabilize unified scene route test
Fix default realtime stage scene behavior
Keep realtime video across view switches
Refine immersive chrome and keyboard handling
Preserve audio when switching realtime views
Keep realtime media mounted in immersive mode
Restore realtime audio and refine immersive controls
Scope scene backgrounds to matching avatar
Preserve transparent avatar backgrounds in scenes
Keep WebRTC remote stream identity
Fallback to muted stage video playback
Refine immersive QuickTalk controls
Make immersive adjustment panel opaque
Expand immersive adjustment ranges
Use avatar-specific scene defaults
Fix mypy type narrowing for scene assets
This commit is contained in:
kero-ly
2026-06-21 21:35:21 +08:00
committed by zyairehhh
parent 2eb85ffe4d
commit 57b3e48718
28 changed files with 2235 additions and 82 deletions

1
.gitignore vendored
View File

@@ -79,6 +79,7 @@ machine.md
# Internal dev specs (not for public repo)
docs/superpowers/
.worktrees/
# MkDocs build output
site/

View File

@@ -9,7 +9,7 @@ from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from apps.api.core.config import get_settings
from apps.api.routes import agent, avatars, events, exports, health, memory, models, personas, runtime_config, sessions, tts_preview, video_clone, video_creation, voices
from apps.api.routes import agent, avatars, events, exports, health, memory, models, personas, runtime_config, scene_assets, sessions, tts_preview, video_clone, video_creation, voices
from opentalking.voice.store import init_voice_store
@@ -43,6 +43,7 @@ def create_app() -> FastAPI:
app.include_router(personas.router)
app.include_router(events.router)
app.include_router(exports.router)
app.include_router(scene_assets.router)
app.include_router(runtime_config.router)
app.include_router(tts_preview.router)
app.include_router(video_clone.router)

View File

@@ -60,6 +60,15 @@ def _is_hidden_avatar(manifest_path: Path) -> bool:
return bool((raw.get("metadata") or {}).get("hidden"))
def _avatar_matting_status(manifest_path: Path) -> str:
try:
raw = json.loads(manifest_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return "unknown"
value = str((raw.get("metadata") or {}).get("matting_status") or "").strip()
return value if value in {"unknown", "opaque", "transparent_ready"} else "unknown"
def _avatar_preview_video_path(avatar_dir: Path) -> Path | None:
manifest_path = avatar_dir / "manifest.json"
try:
@@ -112,6 +121,7 @@ def _summary_from_dir(path: Path) -> AvatarSummary:
height=m.height,
is_custom=_is_custom_avatar(path / "manifest.json"),
has_preview_video=_avatar_preview_video_path(path) is not None,
matting_status=_avatar_matting_status(path / "manifest.json"),
)
@@ -144,7 +154,7 @@ async def _read_upload_image(upload: UploadFile) -> Image.Image:
image.load()
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=400, detail="invalid image") from exc
return image.convert("RGB")
return image.convert("RGBA") if "A" in image.getbands() else image.convert("RGB")
async def _read_upload_video(upload: UploadFile) -> tuple[Image.Image, bytes, str]:
@@ -209,6 +219,7 @@ def _write_custom_avatar_manifest(
metadata["idle_mode"] = "static"
metadata["reference_mode"] = "image"
metadata["source_image"] = "source/source.png"
metadata["matting_status"] = "opaque"
raw["metadata"] = metadata
target_manifest_path.write_text(json.dumps(raw, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
@@ -235,6 +246,24 @@ def _resize_uploaded_avatar_image(image: Image.Image, *, max_width: int, max_hei
return fitted
def _avatar_image_has_alpha(image: Image.Image) -> bool:
if "A" not in image.getbands():
return False
alpha = image.getchannel("A")
low, high = alpha.getextrema()
if not isinstance(low, int | float) or not isinstance(high, int | float):
return False
return low < 255 or high < 255
def _update_manifest_matting_status(manifest_path: Path, image: Image.Image) -> None:
raw = json.loads(manifest_path.read_text(encoding="utf-8"))
metadata = dict(raw.get("metadata") or {})
metadata["matting_status"] = "transparent_ready" if _avatar_image_has_alpha(image) else "opaque"
raw["metadata"] = metadata
manifest_path.write_text(json.dumps(raw, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def _update_manifest_dimensions(manifest_path: Path, image: Image.Image) -> None:
raw = json.loads(manifest_path.read_text(encoding="utf-8"))
raw["width"] = int(image.width)
@@ -916,6 +945,7 @@ async def create_custom_avatar(
max_w, max_h = _custom_avatar_max_size()
fitted_image = _resize_uploaded_avatar_image(image_rgb, max_width=max_w, max_height=max_h)
_update_manifest_dimensions(target_dir / "manifest.json", fitted_image)
_update_manifest_matting_status(target_dir / "manifest.json", fitted_image)
fitted_image.save(target_dir / "preview.png", format="PNG")
fitted_image.save(target_dir / "reference.png", format="PNG")
source_dir = target_dir / "source"

View File

@@ -0,0 +1,104 @@
from __future__ import annotations
from pathlib import Path
from fastapi import APIRouter, File, Form, HTTPException, Request, UploadFile
from fastapi.responses import FileResponse
from apps.api.schemas.scene_assets import (
CreateSceneCompositionRequest,
UpdateSceneCompositionRequest,
)
from opentalking.scene_assets import SceneAssetStore
router = APIRouter(prefix="/scene-assets", tags=["scene-assets"])
def _store(request: Request) -> SceneAssetStore:
settings = request.app.state.settings
root = Path(getattr(settings, "scene_assets_dir", "./data/scene-assets"))
return SceneAssetStore(root)
@router.get("/backgrounds", response_model=None)
async def list_backgrounds(request: Request) -> dict[str, object]:
return {"items": _store(request).list_backgrounds()}
@router.post("/backgrounds", response_model=None)
async def upload_background(
request: Request,
file: UploadFile = File(...),
name: str = Form(""),
) -> dict[str, object]:
max_bytes = int(getattr(request.app.state.settings, "scene_asset_max_bytes", 200 * 1024 * 1024))
content = await file.read(max_bytes + 1)
if not content:
raise HTTPException(status_code=400, detail="empty background asset")
if len(content) > max_bytes:
raise HTTPException(status_code=413, detail="background asset exceeds size limit")
try:
return _store(request).create_background(
content=content,
filename=file.filename or "background",
mime_type=file.content_type or "application/octet-stream",
name=name or Path(file.filename or "background").stem,
)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@router.get("/backgrounds/{background_id}/file", response_model=None)
async def download_background(background_id: str, request: Request) -> FileResponse:
store = _store(request)
path = store.background_file_path(background_id)
if path is None:
raise HTTPException(status_code=404, detail="background not found")
item = next((entry for entry in store.list_backgrounds() if entry.get("id") == background_id), None)
return FileResponse(path, media_type=str(item.get("mime_type") if item else "application/octet-stream"))
@router.delete("/backgrounds/{background_id}", response_model=None)
async def delete_background(background_id: str, request: Request) -> dict[str, object]:
deleted = _store(request).delete_background(background_id)
if not deleted:
raise HTTPException(status_code=404, detail="background not found")
return {"id": background_id, "deleted": True}
@router.get("/compositions", response_model=None)
async def list_compositions(request: Request) -> dict[str, object]:
return {"items": _store(request).list_compositions()}
@router.post("/compositions", response_model=None)
async def create_composition(payload: CreateSceneCompositionRequest, request: Request) -> dict[str, object]:
try:
return _store(request).create_composition(payload.model_dump())
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@router.patch("/compositions/{composition_id}", response_model=None)
async def update_composition(
composition_id: str,
payload: UpdateSceneCompositionRequest,
request: Request,
) -> dict[str, object]:
data = payload.model_dump(exclude_unset=True)
try:
updated = _store(request).update_composition(composition_id, data)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
if updated is None:
raise HTTPException(status_code=404, detail="composition not found")
return updated
@router.delete("/compositions/{composition_id}", response_model=None)
async def delete_composition(composition_id: str, request: Request) -> dict[str, object]:
deleted = _store(request).delete_composition(composition_id)
if not deleted:
raise HTTPException(status_code=404, detail="composition not found")
return {"id": composition_id, "deleted": True}

View File

@@ -14,3 +14,4 @@ class AvatarSummary(BaseModel):
# True for avatars created via POST /avatars/custom; only these are deletable.
is_custom: bool = False
has_preview_video: bool = False
matting_status: str = "unknown"

View File

@@ -0,0 +1,53 @@
from __future__ import annotations
from pydantic import BaseModel, Field
class SceneBackgroundAsset(BaseModel):
id: str
name: str
kind: str
mime_type: str
filename: str
size_bytes: int
url: str
created_at: str
class SceneComposition(BaseModel):
id: str
name: str
avatar_id: str
background_id: str | None = None
background_color: str = "#0f172a"
avatar_fit: str = "contain"
avatar_scale: float = Field(default=1.0, ge=0.5, le=2.0)
avatar_anchor: str = "center"
matting_required: bool = False
subtitle_style: str = "lower-third"
created_at: str
updated_at: str
class CreateSceneCompositionRequest(BaseModel):
name: str
avatar_id: str
background_id: str | None = None
background_color: str = "#0f172a"
avatar_fit: str = "contain"
avatar_scale: float = Field(default=1.0, ge=0.5, le=2.0)
avatar_anchor: str = "center"
matting_required: bool = False
subtitle_style: str = "lower-third"
class UpdateSceneCompositionRequest(BaseModel):
name: str | None = None
avatar_id: str | None = None
background_id: str | None = None
background_color: str | None = None
avatar_fit: str | None = None
avatar_scale: float | None = Field(default=None, ge=0.5, le=2.0)
avatar_anchor: str | None = None
matting_required: bool | None = None
subtitle_style: str | None = None

View File

@@ -1,6 +1,7 @@
from __future__ import annotations
import json
import hashlib
from io import BytesIO
from pathlib import Path
from types import SimpleNamespace
@@ -12,6 +13,8 @@ from PIL import Image
from apps.cli import prepare_cache
from apps.api.routes import avatars, sessions
REPO_ROOT = Path(__file__).resolve().parents[3]
def _png_bytes(size: tuple[int, int] = (8, 8)) -> bytes:
out = BytesIO()
@@ -19,6 +22,14 @@ def _png_bytes(size: tuple[int, int] = (8, 8)) -> bytes:
return out.getvalue()
def _transparent_png_bytes(size: tuple[int, int] = (8, 8)) -> bytes:
image = Image.new("RGBA", size, (10, 180, 210, 255))
image.putpixel((0, 0), (10, 180, 210, 0))
out = BytesIO()
image.save(out, format="PNG")
return out.getvalue()
def _path_has_suffix(path: str, *suffix: str) -> bool:
return Path(path).parts[-len(suffix):] == suffix
@@ -74,6 +85,106 @@ def test_create_custom_avatar_adds_listed_asset_with_preview(tmp_path):
assert preview.headers["content-type"] == "image/png"
def test_avatar_summary_includes_matting_status(tmp_path: Path) -> None:
root = tmp_path
avatar_dir = root / "anchor"
avatar_dir.mkdir()
(avatar_dir / "manifest.json").write_text(
json.dumps(
{
"id": "anchor",
"name": "Anchor",
"model_type": "flashtalk",
"fps": 25,
"sample_rate": 16000,
"width": 1,
"height": 1,
"version": "1.0",
"metadata": {"matting_status": "transparent_ready"},
}
),
encoding="utf-8",
)
(avatar_dir / "preview.png").write_bytes(_png_bytes())
(avatar_dir / "reference.png").write_bytes(_png_bytes())
app = FastAPI()
app.state.settings = SimpleNamespace(avatars_dir=str(root))
app.include_router(avatars.router)
response = TestClient(app).get("/avatars")
assert response.status_code == 200
assert response.json()[0]["matting_status"] == "transparent_ready"
def test_builtin_female_host_transparent_avatar_asset_is_ready() -> None:
avatar_dir = REPO_ROOT / "examples" / "avatars" / "female-host-transparent"
manifest = json.loads((avatar_dir / "manifest.json").read_text(encoding="utf-8"))
reference = avatar_dir / "reference.png"
preview = avatar_dir / "preview.png"
source = avatar_dir / "source" / "source.png"
assert manifest["id"] == "female-host-transparent"
assert manifest["name"] == "女主持(透明背景)"
assert manifest["metadata"]["matting_status"] == "transparent_ready"
assert manifest["metadata"]["source_image"] == "source/source.png"
assert manifest["metadata"]["source_image_path"] == "reference.png"
assert reference.is_file()
assert preview.is_file()
assert source.is_file()
image = Image.open(reference)
assert image.mode == "RGBA"
assert image.size == (manifest["width"], manifest["height"])
assert image.getchannel("A").getextrema() == (0, 255)
assert hashlib.sha256(reference.read_bytes()).hexdigest() == manifest["metadata"]["source_image_hash"]
def test_create_custom_avatar_preserves_uploaded_png_alpha(tmp_path, monkeypatch):
base = tmp_path / "base-avatar"
base.mkdir()
(base / "preview.png").write_bytes(_png_bytes())
(base / "reference.png").write_bytes(_png_bytes())
(base / "manifest.json").write_text(
json.dumps(
{
"id": "base-avatar",
"name": "Base Avatar",
"model_type": "mock",
"fps": 25,
"sample_rate": 16000,
"width": 8,
"height": 8,
"version": "1.0",
}
),
encoding="utf-8",
)
monkeypatch.setattr(avatars.mouth_metadata, "detect_mouth_landmarks", lambda frame: None)
app = FastAPI()
app.state.settings = SimpleNamespace(avatars_dir=str(tmp_path))
app.include_router(avatars.router)
client = TestClient(app)
response = client.post(
"/avatars/custom",
data={"base_avatar_id": "base-avatar", "name": "透明形象"},
files={"image": ("avatar.png", _transparent_png_bytes(), "image/png")},
)
assert response.status_code == 200
created = response.json()
assert created["matting_status"] == "transparent_ready"
custom_dir = tmp_path / created["id"]
manifest = json.loads((custom_dir / "manifest.json").read_text(encoding="utf-8"))
assert manifest["metadata"]["matting_status"] == "transparent_ready"
for rel in ("reference.png", "preview.png", "source/source.png"):
image = Image.open(custom_dir / rel)
assert image.mode == "RGBA"
assert image.getchannel("A").getextrema()[0] == 0
def test_quicktalk_model_root_falls_back_to_omnirt_model_root(tmp_path, monkeypatch):
monkeypatch.delenv("OPENTALKING_QUICKTALK_ASSET_ROOT", raising=False)
monkeypatch.delenv("OPENTALKING_QUICKTALK_MODEL_ROOT", raising=False)

View File

@@ -0,0 +1,290 @@
from __future__ import annotations
from pathlib import Path
from types import SimpleNamespace
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
import apps.unified.main as unified_main
from apps.api.routes import scene_assets
from opentalking.scene_assets import SceneAssetStore
PNG_BYTES = (
b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR"
b"\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15\xc4\x89"
b"\x00\x00\x00\rIDATx\x9cc\xf8\xff\xff?\x00\x05\xfe\x02\xfeA\xe2!\xbc"
b"\x00\x00\x00\x00IEND\xaeB`\x82"
)
def test_scene_asset_store_creates_lists_and_deletes_background(tmp_path: Path) -> None:
store = SceneAssetStore(tmp_path)
created = store.create_background(
content=PNG_BYTES,
filename="studio.png",
mime_type="image/png",
name="Studio",
)
assert created["id"].startswith("bg-")
assert created["name"] == "Studio"
assert created["kind"] == "image"
assert created["mime_type"] == "image/png"
assert created["url"] == f"/scene-assets/backgrounds/{created['id']}/file"
assert store.background_file_path(str(created["id"])).is_file()
assert store.list_backgrounds()[0]["id"] == created["id"]
assert store.delete_background(str(created["id"])) is True
assert store.list_backgrounds() == []
def test_scene_asset_store_rejects_unsupported_background_type(tmp_path: Path) -> None:
store = SceneAssetStore(tmp_path)
with pytest.raises(ValueError, match="unsupported background media type"):
store.create_background(
content=b"not media",
filename="notes.txt",
mime_type="text/plain",
name="Notes",
)
def test_scene_asset_store_rejects_spoofed_background_content(tmp_path: Path) -> None:
store = SceneAssetStore(tmp_path)
with pytest.raises(ValueError, match="unsupported background media type"):
store.create_background(
content=b"plain text pretending to be an image",
filename="studio.png",
mime_type="image/png",
name="Spoof",
)
def test_scene_asset_store_rejects_zero_avatar_scale(tmp_path: Path) -> None:
store = SceneAssetStore(tmp_path)
with pytest.raises(ValueError, match="avatar_scale must be between 0.5 and 2.0"):
store.create_composition(
{
"name": "Zero Scale",
"avatar_id": "anchor",
"avatar_scale": 0,
}
)
def test_scene_asset_store_rejects_unsafe_background_delete_id(tmp_path: Path) -> None:
store = SceneAssetStore(tmp_path / "assets")
outside_dir = tmp_path / "outside"
outside_dir.mkdir()
(outside_dir / "keep.txt").write_text("keep", encoding="utf-8")
unsafe_id = f"bg-valid/../../{outside_dir.name}"
store.backgrounds_dir.mkdir(parents=True)
store.background_index_path.write_text(
f'[{{"id": "{unsafe_id}", "mime_type": "image/png"}}]\n',
encoding="utf-8",
)
assert store.delete_background(unsafe_id) is False
assert (outside_dir / "keep.txt").is_file()
def test_scene_asset_store_creates_and_updates_composition(tmp_path: Path) -> None:
store = SceneAssetStore(tmp_path)
background = store.create_background(
content=PNG_BYTES,
filename="studio.png",
mime_type="image/png",
name="Studio",
)
created = store.create_composition(
{
"name": "Demo Scene",
"avatar_id": "anchor",
"background_id": background["id"],
"avatar_fit": "contain",
"avatar_scale": 1.0,
"avatar_anchor": "center",
"matting_required": True,
"subtitle_style": "lower-third",
}
)
assert created["id"].startswith("scene-")
assert created["background_id"] == background["id"]
assert created["matting_required"] is True
assert store.list_compositions()[0]["id"] == created["id"]
updated = store.update_composition(str(created["id"]), {"name": "Updated", "avatar_scale": 1.25})
assert updated is not None
assert updated["name"] == "Updated"
assert updated["avatar_scale"] == 1.25
assert updated["avatar_id"] == "anchor"
assert store.delete_composition(str(created["id"])) is True
assert store.list_compositions() == []
def test_scene_asset_store_rejects_unknown_background_id(tmp_path: Path) -> None:
store = SceneAssetStore(tmp_path)
with pytest.raises(ValueError, match="background_id not found"):
store.create_composition(
{
"name": "Missing Background",
"avatar_id": "anchor",
"background_id": "bg-missing",
}
)
background = store.create_background(
content=PNG_BYTES,
filename="studio.png",
mime_type="image/png",
name="Studio",
)
created = store.create_composition(
{
"name": "Demo Scene",
"avatar_id": "anchor",
"background_id": background["id"],
}
)
with pytest.raises(ValueError, match="background_id not found"):
store.update_composition(str(created["id"]), {"background_id": "bg-missing"})
def _client(tmp_path: Path) -> TestClient:
app = FastAPI()
app.state.settings = SimpleNamespace(
scene_assets_dir=str(tmp_path / "scene-assets"),
scene_asset_max_bytes=len(PNG_BYTES),
)
app.include_router(scene_assets.router)
return TestClient(app)
def test_scene_asset_api_uploads_lists_downloads_and_deletes_background(tmp_path: Path) -> None:
with _client(tmp_path) as client:
upload = client.post(
"/scene-assets/backgrounds",
data={"name": "Studio"},
files={"file": ("studio.png", PNG_BYTES, "image/png")},
)
assert upload.status_code == 200
item = upload.json()
assert item["name"] == "Studio"
assert item["url"] == f"/scene-assets/backgrounds/{item['id']}/file"
listed = client.get("/scene-assets/backgrounds")
assert listed.status_code == 200
assert listed.json()["items"][0]["id"] == item["id"]
downloaded = client.get(item["url"])
assert downloaded.status_code == 200
assert downloaded.headers["content-type"].startswith("image/png")
deleted = client.delete(f"/scene-assets/backgrounds/{item['id']}")
assert deleted.status_code == 200
assert deleted.json()["deleted"] is True
def test_scene_asset_api_rejects_oversized_background_upload(tmp_path: Path) -> None:
with _client(tmp_path) as client:
upload = client.post(
"/scene-assets/backgrounds",
data={"name": "Too Big"},
files={"file": ("studio.png", PNG_BYTES + b"x", "image/png")},
)
assert upload.status_code == 413
assert upload.json()["detail"] == "background asset exceeds size limit"
def test_scene_asset_api_rejects_spoofed_content_type(tmp_path: Path) -> None:
with _client(tmp_path) as client:
upload = client.post(
"/scene-assets/backgrounds",
data={"name": "Spoof"},
files={"file": ("studio.png", b"this is plain text", "image/png")},
)
assert upload.status_code == 400
assert upload.json()["detail"] == "unsupported background media type"
def test_scene_asset_api_manages_compositions(tmp_path: Path) -> None:
with _client(tmp_path) as client:
bg = client.post(
"/scene-assets/backgrounds",
data={"name": "Studio"},
files={"file": ("studio.png", PNG_BYTES, "image/png")},
).json()
created = client.post(
"/scene-assets/compositions",
json={
"name": "Demo Scene",
"avatar_id": "anchor",
"background_id": bg["id"],
"avatar_fit": "contain",
"avatar_scale": 1.0,
"avatar_anchor": "center",
"matting_required": True,
"subtitle_style": "lower-third",
},
)
assert created.status_code == 200
scene = created.json()
assert scene["background_id"] == bg["id"]
patched = client.patch(
f"/scene-assets/compositions/{scene['id']}",
json={"name": "Updated Scene", "avatar_scale": 1.2},
)
assert patched.status_code == 200
assert patched.json()["name"] == "Updated Scene"
assert patched.json()["avatar_scale"] == 1.2
listed = client.get("/scene-assets/compositions")
assert listed.status_code == 200
assert listed.json()["items"][0]["id"] == scene["id"]
deleted = client.delete(f"/scene-assets/compositions/{scene['id']}")
assert deleted.status_code == 200
assert deleted.json() == {"id": scene["id"], "deleted": True}
listed_after_delete = client.get("/scene-assets/compositions")
assert listed_after_delete.status_code == 200
assert listed_after_delete.json()["items"] == []
deleted_again = client.delete(f"/scene-assets/compositions/{scene['id']}")
assert deleted_again.status_code == 404
def test_scene_asset_api_rejects_unknown_background_id(tmp_path: Path) -> None:
with _client(tmp_path) as client:
created = client.post(
"/scene-assets/compositions",
json={
"name": "Missing Background",
"avatar_id": "anchor",
"background_id": "bg-missing",
},
)
assert created.status_code == 400
assert created.json()["detail"] == "background_id not found"
def test_unified_app_registers_scene_asset_routes() -> None:
app = unified_main.create_app()
paths = set(app.openapi()["paths"])
assert "/scene-assets/backgrounds" in paths
assert "/scene-assets/compositions" in paths

View File

@@ -19,7 +19,7 @@ from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from apps.api.core.config import get_settings
from apps.api.routes.avatars import _call_adapter_warmup
from apps.api.routes import agent, avatars, events, exports, health, memory, models, personas, runtime_config, sessions, tts_preview, video_clone, video_creation, voices
from apps.api.routes import agent, avatars, events, exports, health, memory, models, personas, runtime_config, scene_assets, sessions, tts_preview, video_clone, video_creation, voices
from opentalking.voice.store import init_voice_store
from opentalking.core.in_memory_redis import InMemoryRedis
from opentalking.pipeline.session.runner import SessionRunner
@@ -219,6 +219,7 @@ def create_app() -> FastAPI:
app.include_router(personas.router)
app.include_router(events.router)
app.include_router(exports.router)
app.include_router(scene_assets.router)
app.include_router(runtime_config.router)
app.include_router(tts_preview.router)
app.include_router(video_clone.router)

View File

@@ -11,11 +11,12 @@ import {
type Wav2LipPostprocessMode,
} from "./components/SettingsPanel";
import { RuntimeConfigWorkspace } from "./components/RuntimeConfigWorkspace";
import { TopBar, type StudioWorkflow } from "./components/TopBar";
import { SceneStage } from "./components/SceneStage";
import { TopBar, type ConversationViewMode, type StudioWorkflow } from "./components/TopBar";
import { ToastStack, type ToastMessage, type ToastTone } from "./components/ToastStack";
import { VideoBackground } from "./components/VideoBackground";
import { AssetLibraryWorkspace, type AssetLibraryTab } from "./components/AssetLibraryWorkspace";
import { VideoCloneWorkspace } from "./components/VideoCloneWorkspace";
import { playWithMutedFallback } from "./components/VideoBackground";
import {
DEFAULT_VIDEO_CREATION_FASTLIVEPORTRAIT_CONFIG,
VideoCreationWorkspace,
@@ -31,6 +32,8 @@ import {
apiUploadFile,
buildApiUrl,
getMemoryLibraries,
listSceneBackgrounds,
listSceneCompositions,
loadRuntimeConfig,
uploadExportVideo,
type AvatarKnowledgeBasesResponse,
@@ -43,6 +46,8 @@ import {
type PersonasResponse,
type RuntimeConfigApplyInput,
type RuntimeConfigResponse,
type SceneBackgroundAsset,
type SceneComposition,
type SessionKnowledgeBasesRequest,
type SessionKnowledgeBasesResponse,
type VoiceCatalogItem,
@@ -186,6 +191,9 @@ const ASR_PROVIDER_STORAGE_KEY = "opentalking-asr-provider-v1";
const CLIENT_USER_ID_KEY = "opentalking-client-user-id";
const AGENT_CONFIG_STORAGE_KEY = "opentalking-agent-config-v1";
const SELECTED_PERSONA_STORAGE_KEY = "opentalking-selected-persona-id-v1";
const SELECTED_SCENE_STORAGE_KEY = "opentalking-selected-scene-id-v2";
const SELECTED_SCENE_BY_AVATAR_STORAGE_KEY = "opentalking-selected-scene-by-avatar-v1";
const CONVERSATION_VIEW_MODE_KEY = "opentalking-conversation-view-mode-v1";
const LEGACY_FASTLIVEPORTRAIT_DEFAULT_CONFIG: FasterLivePortraitConfig = {
head_motion_multiplier: 1.0,
pose_motion_multiplier: 0.35,
@@ -347,6 +355,22 @@ function writeStoredAvatarId(avatarId: string, source: "auto" | "explicit" = "ex
}
}
function readStoredSceneIdsByAvatar(): Record<string, string> {
try {
const raw = window.localStorage.getItem(SELECTED_SCENE_BY_AVATAR_STORAGE_KEY);
if (!raw) return {};
const parsed = JSON.parse(raw);
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return {};
return Object.fromEntries(
Object.entries(parsed).filter((entry): entry is [string, string] => (
typeof entry[0] === "string" && typeof entry[1] === "string" && entry[0].length > 0 && entry[1].length > 0
)),
);
} catch {
return {};
}
}
function readOrCreateClientUserId(): string {
try {
const existing = window.localStorage.getItem(CLIENT_USER_ID_KEY)?.trim();
@@ -828,6 +852,7 @@ function realtimeRecordingStartErrorMessage(error: unknown): string {
export default function App() {
const videoRef = useRef<HTMLVideoElement>(null);
const audioRef = useRef<HTMLAudioElement>(null);
const pcRef = useRef<RTCPeerConnection | null>(null);
const remoteStreamRef = useRef<MediaStream | null>(null);
const realtimeRecorderRef = useRef<MediaRecorder | null>(null);
@@ -886,6 +911,7 @@ export default function App() {
// Connection
const [connection, setConnection] = useState<ConnectionStatus>("idle");
const [sessionId, setSessionId] = useState<string | null>(null);
const [remoteStream, setRemoteStream] = useState<MediaStream | null>(null);
const [queueInfo, setQueueInfo] = useState<QueueInfo | null>(null);
const [expiringCountdown, setExpiringCountdown] = useState<number | null>(null);
@@ -972,6 +998,14 @@ export default function App() {
const [ftRecordPhase, setFtRecordPhase] = useState<"idle" | "recording" | "stopped">("idle");
const [ftRecordBusy, setFtRecordBusy] = useState(false);
const [assetLibraryRefreshKey, setAssetLibraryRefreshKey] = useState(0);
const [conversationViewMode, setConversationViewMode] = useState<ConversationViewMode>(() => {
try {
return window.localStorage.getItem(CONVERSATION_VIEW_MODE_KEY) === "immersive" ? "immersive" : "studio";
} catch {
return "studio";
}
});
const [immersiveAvatarAdjust, setImmersiveAvatarAdjust] = useState({ x: 0, y: 0, scale: 1 });
const [voiceCatalog, setVoiceCatalog] = useState<VoiceCatalogItem[]>([]);
const [voiceApplyNotice, setVoiceApplyNotice] = useState<string | null>(null);
const [ttsPreviewText, setTtsPreviewText] = useState(DEFAULT_TTS_PREVIEW_TEXT);
@@ -982,6 +1016,9 @@ export default function App() {
const [memoryEnabled, setMemoryEnabled] = useState(false);
const [memoryLibraryId, setMemoryLibraryId] = useState<string | null>(null);
const [memoryLibraries, setMemoryLibraries] = useState<MemoryLibrary[]>([]);
const [sceneBackgrounds, setSceneBackgrounds] = useState<SceneBackgroundAsset[]>([]);
const [sceneCompositions, setSceneCompositions] = useState<SceneComposition[]>([]);
const [selectedSceneIdsByAvatar, setSelectedSceneIdsByAvatar] = useState<Record<string, string>>(readStoredSceneIdsByAvatar);
const avatarKnowledgeBasesSyncReadyRef = useRef(false);
const lastPersistedAvatarKnowledgeBasesRef = useRef<Map<string, string[]>>(new Map());
const avatarKnowledgeBasesLoadSeqRef = useRef(0);
@@ -1041,6 +1078,17 @@ export default function App() {
});
const compactSquareStage = usesCompactSquareStage(model);
const selectedScene = useMemo(
() => {
const selectedSceneId = selectedSceneIdsByAvatar[avatarId];
const matchingScenes = sceneCompositions.filter((scene) => scene.avatar_id === avatarId);
if (selectedSceneId) {
return matchingScenes.find((scene) => scene.id === selectedSceneId) ?? null;
}
return matchingScenes.length === 1 ? matchingScenes[0] : null;
},
[avatarId, sceneCompositions, selectedSceneIdsByAvatar],
);
const dismissToast = useCallback((id: string) => {
setToasts((prev) => prev.filter((toast) => toast.id !== id));
@@ -1177,6 +1225,63 @@ export default function App() {
}
}, [notify]);
const refreshScenes = useCallback(async () => {
try {
const [backgrounds, scenes] = await Promise.all([
listSceneBackgrounds(),
listSceneCompositions(),
]);
setSceneBackgrounds(backgrounds.items);
setSceneCompositions(scenes.items);
setSelectedSceneIdsByAvatar((current) => {
const next = Object.fromEntries(
Object.entries(current).filter(([sceneAvatarId, sceneId]) => (
scenes.items.some((scene) => scene.id === sceneId && scene.avatar_id === sceneAvatarId)
)),
);
try {
const legacySceneId = window.localStorage.getItem(SELECTED_SCENE_STORAGE_KEY);
const legacyScene = legacySceneId ? scenes.items.find((scene) => scene.id === legacySceneId) : null;
if (legacyScene && !next[legacyScene.avatar_id]) {
next[legacyScene.avatar_id] = legacyScene.id;
}
window.localStorage.removeItem(SELECTED_SCENE_STORAGE_KEY);
} catch {
/* ignore */
}
return next;
});
} catch (error) {
console.warn("load scene assets failed", error);
}
}, []);
const handleSceneCompositionsChange = useCallback((scenes: SceneComposition[]) => {
setSceneCompositions(scenes);
setSelectedSceneIdsByAvatar((current) => {
return Object.fromEntries(
Object.entries(current).filter(([sceneAvatarId, sceneId]) => (
scenes.some((scene) => scene.id === sceneId && scene.avatar_id === sceneAvatarId)
)),
);
});
}, []);
const handleSceneSelect = useCallback((scene: SceneComposition) => {
setSelectedSceneIdsByAvatar((current) => ({
...current,
[scene.avatar_id]: scene.id,
}));
}, []);
const handleSceneClear = useCallback((sceneAvatarId: string) => {
setSelectedSceneIdsByAvatar((current) => {
const next = { ...current };
delete next[sceneAvatarId];
return next;
});
}, []);
const refreshAvatarKnowledgeBases = useCallback(async (targetAvatarId: string) => {
if (!targetAvatarId) return;
const seq = ++avatarKnowledgeBasesLoadSeqRef.current;
@@ -1259,6 +1364,46 @@ export default function App() {
if (workflow === "realtime") void refreshPersonas();
}, [refreshPersonas, workflow]);
useEffect(() => {
void refreshScenes();
}, [refreshScenes]);
useEffect(() => {
try {
if (Object.keys(selectedSceneIdsByAvatar).length) {
window.localStorage.setItem(SELECTED_SCENE_BY_AVATAR_STORAGE_KEY, JSON.stringify(selectedSceneIdsByAvatar));
} else {
window.localStorage.removeItem(SELECTED_SCENE_BY_AVATAR_STORAGE_KEY);
}
} catch {
/* ignore */
}
}, [selectedSceneIdsByAvatar]);
useEffect(() => {
try {
window.localStorage.setItem(CONVERSATION_VIEW_MODE_KEY, conversationViewMode);
} catch {
/* ignore */
}
}, [conversationViewMode]);
useEffect(() => {
const startupVisible = connection === "idle" || connection === "error" || connection === "connecting" || connection === "queued";
if (startupVisible && conversationViewMode === "immersive") setConversationViewMode("studio");
}, [connection, conversationViewMode]);
useEffect(() => {
if (workflow !== "realtime" || conversationViewMode !== "immersive") return;
const handleKeyDown = (event: KeyboardEvent) => {
if (event.key !== "Escape") return;
event.preventDefault();
setConversationViewMode("studio");
};
window.addEventListener("keydown", handleKeyDown);
return () => window.removeEventListener("keydown", handleKeyDown);
}, [conversationViewMode, workflow]);
useEffect(() => {
if (selectedPersonaId) return;
void refreshAvatarKnowledgeBases(avatarId);
@@ -1560,6 +1705,37 @@ export default function App() {
sessionIdRef.current = sessionId;
}, [sessionId]);
useEffect(() => {
const video = videoRef.current;
if (!video) return;
if (video.srcObject !== remoteStream) {
video.srcObject = remoteStream;
}
if (remoteStream) {
video.muted = false;
video.volume = 1;
playWithMutedFallback(video);
} else {
video.muted = true;
}
}, [conversationViewMode, remoteStream, workflow]);
useEffect(() => {
const audio = audioRef.current;
if (!audio) return;
if (!remoteStream) {
audio.srcObject = null;
return;
}
audio.srcObject = remoteStream;
audio.muted = false;
audio.volume = 1;
void audio.play().catch(() => {});
return () => {
audio.srcObject = null;
};
}, [remoteStream]);
useEffect(() => {
return () => {
if (ttsPreviewUrlRef.current) {
@@ -1656,6 +1832,7 @@ export default function App() {
for (const track of remoteStreamRef.current.getTracks()) track.stop();
remoteStreamRef.current = null;
}
setRemoteStream(null);
}, []);
const releaseSession = useCallback(async (sid: string, keepalive = false) => {
@@ -2047,10 +2224,12 @@ export default function App() {
const playback = await startPlayback(created.session_id, videoRef.current!, {
onRemoteStream: (remoteStream) => {
remoteStreamRef.current = remoteStream;
setRemoteStream(remoteStream);
},
});
pcRef.current = playback.pc;
remoteStreamRef.current = playback.remoteStream;
setRemoteStream(playback.remoteStream);
setActiveAsrProvider(lockedAsrProvider);
videoRef.current!.muted = false;
setConnection("live");
@@ -2605,9 +2784,14 @@ export default function App() {
}, [closePeerConnection, releaseSession]);
const currentAvatar = avatars.find((a) => a.id === avatarId) ?? null;
const selectedAvatarMaskUrl = selectedScene && currentAvatar?.matting_status === "transparent_ready"
? buildApiUrl(`/avatars/${encodeURIComponent(currentAvatar.id)}/preview`)
: null;
const sessionConfigLocked = connection === "connecting" || connection === "queued" || connection === "live" || connection === "expiring";
const effectiveAsrProvider = activeAsrProvider || normalizeAsrProvider(asrProvider, "dashscope");
const showStart = connection === "idle" || connection === "error" || connection === "connecting" || connection === "queued";
const effectiveConversationViewMode = showStart ? "studio" : conversationViewMode;
const immersiveActive = workflow === "realtime" && effectiveConversationViewMode === "immersive";
const chatMaxVisible = readChatMaxVisible();
const selectedModelLabel = MODEL_LABELS_FOR_STAGE[model] ?? model;
const wav2lipPostprocessModeLocked = sessionId !== null && connection !== "idle" && connection !== "error";
@@ -2640,9 +2824,12 @@ export default function App() {
};
return (
<div className="min-h-screen bg-slate-100 text-slate-900 lg:h-screen lg:overflow-hidden">
<audio ref={audioRef} autoPlay playsInline className="hidden" />
<TopBar
connection={connection}
workflow={workflow}
conversationViewMode={effectiveConversationViewMode}
immersiveChrome={immersiveActive}
flashtalkRecording={!!sessionId && (connection === "live" || connection === "expiring")}
flashtalkRecordPhase={ftRecordPhase}
flashtalkRecordBusy={ftRecordBusy}
@@ -2650,6 +2837,7 @@ export default function App() {
runtimeConfigReady={runtimeConfigReady}
runtimeConfigLoading={runtimeConfigLoading}
onInactiveModuleClick={(label) => notify(`${label}模块规划中。当前可用的是实时对话、视频克隆、数字人配置、语音驱动和导出能力。`, "info")}
onConversationViewModeChange={setConversationViewMode}
onWorkflowChange={(next) => {
setWorkflow(next);
if (next === "videoClone" && sessionIdRef.current) {
@@ -2708,6 +2896,12 @@ export default function App() {
onMemoryEnabledChange={setMemoryEnabled}
onMemoryLibrariesChange={setMemoryLibraries}
onRefreshMemoryLibraries={() => void refreshMemoryLibraries()}
avatars={avatars}
selectedSceneIdsByAvatar={selectedSceneIdsByAvatar}
onSceneSelect={handleSceneSelect}
onSceneClear={handleSceneClear}
onSceneBackgroundsChange={setSceneBackgrounds}
onSceneCompositionsChange={handleSceneCompositionsChange}
/>
</div>
) : workflow === "videoCreation" ? (
@@ -2761,8 +2955,14 @@ export default function App() {
/>
</div>
) : (
<div className="flex min-h-0 flex-col lg:h-[calc(100vh-3.5rem)] lg:flex-row">
<div className="order-2 min-h-0 lg:order-none lg:h-full lg:shrink-0">
<div
className={
immersiveActive
? "relative flex h-dvh min-h-0 flex-col bg-slate-950"
: "flex min-h-0 flex-col lg:h-[calc(100vh-3.5rem)] lg:flex-row"
}
>
<div className={`${immersiveActive ? "hidden" : "order-2 min-h-0 lg:order-none lg:h-full lg:shrink-0"}`}>
<SettingsPanel
expanded={settingsExpanded}
onExpandedChange={setSettingsExpanded}
@@ -2826,55 +3026,141 @@ export default function App() {
/>
</div>
<main className="order-1 flex min-h-0 flex-1 flex-col bg-slate-100 lg:order-none">
<div className="flex min-h-0 flex-1 flex-col p-4">
<div className="relative min-h-[360px] flex-1 overflow-hidden rounded-lg border border-slate-200 bg-white shadow-sm shadow-slate-200/70 lg:min-h-[420px]">
<div className="absolute inset-0 bg-slate-50" />
<div className="absolute inset-3 rounded-lg border border-slate-200 bg-white shadow-inner shadow-slate-200/60" />
<div className="absolute inset-0 flex items-center justify-center p-4 sm:p-6 lg:p-8">
<div
className={
compactSquareStage
? "relative aspect-square w-full max-w-[42rem] max-h-full"
: "relative h-full w-full"
}
>
<VideoBackground ref={videoRef} className="absolute inset-0 h-full w-full object-contain" />
</div>
</div>
<div className="absolute left-4 right-4 top-4 z-30 flex flex-col gap-2 sm:flex-row sm:items-start sm:justify-between">
<div className="flex min-w-0 flex-wrap gap-2">
<span className="inline-flex items-center gap-1 rounded-full border border-white/15 bg-white/90 px-2.5 py-1 text-xs font-medium text-slate-700 shadow-sm">
<span className={`h-1.5 w-1.5 rounded-full ${
connection === "live" || connection === "expiring" ? "bg-emerald-500" : "bg-slate-400"
}`} />
{connection === "live" || connection === "expiring" ? "已连接" : "待启动"}
</span>
<span className="inline-flex items-center gap-1 rounded-full border border-cyan-200 bg-cyan-50 px-2.5 py-1 text-xs font-medium text-cyan-700 shadow-sm">
WebRTC
</span>
<span className="inline-flex items-center gap-1 rounded-full border border-slate-200 bg-slate-50 px-2.5 py-1 text-xs font-medium text-slate-600 shadow-sm">
{MODEL_LABELS_FOR_STAGE[model] ?? model}
</span>
<span className="inline-flex max-w-[14rem] items-center gap-1 truncate rounded-full border border-slate-200 bg-white/90 px-2.5 py-1 text-xs font-medium text-slate-600 shadow-sm">
{currentAvatar?.name ?? currentAvatar?.id ?? "未选形象"}
</span>
</div>
<button
type="button"
onClick={handleReturnToAvatarSelection}
className="shrink-0 rounded-lg border border-slate-200 bg-white/95 px-3 py-2 text-xs font-semibold text-slate-700 shadow-sm transition hover:border-cyan-200 hover:text-cyan-700"
>
</button>
</div>
{currentSubtitle && !showStart ? (
<div className="absolute inset-x-4 bottom-4 z-10 mx-auto max-w-xl rounded-lg border border-slate-200 bg-white/95 px-4 py-3 text-center text-sm font-medium leading-relaxed text-slate-900 shadow-lg shadow-slate-200/80 backdrop-blur">
{currentSubtitle}
</div>
) : null}
<main
className={
immersiveActive
? "order-1 flex min-h-0 flex-1 flex-col overflow-hidden bg-slate-950 lg:order-none"
: "order-1 flex min-h-0 flex-1 flex-col bg-slate-100 lg:order-none"
}
>
<div
className={
immersiveActive
? "relative flex min-h-0 flex-1 flex-col overflow-hidden"
: "flex min-h-0 flex-1 flex-col p-4"
}
>
<div
className={
immersiveActive
? "absolute inset-x-0 top-0 bottom-12 overflow-hidden bg-black sm:bottom-14"
: "relative min-h-[360px] flex-1 overflow-hidden rounded-lg border border-slate-200 bg-white shadow-sm shadow-slate-200/70 lg:min-h-[420px]"
}
>
<SceneStage
videoRef={videoRef}
videoStream={remoteStream}
scene={showStart ? null : selectedScene}
backgrounds={sceneBackgrounds}
subtitle={!showStart ? currentSubtitle : null}
avatarMaskUrl={showStart ? null : selectedAvatarMaskUrl}
avatarAdjust={immersiveActive ? immersiveAvatarAdjust : undefined}
compactSquareStage={compactSquareStage}
className="h-full w-full"
>
{immersiveActive ? (
<>
<div className="absolute right-0 top-0 z-30 flex h-24 w-48 items-start justify-end p-4">
<button
type="button"
onClick={() => setConversationViewMode("studio")}
className="rounded-lg border border-white/15 bg-slate-950/45 px-3 py-2 text-xs font-semibold text-white opacity-0 shadow-lg backdrop-blur transition hover:bg-slate-950/65 hover:opacity-100 focus:opacity-100"
>
</button>
</div>
<div className="group absolute right-0 top-1/2 z-30 flex -translate-y-1/2 translate-x-[calc(100%-1.25rem)] items-center transition-transform duration-200 hover:translate-x-0 focus-within:translate-x-0">
<div className="flex h-20 w-5 items-center justify-center rounded-l-lg border border-r-0 border-slate-700 bg-slate-950 text-[10px] font-semibold text-white shadow-lg">
</div>
<div className="w-64 rounded-l-xl border border-slate-700 bg-slate-950 p-4 text-white shadow-2xl shadow-slate-950/30">
<div className="mb-3 flex items-center justify-between gap-3">
<p className="text-sm font-semibold"></p>
<button
type="button"
onClick={() => setImmersiveAvatarAdjust({ x: 0, y: 0, scale: 1 })}
className="rounded-md border border-white/15 px-2 py-1 text-xs font-semibold text-white/80 transition hover:bg-white/10 hover:text-white"
>
</button>
</div>
<label className="mb-3 block text-xs font-medium text-white/80">
<span className="mb-1 flex justify-between">
<span></span>
<span className="tabular-nums">{immersiveAvatarAdjust.x}px</span>
</span>
<input
type="range"
min="-480"
max="480"
step="4"
value={immersiveAvatarAdjust.x}
onChange={(event) => setImmersiveAvatarAdjust((prev) => ({ ...prev, x: Number(event.target.value) }))}
className="w-full accent-cyan-300"
/>
</label>
<label className="mb-3 block text-xs font-medium text-white/80">
<span className="mb-1 flex justify-between">
<span></span>
<span className="tabular-nums">{immersiveAvatarAdjust.y}px</span>
</span>
<input
type="range"
min="-320"
max="320"
step="4"
value={immersiveAvatarAdjust.y}
onChange={(event) => setImmersiveAvatarAdjust((prev) => ({ ...prev, y: Number(event.target.value) }))}
className="w-full accent-cyan-300"
/>
</label>
<label className="block text-xs font-medium text-white/80">
<span className="mb-1 flex justify-between">
<span></span>
<span className="tabular-nums">{immersiveAvatarAdjust.scale.toFixed(2)}x</span>
</span>
<input
type="range"
min="0.4"
max="2.2"
step="0.02"
value={immersiveAvatarAdjust.scale}
onChange={(event) => setImmersiveAvatarAdjust((prev) => ({ ...prev, scale: Number(event.target.value) }))}
className="w-full accent-cyan-300"
/>
</label>
</div>
</div>
</>
) : (
<div className="absolute left-4 right-4 top-4 z-30 flex flex-col gap-2 sm:flex-row sm:items-start sm:justify-between">
<div className="flex min-w-0 flex-wrap gap-2">
<span className="inline-flex items-center gap-1 rounded-full border border-white/15 bg-white/90 px-2.5 py-1 text-xs font-medium text-slate-700 shadow-sm">
<span className={`h-1.5 w-1.5 rounded-full ${
connection === "live" || connection === "expiring" ? "bg-emerald-500" : "bg-slate-400"
}`} />
{connection === "live" || connection === "expiring" ? "已连接" : "待启动"}
</span>
<span className="inline-flex items-center gap-1 rounded-full border border-cyan-200 bg-cyan-50 px-2.5 py-1 text-xs font-medium text-cyan-700 shadow-sm">
WebRTC
</span>
<span className="inline-flex items-center gap-1 rounded-full border border-slate-200 bg-slate-50 px-2.5 py-1 text-xs font-medium text-slate-600 shadow-sm">
{MODEL_LABELS_FOR_STAGE[model] ?? model}
</span>
<span className="inline-flex max-w-[14rem] items-center gap-1 truncate rounded-full border border-slate-200 bg-white/90 px-2.5 py-1 text-xs font-medium text-slate-600 shadow-sm">
{currentAvatar?.name ?? currentAvatar?.id ?? "未选形象"}
</span>
</div>
<button
type="button"
onClick={handleReturnToAvatarSelection}
className="shrink-0 rounded-lg border border-slate-200 bg-white/95 px-3 py-2 text-xs font-semibold text-slate-700 shadow-sm transition hover:border-cyan-200 hover:text-cyan-700"
>
</button>
</div>
)}
</SceneStage>
{showStart ? (
<div className="absolute inset-0 z-40 bg-white">
@@ -2909,7 +3195,18 @@ export default function App() {
</div>
{connection === "live" || connection === "expiring" ? (
<div className="mt-4">
<div
className={
immersiveActive
? "group absolute inset-x-3 bottom-0 z-30 mx-auto max-w-4xl translate-y-[calc(100%-1.25rem)] pb-3 transition-transform duration-200 hover:translate-y-0 focus-within:translate-y-0 sm:pb-5"
: "mt-4"
}
>
{immersiveActive ? (
<div className="pointer-events-none mb-2 flex justify-center opacity-100 transition-opacity duration-150 group-hover:opacity-0 group-focus-within:opacity-0">
<div className="h-1.5 w-16 rounded-full bg-white/80 shadow-sm" />
</div>
) : null}
<ChatInput
onSend={handleSend}
onSpeakAudio={handleSpeakAudio}
@@ -2936,9 +3233,13 @@ export default function App() {
</main>
<aside
className={`order-3 min-h-0 overflow-hidden border-l border-slate-200 bg-white transition-[width] duration-200 lg:shrink-0 ${
sessionPanelCollapsed ? "lg:w-12" : "lg:w-[360px]"
}`}
className={
immersiveActive
? "hidden"
: `order-3 min-h-0 overflow-hidden border-l border-slate-200 bg-white transition-[width] duration-200 lg:shrink-0 ${
sessionPanelCollapsed ? "lg:w-12" : "lg:w-[360px]"
}`
}
>
<div className="flex h-full min-h-0">
<div className="flex w-12 shrink-0 flex-col items-center justify-center gap-4 border-r border-slate-100 bg-slate-50 py-3">

View File

@@ -7,16 +7,25 @@ import {
apiPostForm,
buildApiDownloadUrl,
buildApiUrl,
createSceneComposition,
deleteSceneBackground,
deleteSceneComposition,
listSceneBackgrounds,
listSceneCompositions,
uploadSceneBackground,
type AvatarSummary,
type ExportVideoItem,
type KnowledgeBaseSummary,
type KnowledgeBasesResponse,
type KnowledgeDocument,
type KnowledgeDocumentsResponse,
type SceneBackgroundAsset,
type SceneComposition,
} from "../lib/api";
import { MemoryPanel } from "./MemoryPanel";
import type { MemoryLibrary } from "../types";
type AssetTab = "exports" | "knowledge" | "memory" | "avatars" | "voices";
type AssetTab = "exports" | "knowledge" | "memory" | "scenes" | "voices";
export type AssetLibraryTab = AssetTab;
type AssetLibraryWorkspaceProps = {
@@ -34,13 +43,19 @@ type AssetLibraryWorkspaceProps = {
onMemoryEnabledChange?: (enabled: boolean) => void;
onMemoryLibrariesChange?: (libraries: MemoryLibrary[]) => void;
onRefreshMemoryLibraries?: () => void;
avatars?: AvatarSummary[];
onSceneCompositionsChange?: (scenes: SceneComposition[]) => void;
selectedSceneIdsByAvatar?: Record<string, string>;
onSceneSelect?: (scene: SceneComposition) => void;
onSceneClear?: (avatarId: string) => void;
onSceneBackgroundsChange?: (backgrounds: SceneBackgroundAsset[]) => void;
};
const ASSET_TABS: { id: AssetTab; label: string; disabled?: boolean }[] = [
{ id: "exports", label: "导出视频" },
{ id: "knowledge", label: "知识库" },
{ id: "memory", label: "记忆库" },
{ id: "avatars", label: "Avatar资产", disabled: true },
{ id: "scenes", label: "场景资产" },
{ id: "voices", label: "声音资产", disabled: true },
];
@@ -224,6 +239,12 @@ export function AssetLibraryWorkspace({
onMemoryEnabledChange,
onMemoryLibrariesChange,
onRefreshMemoryLibraries,
avatars,
onSceneCompositionsChange,
selectedSceneIdsByAvatar = {},
onSceneSelect,
onSceneClear,
onSceneBackgroundsChange,
}: AssetLibraryWorkspaceProps) {
const [activeTab, setActiveTab] = useState<AssetTab>(initialTab);
const [items, setItems] = useState<ExportVideoItem[]>([]);
@@ -253,6 +274,14 @@ export function AssetLibraryWorkspace({
const [creatingKnowledge, setCreatingKnowledge] = useState(false);
const [filePoolUploading, setFilePoolUploading] = useState(false);
const [memoryRefreshToken, setMemoryRefreshToken] = useState(0);
const [sceneBackgrounds, setSceneBackgrounds] = useState<SceneBackgroundAsset[]>([]);
const [sceneCompositions, setSceneCompositions] = useState<SceneComposition[]>([]);
const [sceneLoading, setSceneLoading] = useState(false);
const [sceneName, setSceneName] = useState("");
const [sceneAvatarId, setSceneAvatarId] = useState("");
const [sceneBackgroundId, setSceneBackgroundId] = useState("");
const [backgroundName, setBackgroundName] = useState("");
const [backgroundFile, setBackgroundFile] = useState<File | null>(null);
const newKnowledgeFileInputRef = useRef<HTMLInputElement>(null);
const uploadKnowledgeFileInputRef = useRef<HTMLInputElement>(null);
const filePoolUploadInputRef = useRef<HTMLInputElement>(null);
@@ -324,6 +353,27 @@ export function AssetLibraryWorkspace({
}
}, [onNotify]);
const loadScenes = useCallback(async () => {
setSceneLoading(true);
try {
const [backgrounds, scenes] = await Promise.all([
listSceneBackgrounds(),
listSceneCompositions(),
]);
setSceneBackgrounds(backgrounds.items);
setSceneCompositions(scenes.items);
onSceneBackgroundsChange?.(backgrounds.items);
onSceneCompositionsChange?.(scenes.items);
setSceneAvatarId((current) => current || avatars?.[0]?.id || "");
} catch (err) {
console.warn("load scene assets failed", err);
const detail = err instanceof ApiError ? err.detail : null;
onNotify?.(detail ? `场景资产加载失败:${detail}` : "场景资产加载失败。", "error");
} finally {
setSceneLoading(false);
}
}, [avatars, onNotify, onSceneBackgroundsChange, onSceneCompositionsChange]);
useEffect(() => {
if (activeTabOverride) setActiveTab(activeTabOverride);
}, [activeTabOverride]);
@@ -347,6 +397,10 @@ export function AssetLibraryWorkspace({
}
}, [activeTab, loadKnowledgeDocuments, selectedKnowledgeId]);
useEffect(() => {
if (activeTab === "scenes") void loadScenes();
}, [activeTab, loadScenes, refreshToken]);
const totalSize = useMemo(
() => items.reduce((sum, item) => sum + item.size_bytes, 0),
[items],
@@ -386,8 +440,12 @@ export function AssetLibraryWorkspace({
setMemoryRefreshToken((value) => value + 1);
return;
}
if (activeTab === "scenes") {
void loadScenes();
return;
}
if (activeTab === "exports") void loadExports();
}, [activeTab, loadAllKnowledgeDocuments, loadExports, loadKnowledgeBases, loadKnowledgeDocuments, onRefreshMemoryLibraries, selectedKnowledgeId]);
}, [activeTab, loadAllKnowledgeDocuments, loadExports, loadKnowledgeBases, loadKnowledgeDocuments, loadScenes, onRefreshMemoryLibraries, selectedKnowledgeId]);
const handleCopyPath = useCallback(async (path: string) => {
try {
@@ -601,6 +659,76 @@ export function AssetLibraryWorkspace({
}
}, [filePoolFiles, filePoolUploadDisabled, loadAllKnowledgeDocuments, onNotify, uploadFilesToFilePool]);
const handleUploadSceneBackground = useCallback(async () => {
if (!backgroundFile) return;
try {
const uploaded = await uploadSceneBackground({
file: backgroundFile,
name: backgroundName.trim() || backgroundFile.name,
});
const nextBackgrounds = [uploaded, ...sceneBackgrounds.filter((item) => item.id !== uploaded.id)];
setSceneBackgrounds(nextBackgrounds);
onSceneBackgroundsChange?.(nextBackgrounds);
setBackgroundFile(null);
setBackgroundName("");
onNotify?.("背景资产已上传。", "success");
} catch (err) {
console.warn("upload scene background failed", err);
const detail = err instanceof ApiError ? err.detail : null;
onNotify?.(detail ? `上传失败:${detail}` : "上传失败,请稍后重试。", "error");
}
}, [backgroundFile, backgroundName, onNotify, onSceneBackgroundsChange, sceneBackgrounds]);
const handleCreateSceneComposition = useCallback(async () => {
if (!sceneName.trim() || !sceneAvatarId) return;
try {
const created = await createSceneComposition({
name: sceneName.trim(),
avatar_id: sceneAvatarId,
background_id: sceneBackgroundId || null,
avatar_fit: "contain",
avatar_scale: 1,
avatar_anchor: "center",
matting_required: true,
subtitle_style: "lower-third",
});
const nextScenes = [created, ...sceneCompositions.filter((item) => item.id !== created.id)];
setSceneCompositions(nextScenes);
onSceneCompositionsChange?.(nextScenes);
onSceneSelect?.(created);
setSceneName("");
onNotify?.("场景组合已创建。", "success");
} catch (err) {
console.warn("create scene composition failed", err);
const detail = err instanceof ApiError ? err.detail : null;
onNotify?.(detail ? `创建失败:${detail}` : "创建失败,请稍后重试。", "error");
}
}, [onNotify, onSceneCompositionsChange, onSceneSelect, sceneAvatarId, sceneBackgroundId, sceneCompositions, sceneName]);
const handleDeleteSceneBackground = useCallback(async (background: SceneBackgroundAsset) => {
try {
await deleteSceneBackground(background.id);
await loadScenes();
onNotify?.("背景资产已删除。", "success");
} catch (err) {
console.warn("delete scene background failed", err);
const detail = err instanceof ApiError ? err.detail : null;
onNotify?.(detail ? `删除失败:${detail}` : "删除失败,请稍后重试。", "error");
}
}, [loadScenes, onNotify]);
const handleDeleteSceneComposition = useCallback(async (scene: SceneComposition) => {
try {
await deleteSceneComposition(scene.id);
await loadScenes();
onNotify?.("场景组合已删除。", "success");
} catch (err) {
console.warn("delete scene composition failed", err);
const detail = err instanceof ApiError ? err.detail : null;
onNotify?.(detail ? `删除失败:${detail}` : "删除失败,请稍后重试。", "error");
}
}, [loadScenes, onNotify]);
const handleUploadKnowledgeDocuments = useCallback(async () => {
if (!selectedKnowledgeId || uploadDisabled) return;
setDocumentActionId("__upload__");
@@ -911,6 +1039,195 @@ export function AssetLibraryWorkspace({
/>
);
const avatarById = useMemo(() => new Map((avatars ?? []).map((avatar) => [avatar.id, avatar])), [avatars]);
const sceneGroups = useMemo(() => {
const avatarGroups = (avatars ?? [])
.map((avatar) => ({
avatar,
scenes: sceneCompositions.filter((scene) => scene.avatar_id === avatar.id),
}))
.filter((group) => group.scenes.length > 0);
const knownAvatarIds = new Set((avatars ?? []).map((avatar) => avatar.id));
const orphanScenes = sceneCompositions.filter((scene) => !knownAvatarIds.has(scene.avatar_id));
if (orphanScenes.length > 0) {
avatarGroups.push({
avatar: { id: "__unknown__", name: "未识别形象" } as AvatarSummary,
scenes: orphanScenes,
});
}
return avatarGroups;
}, [avatars, sceneCompositions]);
const renderScenesTab = () => (
<div className="grid min-h-[24rem] gap-4 xl:grid-cols-[22rem_minmax(0,1fr)]">
<aside className="rounded-lg border border-slate-200 bg-white p-4">
<h2 className="text-sm font-semibold text-slate-950"></h2>
<p className="mt-1 text-xs leading-relaxed text-slate-500">
</p>
<div className="mt-4 space-y-3">
<label className="block text-xs font-semibold text-slate-600" htmlFor="scene-background-name">
</label>
<input
id="scene-background-name"
value={backgroundName}
onChange={(event) => setBackgroundName(event.target.value)}
className="w-full rounded-lg border border-slate-200 bg-slate-50 px-3 py-2 text-sm outline-none focus:border-cyan-300 focus:bg-white"
/>
<input
type="file"
accept="image/png,image/jpeg,image/webp,video/mp4,video/webm,video/quicktime"
onChange={(event) => setBackgroundFile(event.currentTarget.files?.[0] ?? null)}
className="block w-full text-xs text-slate-600"
/>
<button
type="button"
disabled={!backgroundFile}
onClick={() => void handleUploadSceneBackground()}
className="w-full rounded-lg bg-cyan-600 px-3 py-2 text-sm font-semibold text-white transition hover:bg-cyan-500 disabled:cursor-not-allowed disabled:opacity-50"
>
</button>
</div>
<div className="mt-5 space-y-2">
{sceneLoading ? (
<p className="text-sm text-slate-500">...</p>
) : sceneBackgrounds.length ? sceneBackgrounds.map((background) => (
<div key={background.id} className="rounded-lg border border-slate-200 bg-slate-50 p-3">
<p className="truncate text-sm font-semibold text-slate-900">{background.name}</p>
<p className="mt-1 text-xs text-slate-500">{background.kind} · {formatSize(background.size_bytes)}</p>
<button
type="button"
onClick={() => void handleDeleteSceneBackground(background)}
className="mt-2 text-xs font-semibold text-rose-600 hover:text-rose-500"
>
</button>
</div>
)) : (
<p className="rounded-lg border border-dashed border-slate-200 bg-slate-50 p-4 text-center text-sm text-slate-500">
</p>
)}
</div>
</aside>
<section className="rounded-lg border border-slate-200 bg-white p-4">
<h2 className="text-sm font-semibold text-slate-950"></h2>
<p className="mt-1 text-xs leading-relaxed text-slate-500">
</p>
<div className="mt-4 grid gap-3 md:grid-cols-[minmax(0,1fr)_12rem_12rem_auto] md:items-end">
<label className="block text-xs font-semibold text-slate-600">
<input
value={sceneName}
onChange={(event) => setSceneName(event.target.value)}
className="mt-1 w-full rounded-lg border border-slate-200 bg-slate-50 px-3 py-2 text-sm outline-none focus:border-cyan-300 focus:bg-white"
/>
</label>
<label className="block text-xs font-semibold text-slate-600">
<select
value={sceneAvatarId}
onChange={(event) => setSceneAvatarId(event.target.value)}
className="mt-1 w-full rounded-lg border border-slate-200 bg-white px-3 py-2 text-sm"
>
{(avatars ?? []).map((avatar) => (
<option key={avatar.id} value={avatar.id}>{avatar.name ?? avatar.id}</option>
))}
</select>
</label>
<label className="block text-xs font-semibold text-slate-600">
<select
value={sceneBackgroundId}
onChange={(event) => setSceneBackgroundId(event.target.value)}
className="mt-1 w-full rounded-lg border border-slate-200 bg-white px-3 py-2 text-sm"
>
<option value=""></option>
{sceneBackgrounds.map((background) => (
<option key={background.id} value={background.id}>{background.name}</option>
))}
</select>
</label>
<button
type="button"
disabled={!sceneName.trim() || !sceneAvatarId}
onClick={() => void handleCreateSceneComposition()}
className="rounded-lg bg-slate-950 px-4 py-2 text-sm font-semibold text-white transition hover:bg-slate-800 disabled:cursor-not-allowed disabled:opacity-50"
>
</button>
</div>
<div className="mt-5 space-y-4">
{sceneGroups.length ? sceneGroups.map(({ avatar, scenes }) => (
<section key={avatar.id} className="rounded-lg border border-slate-200 bg-slate-50 p-3">
<div className="mb-3 flex flex-wrap items-center justify-between gap-2">
<div>
<p className="text-sm font-semibold text-slate-950">{avatar.name ?? avatar.id}</p>
<p className="text-xs text-slate-500">{scenes.length} </p>
</div>
{selectedSceneIdsByAvatar[avatar.id] ? (
<button
type="button"
onClick={() => onSceneClear?.(avatar.id)}
className="text-xs font-semibold text-slate-500 hover:text-slate-800"
>
</button>
) : null}
</div>
<div className="grid gap-3 md:grid-cols-2 xl:grid-cols-3">
{scenes.map((scene) => {
const selected = selectedSceneIdsByAvatar[scene.avatar_id] === scene.id;
const sceneAvatar = avatarById.get(scene.avatar_id);
return (
<article
key={scene.id}
className={`rounded-lg border p-3 transition ${
selected
? "border-cyan-300 bg-cyan-50"
: "border-slate-200 bg-white"
}`}
>
<p className="truncate text-sm font-semibold text-slate-950">{scene.name}</p>
<p className="mt-1 truncate text-xs text-slate-500">Avatar {sceneAvatar?.name ?? scene.avatar_id}</p>
<p className="mt-1 truncate text-xs text-slate-500">Background {scene.background_id ?? scene.background_color}</p>
<div className="mt-3 flex items-center gap-3">
<button
type="button"
aria-pressed={selected}
onClick={() => onSceneSelect?.(scene)}
className={`text-xs font-semibold ${
selected ? "text-cyan-800" : "text-cyan-700 hover:text-cyan-600"
}`}
>
{selected ? "当前默认" : "设为默认"}
</button>
<button
type="button"
onClick={() => void handleDeleteSceneComposition(scene)}
className="text-xs font-semibold text-rose-600 hover:text-rose-500"
>
</button>
</div>
</article>
);
})}
</div>
</section>
)) : (
<div className="rounded-lg border border-dashed border-slate-200 bg-slate-50 p-6 text-center text-sm text-slate-500">
</div>
)}
</div>
</section>
</div>
);
return (
<main className="flex min-h-0 flex-1 flex-col bg-slate-100 p-4">
<section className="flex min-h-0 flex-1 flex-col rounded-lg border border-slate-200 bg-white shadow-sm">
@@ -930,6 +1247,11 @@ export function AssetLibraryWorkspace({
<span>{memoryLibraries.length} </span>
<span>{memoryLibraries.reduce((sum, library) => sum + library.memory_count, 0)} </span>
</>
) : activeTab === "scenes" ? (
<>
<span>{sceneBackgrounds.length} </span>
<span>{sceneCompositions.length} </span>
</>
) : (
<>
<span>{items.length} </span>
@@ -939,10 +1261,10 @@ export function AssetLibraryWorkspace({
<button
type="button"
onClick={handleRefresh}
disabled={activeTab === "knowledge" ? knowledgeLoading : loading}
disabled={activeTab === "knowledge" ? knowledgeLoading : activeTab === "scenes" ? sceneLoading : loading}
className="rounded-lg border border-slate-200 bg-white px-3 py-1.5 font-semibold text-slate-700 transition hover:border-cyan-200 hover:text-cyan-700 disabled:cursor-not-allowed disabled:opacity-50"
>
{(activeTab === "knowledge" ? knowledgeLoading : loading) ? "刷新中..." : "刷新"}
{(activeTab === "knowledge" ? knowledgeLoading : activeTab === "scenes" ? sceneLoading : loading) ? "刷新中..." : "刷新"}
</button>
</div>
</div>
@@ -1036,9 +1358,11 @@ export function AssetLibraryWorkspace({
renderKnowledgeTab()
) : activeTab === "memory" ? (
renderMemoryTab()
) : activeTab === "scenes" ? (
renderScenesTab()
) : (
<div className="flex min-h-[18rem] items-center justify-center rounded-lg border border-dashed border-slate-300 bg-slate-50 text-sm font-medium text-slate-500">
{activeTab === "avatars" ? "Avatar资产管理规划中" : "声音资产管理规划中"}
</div>
)}
</div>

View File

@@ -271,7 +271,7 @@ export function AvatarSelectionStage({
/>
</div>
<div className="px-3 py-2">
<span className="truncate text-xs font-medium text-slate-500">
<span className="block truncate text-xs font-medium text-slate-500">
{avatar.is_custom ? "自定义形象" : "数字人形象"}
</span>
</div>

View File

@@ -0,0 +1,114 @@
import { useEffect, type RefObject } from "react";
import type { TtsProviderExtended } from "../constants/ttsBailian";
import type { SceneBackgroundAsset, SceneComposition } from "../lib/api";
import type { ConnectionStatus } from "../types";
import { ChatInput } from "./ChatInput";
import { SceneStage } from "./SceneStage";
type ImmersiveConversationProps = {
videoRef: RefObject<HTMLVideoElement>;
videoStream?: MediaStream | null;
scene: SceneComposition | null;
backgrounds: SceneBackgroundAsset[];
connection: ConnectionStatus;
sessionId: string | null;
subtitle: string | null;
isSpeaking: boolean;
ttsProvider: TtsProviderExtended;
sttProvider: string;
edgeVoice: string;
qwenModel: string;
qwenVoice: string;
onExit: () => void;
onSend: (text: string) => void;
onSpeakAudio: (blob: Blob) => void | Promise<void>;
onSpeakFlashtalkAudioFile?: (file: File) => void | Promise<void>;
onSpeakAudioStreamResult: (result: { text: string }) => void | Promise<void>;
onSpeakAudioStreamError: (message: string) => void;
onInterrupt: () => void;
onNotify?: (message: string, tone?: "info" | "success" | "error") => void;
};
export function ImmersiveConversation({
videoRef,
videoStream = null,
scene,
backgrounds,
connection,
sessionId,
subtitle,
isSpeaking,
ttsProvider,
sttProvider,
edgeVoice,
qwenModel,
qwenVoice,
onExit,
onSend,
onSpeakAudio,
onSpeakFlashtalkAudioFile,
onSpeakAudioStreamResult,
onSpeakAudioStreamError,
onInterrupt,
onNotify,
}: ImmersiveConversationProps) {
useEffect(() => {
const handleKeyDown = (event: KeyboardEvent) => {
const target = event.target as HTMLElement | null;
const editing = target
? ["INPUT", "TEXTAREA", "SELECT"].includes(target.tagName) || target.isContentEditable
: false;
if (editing) return;
if (event.key === "Escape") onExit();
};
window.addEventListener("keydown", handleKeyDown);
return () => window.removeEventListener("keydown", handleKeyDown);
}, [onExit]);
const live = connection === "live" || connection === "expiring";
return (
<main className="relative h-dvh min-h-0 overflow-hidden bg-slate-950 text-white">
<SceneStage
videoRef={videoRef}
videoStream={videoStream}
scene={scene}
backgrounds={backgrounds}
subtitle={subtitle}
className="h-full w-full"
>
<div className="absolute right-0 top-0 z-30 flex h-24 w-48 items-start justify-end p-4">
<button
type="button"
onClick={onExit}
className="rounded-lg border border-white/15 bg-slate-950/45 px-3 py-2 text-xs font-semibold text-white opacity-0 shadow-lg backdrop-blur transition hover:bg-slate-950/65 hover:opacity-100 focus:opacity-100"
>
</button>
</div>
{live ? (
<div className="absolute inset-x-3 bottom-3 z-30 mx-auto max-w-4xl sm:bottom-5">
<ChatInput
onSend={onSend}
onSpeakAudio={onSpeakAudio}
onSpeakFlashtalkAudioFile={onSpeakFlashtalkAudioFile}
streamingAsrSessionId={sessionId}
onSpeakAudioStreamResult={onSpeakAudioStreamResult}
onSpeakAudioStreamError={onSpeakAudioStreamError}
onInterrupt={onInterrupt}
isSpeaking={isSpeaking}
disabled={!live}
onNotify={onNotify}
ttsProvider={ttsProvider}
sttProvider={sttProvider}
edgeVoice={edgeVoice}
qwenModel={qwenModel}
qwenVoice={qwenVoice}
/>
</div>
) : null}
</SceneStage>
</main>
);
}

View File

@@ -0,0 +1,138 @@
import type { CSSProperties, ReactNode, RefObject } from "react";
import type { SceneBackgroundAsset, SceneComposition } from "../lib/api";
import { buildApiUrl } from "../lib/api";
import { VideoBackground } from "./VideoBackground";
type SceneStageProps = {
videoRef: RefObject<HTMLVideoElement>;
videoStream?: MediaStream | null;
scene: SceneComposition | null;
backgrounds: SceneBackgroundAsset[];
subtitle?: string | null;
avatarMaskUrl?: string | null;
avatarAdjust?: {
x: number;
y: number;
scale: number;
};
children?: ReactNode;
className?: string;
compactSquareStage?: boolean;
};
function backgroundUrl(background: SceneBackgroundAsset): string {
return buildApiUrl(background.url);
}
const AVATAR_ANCHOR_CLASSES = {
center: "items-center justify-center",
bottom: "items-end justify-center",
left: "items-center justify-start",
right: "items-center justify-end",
} as const;
const AVATAR_ANCHOR_OBJECT_POSITIONS = {
center: "object-center",
bottom: "object-[center_bottom]",
left: "object-[left_center]",
right: "object-[right_center]",
} as const;
const AVATAR_ANCHOR_TRANSFORM_ORIGINS = {
center: "center",
bottom: "center bottom",
left: "left center",
right: "right center",
} as const;
export function SceneStage({
videoRef,
videoStream = null,
scene,
backgrounds,
subtitle,
avatarMaskUrl = null,
avatarAdjust,
children,
className = "",
compactSquareStage = false,
}: SceneStageProps) {
const background = scene?.background_id
? backgrounds.find((item) => item.id === scene.background_id) ?? null
: null;
const subtitleStyle = scene?.subtitle_style ?? "lower-third";
const avatarFit = scene?.avatar_fit === "cover" ? "object-cover" : "object-contain";
const avatarAnchor = scene?.avatar_anchor ?? "center";
const avatarAnchorClass = AVATAR_ANCHOR_CLASSES[avatarAnchor as keyof typeof AVATAR_ANCHOR_CLASSES] ?? AVATAR_ANCHOR_CLASSES.center;
const avatarObjectPosition = AVATAR_ANCHOR_OBJECT_POSITIONS[avatarAnchor as keyof typeof AVATAR_ANCHOR_OBJECT_POSITIONS] ?? AVATAR_ANCHOR_OBJECT_POSITIONS.center;
const avatarTransformOrigin = AVATAR_ANCHOR_TRANSFORM_ORIGINS[avatarAnchor as keyof typeof AVATAR_ANCHOR_TRANSFORM_ORIGINS] ?? AVATAR_ANCHOR_TRANSFORM_ORIGINS.center;
const avatarMaskSize = scene?.avatar_fit === "cover" ? "cover" : "contain";
const avatarMaskPosition = avatarAnchor === "bottom"
? "center bottom"
: avatarAnchor === "left"
? "left center"
: avatarAnchor === "right"
? "right center"
: "center";
const hasSceneBackground = Boolean(scene);
const backgroundColor = scene?.background_color || "#ffffff";
const sceneAvatarScale = scene?.avatar_scale ?? 1;
const avatarDisplayScale = sceneAvatarScale * (avatarAdjust?.scale ?? 1);
const avatarTransform = avatarAdjust
? `translate(${avatarAdjust.x}px, ${avatarAdjust.y}px) scale(${avatarDisplayScale})`
: `scale(${sceneAvatarScale})`;
const avatarMaskStyle: CSSProperties | undefined = avatarMaskUrl
? {
WebkitMaskImage: `url("${avatarMaskUrl}")`,
WebkitMaskRepeat: "no-repeat",
WebkitMaskSize: avatarMaskSize,
WebkitMaskPosition: avatarMaskPosition,
maskImage: `url("${avatarMaskUrl}")`,
maskMode: "alpha",
maskRepeat: "no-repeat",
maskSize: avatarMaskSize,
maskPosition: avatarMaskPosition,
}
: undefined;
return (
<div className={`relative min-h-0 overflow-hidden ${hasSceneBackground ? "bg-slate-950" : "bg-white"} ${className}`}>
<div className="scene-background-layer absolute inset-0" style={{ backgroundColor }}>
{background?.kind === "image" ? (
<img src={backgroundUrl(background)} alt={background.name} className="h-full w-full object-cover" />
) : null}
{background?.kind === "video" ? (
<video src={backgroundUrl(background)} className="h-full w-full object-cover" autoPlay muted loop playsInline />
) : null}
{hasSceneBackground ? <div className="absolute inset-0 bg-slate-950/10" /> : null}
</div>
<div className={`absolute inset-0 flex p-4 sm:p-6 lg:p-8 ${avatarAnchorClass}`}>
<div
className={
compactSquareStage
? "relative aspect-square w-full max-w-[42rem] max-h-full"
: "relative h-full w-full"
}
style={{ transform: avatarTransform, transformOrigin: avatarTransformOrigin }}
>
<VideoBackground ref={videoRef} stream={videoStream} className={`absolute inset-0 h-full w-full ${avatarFit} ${avatarObjectPosition}`} style={avatarMaskStyle} />
</div>
</div>
{subtitle && subtitleStyle !== "none" ? (
<div
className={
subtitleStyle === "compact"
? "absolute inset-x-4 bottom-4 z-20 mx-auto max-w-lg rounded-lg bg-slate-950/72 px-4 py-2 text-center text-sm font-medium leading-relaxed text-white shadow-lg backdrop-blur"
: "absolute inset-x-4 bottom-6 z-20 mx-auto max-w-2xl rounded-lg border border-white/15 bg-slate-950/75 px-5 py-3 text-center text-base font-semibold leading-relaxed text-white shadow-xl backdrop-blur"
}
>
{subtitle}
</div>
) : null}
{children}
</div>
);
}

View File

@@ -29,10 +29,13 @@ const DOT_LABELS: Record<ConnectionStatus, string> = {
export type FlashtalkRecordPhase = "idle" | "recording" | "stopped";
export type StudioWorkflow = "realtime" | "videoCreation" | "videoClone" | "assetLibrary" | "runtimeConfig";
export type ConversationViewMode = "studio" | "immersive";
interface TopBarProps {
connection: ConnectionStatus;
workflow?: StudioWorkflow;
conversationViewMode?: ConversationViewMode;
immersiveChrome?: boolean;
flashtalkRecording?: boolean;
flashtalkRecordPhase?: FlashtalkRecordPhase;
flashtalkRecordBusy?: boolean;
@@ -40,6 +43,7 @@ interface TopBarProps {
runtimeConfigReady?: boolean;
runtimeConfigLoading?: boolean;
onInactiveModuleClick?: (label: string) => void;
onConversationViewModeChange?: (mode: ConversationViewMode) => void;
onFlashtalkRecordStart?: () => void;
onFlashtalkRecordStop?: () => void;
onFlashtalkRecordSave?: () => void;
@@ -49,6 +53,8 @@ interface TopBarProps {
export function TopBar({
connection,
workflow = "realtime",
conversationViewMode = "studio",
immersiveChrome = false,
flashtalkRecording = false,
flashtalkRecordPhase = "idle",
flashtalkRecordBusy = false,
@@ -56,6 +62,7 @@ export function TopBar({
runtimeConfigReady = false,
runtimeConfigLoading = false,
onInactiveModuleClick,
onConversationViewModeChange,
onFlashtalkRecordStart,
onFlashtalkRecordStop,
onFlashtalkRecordSave,
@@ -64,7 +71,18 @@ export function TopBar({
const busy = flashtalkRecordBusy || recordingSaving;
return (
<header className="flex h-14 shrink-0 items-center justify-between border-b border-slate-200 bg-white px-4 shadow-sm">
<header
className={
immersiveChrome
? "group fixed inset-x-0 top-0 z-50 flex h-14 -translate-y-12 items-center justify-between border-b border-white/10 bg-white/92 px-4 shadow-lg shadow-slate-950/10 backdrop-blur transition-transform duration-200 hover:translate-y-0"
: "flex h-14 shrink-0 items-center justify-between border-b border-slate-200 bg-white px-4 shadow-sm"
}
>
{immersiveChrome ? (
<div className="pointer-events-none absolute inset-x-0 bottom-[-0.65rem] flex justify-center opacity-100 transition-opacity duration-150 group-hover:opacity-0 group-focus-within:opacity-0">
<div className="h-1.5 w-16 rounded-full bg-white/80 shadow-sm" />
</div>
) : null}
<div className="flex min-w-0 items-center gap-3">
<div className="flex h-9 w-9 shrink-0 items-center justify-center rounded-lg bg-slate-950 text-cyan-300">
<svg viewBox="0 0 24 24" className="h-4 w-4" fill="currentColor" aria-hidden>
@@ -80,7 +98,14 @@ export function TopBar({
</div>
</div>
<nav className="hidden items-center gap-1 rounded-lg bg-slate-100 p-1 md:flex" aria-label="工作台模块">
<nav
className={
immersiveChrome
? "hidden"
: "hidden items-center gap-1 rounded-lg bg-slate-100 p-1 md:flex"
}
aria-label="工作台模块"
>
{[
["realtime", "实时对话"],
["videoCreation", "视频创作"],
@@ -114,6 +139,30 @@ export function TopBar({
</nav>
<div className="flex min-w-0 flex-wrap items-center justify-end gap-1.5 sm:gap-2">
{workflow === "realtime" ? (
<div className="hidden rounded-lg bg-slate-100 p-1 sm:flex" aria-label="实时对话视图">
{[
["studio", "工作台"],
["immersive", "沉浸"],
].map(([id, label]) => (
<button
key={id}
type="button"
onClick={(event) => {
event.currentTarget.blur();
onConversationViewModeChange?.(id as ConversationViewMode);
}}
className={`rounded-md px-2.5 py-1.5 text-xs font-semibold transition ${
conversationViewMode === id
? "bg-white text-cyan-700 shadow-sm"
: "text-slate-500 hover:bg-white/70 hover:text-slate-700"
}`}
>
{label}
</button>
))}
</div>
) : null}
{flashtalkRecording ? (
<div className="flex flex-wrap items-center justify-end gap-1.5">
{flashtalkRecordPhase === "idle" ? (

View File

@@ -1,19 +1,50 @@
import { forwardRef } from "react";
import type { CSSProperties } from "react";
import { forwardRef, useEffect, useImperativeHandle, useRef } from "react";
type VideoBackgroundProps = {
className?: string;
style?: CSSProperties;
stream?: MediaStream | null;
};
export function playWithMutedFallback(video: HTMLVideoElement) {
void video.play().catch(() => {
video.muted = true;
void video.play().catch(() => {});
});
}
export const VideoBackground = forwardRef<HTMLVideoElement, VideoBackgroundProps>(
({ className }, ref) => (
<video
ref={ref}
className={className ?? "absolute inset-0 h-full w-full object-contain"}
autoPlay
playsInline
muted
/>
),
({ className, style, stream }, ref) => {
const videoRef = useRef<HTMLVideoElement>(null);
useImperativeHandle(ref, () => videoRef.current as HTMLVideoElement);
useEffect(() => {
const video = videoRef.current;
if (!video) return;
if (video.srcObject !== stream) {
video.srcObject = stream ?? null;
}
if (stream) {
video.muted = false;
video.volume = 1;
playWithMutedFallback(video);
} else {
video.muted = true;
}
}, [stream]);
return (
<video
ref={videoRef}
className={className ?? "absolute inset-0 h-full w-full object-contain"}
style={style}
autoPlay
playsInline
muted={!stream}
/>
);
},
);
VideoBackground.displayName = "VideoBackground";

View File

@@ -242,6 +242,71 @@ export async function uploadExportVideo(input: UploadExportVideoInput): Promise<
return apiPostForm<ExportVideoItem>("/exports/videos", form);
}
export type SceneBackgroundAsset = {
id: string;
name: string;
kind: "image" | "video";
mime_type: string;
filename: string;
size_bytes: number;
url: string;
created_at: string;
};
export type SceneComposition = {
id: string;
name: string;
avatar_id: string;
background_id: string | null;
background_color: string;
avatar_fit: "contain" | "cover";
avatar_scale: number;
avatar_anchor: "center" | "bottom" | "left" | "right";
matting_required: boolean;
subtitle_style: "none" | "compact" | "lower-third";
created_at: string;
updated_at: string;
};
export type CreateSceneCompositionInput = {
name: string;
avatar_id: string;
background_id?: string | null;
background_color?: string;
avatar_fit?: "contain" | "cover";
avatar_scale?: number;
avatar_anchor?: "center" | "bottom" | "left" | "right";
matting_required?: boolean;
subtitle_style?: "none" | "compact" | "lower-third";
};
export async function listSceneBackgrounds(): Promise<{ items: SceneBackgroundAsset[] }> {
return apiGet<{ items: SceneBackgroundAsset[] }>("/scene-assets/backgrounds");
}
export async function uploadSceneBackground(input: { file: File; name: string }): Promise<SceneBackgroundAsset> {
const form = new FormData();
form.set("file", input.file);
form.set("name", input.name);
return apiPostForm<SceneBackgroundAsset>("/scene-assets/backgrounds", form);
}
export async function deleteSceneBackground(backgroundId: string): Promise<{ id: string; deleted: boolean }> {
return apiDelete<{ id: string; deleted: boolean }>(`/scene-assets/backgrounds/${encodeURIComponent(backgroundId)}`);
}
export async function listSceneCompositions(): Promise<{ items: SceneComposition[] }> {
return apiGet<{ items: SceneComposition[] }>("/scene-assets/compositions");
}
export async function createSceneComposition(input: CreateSceneCompositionInput): Promise<SceneComposition> {
return apiPost<SceneComposition>("/scene-assets/compositions", input);
}
export async function deleteSceneComposition(compositionId: string): Promise<{ id: string; deleted: boolean }> {
return apiDelete<{ id: string; deleted: boolean }>(`/scene-assets/compositions/${encodeURIComponent(compositionId)}`);
}
export type VideoCreationAudioSource = "upload" | "tts_text" | "voice_clone" | "reference_video";
@@ -441,6 +506,7 @@ export type AvatarSummary = {
height: number;
is_custom: boolean;
has_preview_video: boolean;
matting_status: "unknown" | "opaque" | "transparent_ready";
};
export type CreateSessionResponse = { session_id: string; status: string };

View File

@@ -0,0 +1,20 @@
{
"id": "female-host-transparent",
"name": "女主持(透明背景)",
"model_type": "wav2lip",
"fps": 25,
"sample_rate": 16000,
"width": 1024,
"height": 1024,
"version": "1.0",
"metadata": {
"description": "Built-in female host avatar asset with a transparent background.",
"idle_mode": "static",
"reference_mode": "image",
"frame_dir": "frames",
"source_image": "source/source.png",
"source_image_path": "reference.png",
"source_image_hash": "712c1cd47f32a18042a2908893f81128ba5176be6f896c8e31fb6ae4e5321f9b",
"matting_status": "transparent_ready"
}
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.4 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.4 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.4 MiB

View File

@@ -328,6 +328,8 @@ class Settings(BaseSettings):
models_dir: str = "./models"
worker_url: str = "http://127.0.0.1:9001"
exports_dir: str = "./data/exports"
scene_assets_dir: str = "./data/scene-assets"
scene_asset_max_bytes: int = 200 * 1024 * 1024
export_max_bytes: int = 1024 * 1024 * 1024
video_creation_audio_max_bytes: int = 50 * 1024 * 1024
video_creation_fasterliveportrait_preroll_ms: int = 400

213
opentalking/scene_assets.py Normal file
View File

@@ -0,0 +1,213 @@
from __future__ import annotations
import json
import re
import shutil
import uuid
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
SUPPORTED_IMAGE_TYPES = {"image/png", "image/jpeg", "image/webp"}
SUPPORTED_VIDEO_TYPES = {"video/mp4", "video/webm", "video/quicktime"}
SUPPORTED_BACKGROUND_TYPES = SUPPORTED_IMAGE_TYPES | SUPPORTED_VIDEO_TYPES
EXT_BY_MIME = {
"image/png": ".png",
"image/jpeg": ".jpg",
"image/webp": ".webp",
"video/mp4": ".mp4",
"video/webm": ".webm",
"video/quicktime": ".mov",
}
VALID_AVATAR_FITS = {"contain", "cover"}
VALID_AVATAR_ANCHORS = {"center", "bottom", "left", "right"}
VALID_SUBTITLE_STYLES = {"none", "compact", "lower-third"}
def sniff_background_mime(content: bytes) -> str | None:
if content.startswith(b"\x89PNG\r\n\x1a\n"):
return "image/png"
if content.startswith(b"\xff\xd8\xff"):
return "image/jpeg"
if len(content) >= 12 and content[:4] == b"RIFF" and content[8:12] == b"WEBP":
return "image/webp"
if len(content) >= 12 and content[4:8] == b"ftyp":
brand = content[8:12]
if brand in {b"qt "}:
return "video/quicktime"
if brand in {b"mp41", b"mp42", b"isom", b"iso2", b"avc1", b"M4V "}:
return "video/mp4"
return "video/mp4"
if content.startswith(b"\x1a\x45\xdf\xa3"):
return "video/webm"
return None
def _now() -> str:
return datetime.now(timezone.utc).isoformat()
def _slug(value: str, fallback: str) -> str:
normalized = re.sub(r"[^a-zA-Z0-9\u4e00-\u9fff_-]+", "-", value.strip()).strip("-_")
return normalized[:48] or fallback
def _read_json(path: Path, fallback: Any) -> Any:
if not path.is_file():
return fallback
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return fallback
def _write_json(path: Path, payload: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
class SceneAssetStore:
def __init__(self, root: Path) -> None:
self.root = root.expanduser().resolve()
self.backgrounds_dir = self.root / "backgrounds"
self.compositions_dir = self.root / "compositions"
self.background_index_path = self.backgrounds_dir / "index.json"
self.composition_index_path = self.compositions_dir / "index.json"
def list_backgrounds(self) -> list[dict[str, object]]:
items = _read_json(self.background_index_path, [])
return [item for item in items if isinstance(item, dict)]
def create_background(self, *, content: bytes, filename: str, mime_type: str, name: str) -> dict[str, object]:
normalized_mime = (mime_type or "").split(";")[0].strip().lower()
if not content:
raise ValueError("empty background asset")
sniffed_mime = sniff_background_mime(content)
if sniffed_mime not in SUPPORTED_BACKGROUND_TYPES:
raise ValueError("unsupported background media type")
normalized_mime = sniffed_mime
ext = EXT_BY_MIME[normalized_mime]
background_id = f"bg-{_slug(name or Path(filename).stem, 'background')}-{uuid.uuid4().hex[:10]}"
media_path = self.backgrounds_dir / background_id / f"source{ext}"
media_path.parent.mkdir(parents=True, exist_ok=True)
media_path.write_bytes(content)
item: dict[str, object] = {
"id": background_id,
"name": (name or Path(filename).stem or background_id).strip(),
"kind": "video" if normalized_mime.startswith("video/") else "image",
"mime_type": normalized_mime,
"filename": filename or f"source{ext}",
"size_bytes": len(content),
"url": f"/scene-assets/backgrounds/{background_id}/file",
"created_at": _now(),
}
items = [entry for entry in self.list_backgrounds() if entry.get("id") != background_id]
items.insert(0, item)
_write_json(self.background_index_path, items)
return item
def background_file_path(self, background_id: str) -> Path | None:
if not re.fullmatch(r"bg-[\w\u4e00-\u9fff-]+", background_id or ""):
return None
item = next((entry for entry in self.list_backgrounds() if entry.get("id") == background_id), None)
if not item:
return None
ext = EXT_BY_MIME.get(str(item.get("mime_type") or ""))
if not ext:
return None
path = (self.backgrounds_dir / background_id / f"source{ext}").resolve()
try:
path.relative_to(self.backgrounds_dir.resolve())
except ValueError:
return None
return path if path.is_file() else None
def delete_background(self, background_id: str) -> bool:
if not re.fullmatch(r"bg-[\w\u4e00-\u9fff-]+", background_id or ""):
return False
items = self.list_backgrounds()
next_items = [item for item in items if item.get("id") != background_id]
if len(next_items) == len(items):
return False
_write_json(self.background_index_path, next_items)
shutil.rmtree(self.backgrounds_dir / background_id, ignore_errors=True)
compositions = [
{**item, "background_id": None}
if item.get("background_id") == background_id else item
for item in self.list_compositions()
]
_write_json(self.composition_index_path, compositions)
return True
def list_compositions(self) -> list[dict[str, object]]:
items = _read_json(self.composition_index_path, [])
return [item for item in items if isinstance(item, dict)]
def create_composition(self, payload: dict[str, object]) -> dict[str, object]:
composition_id = f"scene-{_slug(str(payload.get('name') or 'scene'), 'scene')}-{uuid.uuid4().hex[:10]}"
now = _now()
item = self._normalize_composition({**payload, "id": composition_id, "created_at": now, "updated_at": now})
items = [entry for entry in self.list_compositions() if entry.get("id") != composition_id]
items.insert(0, item)
_write_json(self.composition_index_path, items)
return item
def update_composition(self, composition_id: str, payload: dict[str, object]) -> dict[str, object] | None:
items = self.list_compositions()
for index, item in enumerate(items):
if item.get("id") != composition_id:
continue
updated = self._normalize_composition({**item, **payload, "id": composition_id, "updated_at": _now()})
items[index] = updated
_write_json(self.composition_index_path, items)
return updated
return None
def delete_composition(self, composition_id: str) -> bool:
items = self.list_compositions()
next_items = [item for item in items if item.get("id") != composition_id]
if len(next_items) == len(items):
return False
_write_json(self.composition_index_path, next_items)
return True
def _normalize_composition(self, payload: dict[str, object]) -> dict[str, object]:
avatar_id = str(payload.get("avatar_id") or "").strip()
if not avatar_id:
raise ValueError("avatar_id is required")
background_id = str(payload.get("background_id") or "").strip() or None
if background_id and not any(item.get("id") == background_id for item in self.list_backgrounds()):
raise ValueError("background_id not found")
avatar_fit = str(payload.get("avatar_fit") or "contain").strip()
avatar_anchor = str(payload.get("avatar_anchor") or "center").strip()
subtitle_style = str(payload.get("subtitle_style") or "lower-third").strip()
raw_avatar_scale = payload.get("avatar_scale")
if raw_avatar_scale is None:
avatar_scale = 1.0
elif isinstance(raw_avatar_scale, str | int | float):
avatar_scale = float(raw_avatar_scale)
else:
raise ValueError("avatar_scale must be a number")
if avatar_fit not in VALID_AVATAR_FITS:
raise ValueError("invalid avatar_fit")
if avatar_anchor not in VALID_AVATAR_ANCHORS:
raise ValueError("invalid avatar_anchor")
if subtitle_style not in VALID_SUBTITLE_STYLES:
raise ValueError("invalid subtitle_style")
if not 0.5 <= avatar_scale <= 2.0:
raise ValueError("avatar_scale must be between 0.5 and 2.0")
return {
"id": str(payload["id"]),
"name": str(payload.get("name") or payload["id"]).strip(),
"avatar_id": avatar_id,
"background_id": background_id,
"background_color": str(payload.get("background_color") or "#0f172a").strip(),
"avatar_fit": avatar_fit,
"avatar_scale": avatar_scale,
"avatar_anchor": avatar_anchor,
"matting_required": bool(payload.get("matting_required", False)),
"subtitle_style": subtitle_style,
"created_at": str(payload.get("created_at") or _now()),
"updated_at": str(payload.get("updated_at") or _now()),
}

View File

@@ -6,7 +6,7 @@ from pathlib import Path
def test_asset_library_exposes_knowledge_base_tab_and_modal() -> None:
source = Path("apps/web/src/components/AssetLibraryWorkspace.tsx").read_text(encoding="utf-8")
assert 'type AssetTab = "exports" | "knowledge" | "memory" | "avatars" | "voices"' in source
assert 'type AssetTab = "exports" | "knowledge" | "memory" | "scenes" | "voices"' in source
assert "知识库" in source
assert "新建知识库" in source
assert "从本地中间文件导入" in source

View File

@@ -0,0 +1,190 @@
from __future__ import annotations
from pathlib import Path
def test_scene_stage_component_exists_and_handles_background_layers() -> None:
source = Path("apps/web/src/components/SceneStage.tsx").read_text(encoding="utf-8")
assert "export function SceneStage" in source
assert "SceneBackgroundAsset" in source
assert "SceneComposition" in source
assert "scene-background-layer" in source
assert "VideoBackground" in source
assert "subtitle_style" in source
def test_scene_stage_applies_avatar_anchor_positioning() -> None:
source = Path("apps/web/src/components/SceneStage.tsx").read_text(encoding="utf-8")
assert "scene?.avatar_anchor" in source
assert "AVATAR_ANCHOR_OBJECT_POSITIONS" in source
assert 'bottom: "object-[center_bottom]"' in source
assert 'left: "object-[left_center]"' in source
assert 'right: "object-[right_center]"' in source
assert "${avatarFit} ${avatarObjectPosition}" in source
assert "AVATAR_ANCHOR_TRANSFORM_ORIGINS" in source
assert "transformOrigin: avatarTransformOrigin" in source
assert 'bottom: "items-end justify-center"' in source
assert 'left: "items-center justify-start"' in source
assert 'right: "items-center justify-end"' in source
def test_scene_stage_masks_transparent_avatar_video_with_preview_alpha() -> None:
source = Path("apps/web/src/components/SceneStage.tsx").read_text(encoding="utf-8")
assert "avatarMaskUrl?: string | null;" in source
assert "WebkitMaskImage" in source
assert "maskImage" in source
assert "maskMode" in source
assert "avatarMaskStyle" in source
def test_scene_stage_accepts_immersive_avatar_adjustments() -> None:
source = Path("apps/web/src/components/SceneStage.tsx").read_text(encoding="utf-8")
assert "avatarAdjust?: {" in source
assert "x: number;" in source
assert "y: number;" in source
assert "scale: number;" in source
assert "translate(${avatarAdjust.x}px, ${avatarAdjust.y}px) scale" in source
def test_app_uses_scene_stage_for_realtime_stage() -> None:
source = Path("apps/web/src/App.tsx").read_text(encoding="utf-8")
assert 'import { SceneStage } from "./components/SceneStage";' in source
assert "selectedScene" in source
assert "sceneBackgrounds" in source
assert "<SceneStage" in source
assert "selectedAvatarMaskUrl" in source
assert "avatarMaskUrl={showStart ? null : selectedAvatarMaskUrl}" in source
def test_app_keeps_webrtc_remote_stream_identity_for_async_tracks() -> None:
source = Path("apps/web/src/App.tsx").read_text(encoding="utf-8")
assert "cloneMediaStream" not in source
assert "setRemoteStream(remoteStream)" in source
assert "setRemoteStream(playback.remoteStream)" in source
def test_video_background_falls_back_to_muted_playback_when_autoplay_blocks_audio() -> None:
source = Path("apps/web/src/components/VideoBackground.tsx").read_text(encoding="utf-8")
assert "export function playWithMutedFallback" in source
assert "video.muted = true" in source
assert "video.play().catch" in source
def test_app_replays_remote_video_with_muted_fallback_when_view_changes() -> None:
source = Path("apps/web/src/App.tsx").read_text(encoding="utf-8")
assert 'import { playWithMutedFallback } from "./components/VideoBackground";' in source
effect_start = source.index("const video = videoRef.current;")
effect_end = source.index("}, [conversationViewMode, remoteStream, workflow]);", effect_start)
playback_effect = source[effect_start:effect_end]
assert "playWithMutedFallback(video)" in playback_effect
def test_immersive_conversation_component_focuses_stage_and_input() -> None:
source = Path("apps/web/src/components/ImmersiveConversation.tsx").read_text(encoding="utf-8")
assert "export function ImmersiveConversation" in source
assert "SceneStage" in source
assert "ChatInput" in source
assert "onExit" in source
assert 'event.key === "Escape"' in source
assert "bottom-3" in source
assert "返回工作台" in source
def test_topbar_exposes_realtime_view_mode_toggle() -> None:
source = Path("apps/web/src/components/TopBar.tsx").read_text(encoding="utf-8")
assert 'export type ConversationViewMode = "studio" | "immersive";' in source
assert "onConversationViewModeChange" in source
assert "沉浸" in source
def test_topbar_hides_main_module_navigation_in_immersive_chrome() -> None:
source = Path("apps/web/src/components/TopBar.tsx").read_text(encoding="utf-8")
nav_label = source.index('aria-label="工作台模块"')
nav_start = source.rindex("<nav", 0, nav_label)
nav_end = source.index("</nav>", nav_label)
nav_block = source[nav_start:nav_end]
assert "immersiveChrome" in nav_block
assert '? "hidden"' in nav_block
def test_topbar_immersive_chrome_does_not_stay_open_from_button_focus() -> None:
source = Path("apps/web/src/components/TopBar.tsx").read_text(encoding="utf-8")
header_start = source.index("<header")
header_end = source.index(">", header_start)
header_block = source[header_start:header_end]
assert "focus-within:translate-y-0" not in header_block
def test_app_switches_between_studio_and_immersive_realtime_views() -> None:
source = Path("apps/web/src/App.tsx").read_text(encoding="utf-8")
assert "conversationViewMode" in source
assert "immersiveActive" in source
assert 'conversationViewMode === "immersive"' in source
def test_app_exits_immersive_realtime_view_with_escape() -> None:
source = Path("apps/web/src/App.tsx").read_text(encoding="utf-8")
assert 'event.key !== "Escape"' in source
assert 'setConversationViewMode("studio")' in source
def test_app_offers_immersive_avatar_adjustment_controls() -> None:
source = Path("apps/web/src/App.tsx").read_text(encoding="utf-8")
assert "immersiveAvatarAdjust" in source
assert "avatarAdjust={immersiveActive ? immersiveAvatarAdjust : undefined}" in source
assert "画面微调" in source
assert "水平" in source
assert "垂直" in source
assert "缩放" in source
assert "重置" in source
def test_immersive_avatar_adjustment_controls_have_wide_ranges() -> None:
source = Path("apps/web/src/App.tsx").read_text(encoding="utf-8")
x_label = source.index("<span>水平</span>")
y_label = source.index("<span>垂直</span>")
scale_label = source.index("<span>缩放</span>")
x_block = source[x_label:source.index('value={immersiveAvatarAdjust.x}', x_label)]
y_block = source[y_label:source.index('value={immersiveAvatarAdjust.y}', y_label)]
scale_block = source[scale_label:source.index('value={immersiveAvatarAdjust.scale}', scale_label)]
assert 'min="-480"' in x_block
assert 'max="480"' in x_block
assert 'min="-320"' in y_block
assert 'max="320"' in y_block
assert 'min="0.4"' in scale_block
assert 'max="2.2"' in scale_block
def test_immersive_avatar_adjustment_panel_uses_opaque_background() -> None:
source = Path("apps/web/src/App.tsx").read_text(encoding="utf-8")
panel_start = source.index('className="w-64 rounded-l-xl')
panel_end = source.index('画面微调', panel_start)
panel_block = source[panel_start:panel_end]
assert "bg-slate-950 " in panel_block
assert "bg-slate-950/" not in panel_block
assert "backdrop-blur" not in panel_block
def test_app_gates_persisted_immersive_mode_while_start_is_visible() -> None:
source = Path("apps/web/src/App.tsx").read_text(encoding="utf-8")
assert 'const effectiveConversationViewMode = showStart ? "studio" : conversationViewMode;' in source
assert 'effectiveConversationViewMode === "immersive"' in source

View File

@@ -36,7 +36,7 @@ def test_realtime_settings_memory_does_not_render_editing_tools() -> None:
def test_asset_library_exposes_memory_tab_and_management_panel() -> None:
source = (ROOT / "apps/web/src/components/AssetLibraryWorkspace.tsx").read_text(encoding="utf-8")
assert 'type AssetTab = "exports" | "knowledge" | "memory" | "avatars" | "voices"' in source
assert 'type AssetTab = "exports" | "knowledge" | "memory" | "scenes" | "voices"' in source
assert '{ id: "memory", label: "记忆库" }' in source
assert "renderMemoryTab" in source
assert "<MemoryPanel" in source

View File

@@ -0,0 +1,111 @@
from __future__ import annotations
from pathlib import Path
def test_api_exposes_scene_asset_types_and_helpers() -> None:
source = Path("apps/web/src/lib/api.ts").read_text(encoding="utf-8")
assert "export type SceneBackgroundAsset" in source
assert "export type SceneComposition" in source
assert "uploadSceneBackground" in source
assert 'apiPostForm<SceneBackgroundAsset>("/scene-assets/backgrounds", form)' in source
assert 'apiPost<SceneComposition>("/scene-assets/compositions", input)' in source
def test_asset_library_has_scene_assets_tab() -> None:
source = Path("apps/web/src/components/AssetLibraryWorkspace.tsx").read_text(encoding="utf-8")
assert 'type AssetTab = "exports" | "knowledge" | "memory" | "scenes" | "voices"' in source
assert '{ id: "scenes", label: "场景资产" }' in source
assert "renderScenesTab" in source
assert "背景资产" in source
assert "场景组合" in source
assert "uploadSceneBackground" in source
assert "createSceneComposition" in source
def test_app_passes_avatars_to_asset_library_workspace() -> None:
source = Path("apps/web/src/App.tsx").read_text(encoding="utf-8")
asset_library_mount = source.split('workflow === "assetLibrary" ? (', 1)[1].split(
') : workflow === "videoCreation" ? (',
1,
)[0]
assert "<AssetLibraryWorkspace" in asset_library_mount
assert "avatars={avatars}" in asset_library_mount
def test_app_wires_scene_selection_and_background_updates_to_asset_library_workspace() -> None:
source = Path("apps/web/src/App.tsx").read_text(encoding="utf-8")
asset_library_mount = source.split('workflow === "assetLibrary" ? (', 1)[1].split(
') : workflow === "videoCreation" ? (',
1,
)[0]
assert "selectedSceneIdsByAvatar={selectedSceneIdsByAvatar}" in asset_library_mount
assert "onSceneSelect={handleSceneSelect}" in asset_library_mount
assert "onSceneBackgroundsChange={setSceneBackgrounds}" in asset_library_mount
def test_realtime_stage_uses_avatar_default_scene_without_cross_avatar_leakage() -> None:
source = Path("apps/web/src/App.tsx").read_text(encoding="utf-8")
selected_scene_block = source.split("const selectedScene = useMemo(", 1)[1].split(
"const dismissToast",
1,
)[0]
assert "selectedSceneIdsByAvatar[avatarId]" in selected_scene_block
assert "scene.avatar_id === avatarId" in selected_scene_block
assert "matchingScenes.length === 1 ? matchingScenes[0] : null" in selected_scene_block
assert "[avatarId, sceneCompositions, selectedSceneIdsByAvatar]" in selected_scene_block
def test_asset_library_scene_cards_can_select_created_scene_and_sync_backgrounds() -> None:
source = Path("apps/web/src/components/AssetLibraryWorkspace.tsx").read_text(encoding="utf-8")
assert "selectedSceneIdsByAvatar?: Record<string, string>;" in source
assert "onSceneSelect?: (scene: SceneComposition) => void;" in source
assert "onSceneClear?: (avatarId: string) => void;" in source
assert "onSceneBackgroundsChange?: (backgrounds: SceneBackgroundAsset[]) => void;" in source
assert "onSceneSelect?.(created)" in source
assert "onSceneBackgroundsChange?.(nextBackgrounds)" in source
assert "onSceneSelect?.(scene)" in source
assert "selectedSceneIdsByAvatar[scene.avatar_id] === scene.id" in source
assert "设为默认" in source
assert "当前默认" in source
assert "取消默认" in source
def test_asset_library_groups_scene_compositions_by_avatar() -> None:
source = Path("apps/web/src/components/AssetLibraryWorkspace.tsx").read_text(encoding="utf-8")
assert "sceneGroups" in source
assert "avatarById" in source
assert "sceneCompositions.filter((scene) => scene.avatar_id === avatar.id)" in source
def test_scene_delete_actions_use_error_handled_handlers() -> None:
source = Path("apps/web/src/components/AssetLibraryWorkspace.tsx").read_text(encoding="utf-8")
assert "const handleDeleteSceneBackground = useCallback(async (background: SceneBackgroundAsset)" in source
assert "const handleDeleteSceneComposition = useCallback(async (scene: SceneComposition)" in source
assert "delete scene background failed" in source
assert "delete scene composition failed" in source
assert "err instanceof ApiError ? err.detail : null" in source
assert "onNotify?.(detail ? `删除失败:${detail}` : \"删除失败,请稍后重试。\", \"error\")" in source
assert "await loadScenes()" in source
assert ".then(loadScenes)" not in source
def test_scene_ui_omits_matting_readiness_copy() -> None:
asset_source = Path("apps/web/src/components/AssetLibraryWorkspace.tsx").read_text(encoding="utf-8")
avatar_source = Path("apps/web/src/components/AvatarSelectionStage.tsx").read_text(encoding="utf-8")
api_source = Path("apps/web/src/lib/api.ts").read_text(encoding="utf-8")
assert 'matting_status: "unknown" | "opaque" | "transparent_ready";' in api_source
assert "已抠像/透明数字人" not in asset_source
assert "抠像状态未知" not in asset_source
assert "未抠像" not in asset_source
assert 'avatar?.matting_status === "unknown"' not in asset_source
assert "matting_status" not in avatar_source

View File

@@ -338,12 +338,14 @@ def test_asset_library_workspace_lists_exported_videos():
assert "AssetLibraryWorkspace" in app
assert "assetLibrary" in topbar
assert "导出视频" in asset
assert "Avatar资产" in asset
assert "场景资产" in asset
assert "声音资产" in asset
assert 'apiGet<{ items: ExportVideoItem[] }>("/exports/videos")' in asset
assert "listSceneCompositions()" in asset
assert "download_url" in asset
assert "navigator.clipboard.writeText" in asset
assert "apiDelete(`/exports/videos/${item.id}`)" in asset
assert "deleteSceneComposition(scene.id)" in asset
def test_realtime_recording_uses_browser_media_recorder_and_uploads_export():