mirror of
https://github.com/datascale-ai/opentalking.git
synced 2026-07-11 21:13:52 +08:00
chore: ignore local worktrees
feat: add scene asset store fix: validate scene asset edge cases docs: record task 1 review fix feat: expose scene asset api test api composition deletion feat: add scene assets to asset library Fix scene asset library integration Update asset tab test expectations feat: render realtime stage with scene assets feat: add immersive conversation mode Gate immersive mode during startup feat: surface avatar matting readiness Fix scene asset selection and upload validation Fix scene avatar anchor rendering Fix unified scene asset routes Stabilize unified scene route test Fix default realtime stage scene behavior Keep realtime video across view switches Refine immersive chrome and keyboard handling Preserve audio when switching realtime views Keep realtime media mounted in immersive mode Restore realtime audio and refine immersive controls Scope scene backgrounds to matching avatar Preserve transparent avatar backgrounds in scenes Keep WebRTC remote stream identity Fallback to muted stage video playback Refine immersive QuickTalk controls Make immersive adjustment panel opaque Expand immersive adjustment ranges Use avatar-specific scene defaults Fix mypy type narrowing for scene assets
This commit is contained in:
1
.gitignore
vendored
1
.gitignore
vendored
@@ -79,6 +79,7 @@ machine.md
|
||||
|
||||
# Internal dev specs (not for public repo)
|
||||
docs/superpowers/
|
||||
.worktrees/
|
||||
|
||||
# MkDocs build output
|
||||
site/
|
||||
|
||||
@@ -9,7 +9,7 @@ from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from apps.api.core.config import get_settings
|
||||
from apps.api.routes import agent, avatars, events, exports, health, memory, models, personas, runtime_config, sessions, tts_preview, video_clone, video_creation, voices
|
||||
from apps.api.routes import agent, avatars, events, exports, health, memory, models, personas, runtime_config, scene_assets, sessions, tts_preview, video_clone, video_creation, voices
|
||||
from opentalking.voice.store import init_voice_store
|
||||
|
||||
|
||||
@@ -43,6 +43,7 @@ def create_app() -> FastAPI:
|
||||
app.include_router(personas.router)
|
||||
app.include_router(events.router)
|
||||
app.include_router(exports.router)
|
||||
app.include_router(scene_assets.router)
|
||||
app.include_router(runtime_config.router)
|
||||
app.include_router(tts_preview.router)
|
||||
app.include_router(video_clone.router)
|
||||
|
||||
@@ -60,6 +60,15 @@ def _is_hidden_avatar(manifest_path: Path) -> bool:
|
||||
return bool((raw.get("metadata") or {}).get("hidden"))
|
||||
|
||||
|
||||
def _avatar_matting_status(manifest_path: Path) -> str:
|
||||
try:
|
||||
raw = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return "unknown"
|
||||
value = str((raw.get("metadata") or {}).get("matting_status") or "").strip()
|
||||
return value if value in {"unknown", "opaque", "transparent_ready"} else "unknown"
|
||||
|
||||
|
||||
def _avatar_preview_video_path(avatar_dir: Path) -> Path | None:
|
||||
manifest_path = avatar_dir / "manifest.json"
|
||||
try:
|
||||
@@ -112,6 +121,7 @@ def _summary_from_dir(path: Path) -> AvatarSummary:
|
||||
height=m.height,
|
||||
is_custom=_is_custom_avatar(path / "manifest.json"),
|
||||
has_preview_video=_avatar_preview_video_path(path) is not None,
|
||||
matting_status=_avatar_matting_status(path / "manifest.json"),
|
||||
)
|
||||
|
||||
|
||||
@@ -144,7 +154,7 @@ async def _read_upload_image(upload: UploadFile) -> Image.Image:
|
||||
image.load()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise HTTPException(status_code=400, detail="invalid image") from exc
|
||||
return image.convert("RGB")
|
||||
return image.convert("RGBA") if "A" in image.getbands() else image.convert("RGB")
|
||||
|
||||
|
||||
async def _read_upload_video(upload: UploadFile) -> tuple[Image.Image, bytes, str]:
|
||||
@@ -209,6 +219,7 @@ def _write_custom_avatar_manifest(
|
||||
metadata["idle_mode"] = "static"
|
||||
metadata["reference_mode"] = "image"
|
||||
metadata["source_image"] = "source/source.png"
|
||||
metadata["matting_status"] = "opaque"
|
||||
raw["metadata"] = metadata
|
||||
target_manifest_path.write_text(json.dumps(raw, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
@@ -235,6 +246,24 @@ def _resize_uploaded_avatar_image(image: Image.Image, *, max_width: int, max_hei
|
||||
return fitted
|
||||
|
||||
|
||||
def _avatar_image_has_alpha(image: Image.Image) -> bool:
|
||||
if "A" not in image.getbands():
|
||||
return False
|
||||
alpha = image.getchannel("A")
|
||||
low, high = alpha.getextrema()
|
||||
if not isinstance(low, int | float) or not isinstance(high, int | float):
|
||||
return False
|
||||
return low < 255 or high < 255
|
||||
|
||||
|
||||
def _update_manifest_matting_status(manifest_path: Path, image: Image.Image) -> None:
|
||||
raw = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
metadata = dict(raw.get("metadata") or {})
|
||||
metadata["matting_status"] = "transparent_ready" if _avatar_image_has_alpha(image) else "opaque"
|
||||
raw["metadata"] = metadata
|
||||
manifest_path.write_text(json.dumps(raw, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def _update_manifest_dimensions(manifest_path: Path, image: Image.Image) -> None:
|
||||
raw = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
raw["width"] = int(image.width)
|
||||
@@ -916,6 +945,7 @@ async def create_custom_avatar(
|
||||
max_w, max_h = _custom_avatar_max_size()
|
||||
fitted_image = _resize_uploaded_avatar_image(image_rgb, max_width=max_w, max_height=max_h)
|
||||
_update_manifest_dimensions(target_dir / "manifest.json", fitted_image)
|
||||
_update_manifest_matting_status(target_dir / "manifest.json", fitted_image)
|
||||
fitted_image.save(target_dir / "preview.png", format="PNG")
|
||||
fitted_image.save(target_dir / "reference.png", format="PNG")
|
||||
source_dir = target_dir / "source"
|
||||
|
||||
104
apps/api/routes/scene_assets.py
Normal file
104
apps/api/routes/scene_assets.py
Normal file
@@ -0,0 +1,104 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, File, Form, HTTPException, Request, UploadFile
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from apps.api.schemas.scene_assets import (
|
||||
CreateSceneCompositionRequest,
|
||||
UpdateSceneCompositionRequest,
|
||||
)
|
||||
from opentalking.scene_assets import SceneAssetStore
|
||||
|
||||
|
||||
router = APIRouter(prefix="/scene-assets", tags=["scene-assets"])
|
||||
|
||||
|
||||
def _store(request: Request) -> SceneAssetStore:
|
||||
settings = request.app.state.settings
|
||||
root = Path(getattr(settings, "scene_assets_dir", "./data/scene-assets"))
|
||||
return SceneAssetStore(root)
|
||||
|
||||
|
||||
@router.get("/backgrounds", response_model=None)
|
||||
async def list_backgrounds(request: Request) -> dict[str, object]:
|
||||
return {"items": _store(request).list_backgrounds()}
|
||||
|
||||
|
||||
@router.post("/backgrounds", response_model=None)
|
||||
async def upload_background(
|
||||
request: Request,
|
||||
file: UploadFile = File(...),
|
||||
name: str = Form(""),
|
||||
) -> dict[str, object]:
|
||||
max_bytes = int(getattr(request.app.state.settings, "scene_asset_max_bytes", 200 * 1024 * 1024))
|
||||
content = await file.read(max_bytes + 1)
|
||||
if not content:
|
||||
raise HTTPException(status_code=400, detail="empty background asset")
|
||||
if len(content) > max_bytes:
|
||||
raise HTTPException(status_code=413, detail="background asset exceeds size limit")
|
||||
try:
|
||||
return _store(request).create_background(
|
||||
content=content,
|
||||
filename=file.filename or "background",
|
||||
mime_type=file.content_type or "application/octet-stream",
|
||||
name=name or Path(file.filename or "background").stem,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@router.get("/backgrounds/{background_id}/file", response_model=None)
|
||||
async def download_background(background_id: str, request: Request) -> FileResponse:
|
||||
store = _store(request)
|
||||
path = store.background_file_path(background_id)
|
||||
if path is None:
|
||||
raise HTTPException(status_code=404, detail="background not found")
|
||||
item = next((entry for entry in store.list_backgrounds() if entry.get("id") == background_id), None)
|
||||
return FileResponse(path, media_type=str(item.get("mime_type") if item else "application/octet-stream"))
|
||||
|
||||
|
||||
@router.delete("/backgrounds/{background_id}", response_model=None)
|
||||
async def delete_background(background_id: str, request: Request) -> dict[str, object]:
|
||||
deleted = _store(request).delete_background(background_id)
|
||||
if not deleted:
|
||||
raise HTTPException(status_code=404, detail="background not found")
|
||||
return {"id": background_id, "deleted": True}
|
||||
|
||||
|
||||
@router.get("/compositions", response_model=None)
|
||||
async def list_compositions(request: Request) -> dict[str, object]:
|
||||
return {"items": _store(request).list_compositions()}
|
||||
|
||||
|
||||
@router.post("/compositions", response_model=None)
|
||||
async def create_composition(payload: CreateSceneCompositionRequest, request: Request) -> dict[str, object]:
|
||||
try:
|
||||
return _store(request).create_composition(payload.model_dump())
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@router.patch("/compositions/{composition_id}", response_model=None)
|
||||
async def update_composition(
|
||||
composition_id: str,
|
||||
payload: UpdateSceneCompositionRequest,
|
||||
request: Request,
|
||||
) -> dict[str, object]:
|
||||
data = payload.model_dump(exclude_unset=True)
|
||||
try:
|
||||
updated = _store(request).update_composition(composition_id, data)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
if updated is None:
|
||||
raise HTTPException(status_code=404, detail="composition not found")
|
||||
return updated
|
||||
|
||||
|
||||
@router.delete("/compositions/{composition_id}", response_model=None)
|
||||
async def delete_composition(composition_id: str, request: Request) -> dict[str, object]:
|
||||
deleted = _store(request).delete_composition(composition_id)
|
||||
if not deleted:
|
||||
raise HTTPException(status_code=404, detail="composition not found")
|
||||
return {"id": composition_id, "deleted": True}
|
||||
@@ -14,3 +14,4 @@ class AvatarSummary(BaseModel):
|
||||
# True for avatars created via POST /avatars/custom; only these are deletable.
|
||||
is_custom: bool = False
|
||||
has_preview_video: bool = False
|
||||
matting_status: str = "unknown"
|
||||
|
||||
53
apps/api/schemas/scene_assets.py
Normal file
53
apps/api/schemas/scene_assets.py
Normal file
@@ -0,0 +1,53 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class SceneBackgroundAsset(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
kind: str
|
||||
mime_type: str
|
||||
filename: str
|
||||
size_bytes: int
|
||||
url: str
|
||||
created_at: str
|
||||
|
||||
|
||||
class SceneComposition(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
avatar_id: str
|
||||
background_id: str | None = None
|
||||
background_color: str = "#0f172a"
|
||||
avatar_fit: str = "contain"
|
||||
avatar_scale: float = Field(default=1.0, ge=0.5, le=2.0)
|
||||
avatar_anchor: str = "center"
|
||||
matting_required: bool = False
|
||||
subtitle_style: str = "lower-third"
|
||||
created_at: str
|
||||
updated_at: str
|
||||
|
||||
|
||||
class CreateSceneCompositionRequest(BaseModel):
|
||||
name: str
|
||||
avatar_id: str
|
||||
background_id: str | None = None
|
||||
background_color: str = "#0f172a"
|
||||
avatar_fit: str = "contain"
|
||||
avatar_scale: float = Field(default=1.0, ge=0.5, le=2.0)
|
||||
avatar_anchor: str = "center"
|
||||
matting_required: bool = False
|
||||
subtitle_style: str = "lower-third"
|
||||
|
||||
|
||||
class UpdateSceneCompositionRequest(BaseModel):
|
||||
name: str | None = None
|
||||
avatar_id: str | None = None
|
||||
background_id: str | None = None
|
||||
background_color: str | None = None
|
||||
avatar_fit: str | None = None
|
||||
avatar_scale: float | None = Field(default=None, ge=0.5, le=2.0)
|
||||
avatar_anchor: str | None = None
|
||||
matting_required: bool | None = None
|
||||
subtitle_style: str | None = None
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import hashlib
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
@@ -12,6 +13,8 @@ from PIL import Image
|
||||
from apps.cli import prepare_cache
|
||||
from apps.api.routes import avatars, sessions
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
def _png_bytes(size: tuple[int, int] = (8, 8)) -> bytes:
|
||||
out = BytesIO()
|
||||
@@ -19,6 +22,14 @@ def _png_bytes(size: tuple[int, int] = (8, 8)) -> bytes:
|
||||
return out.getvalue()
|
||||
|
||||
|
||||
def _transparent_png_bytes(size: tuple[int, int] = (8, 8)) -> bytes:
|
||||
image = Image.new("RGBA", size, (10, 180, 210, 255))
|
||||
image.putpixel((0, 0), (10, 180, 210, 0))
|
||||
out = BytesIO()
|
||||
image.save(out, format="PNG")
|
||||
return out.getvalue()
|
||||
|
||||
|
||||
def _path_has_suffix(path: str, *suffix: str) -> bool:
|
||||
return Path(path).parts[-len(suffix):] == suffix
|
||||
|
||||
@@ -74,6 +85,106 @@ def test_create_custom_avatar_adds_listed_asset_with_preview(tmp_path):
|
||||
assert preview.headers["content-type"] == "image/png"
|
||||
|
||||
|
||||
def test_avatar_summary_includes_matting_status(tmp_path: Path) -> None:
|
||||
root = tmp_path
|
||||
avatar_dir = root / "anchor"
|
||||
avatar_dir.mkdir()
|
||||
(avatar_dir / "manifest.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"id": "anchor",
|
||||
"name": "Anchor",
|
||||
"model_type": "flashtalk",
|
||||
"fps": 25,
|
||||
"sample_rate": 16000,
|
||||
"width": 1,
|
||||
"height": 1,
|
||||
"version": "1.0",
|
||||
"metadata": {"matting_status": "transparent_ready"},
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(avatar_dir / "preview.png").write_bytes(_png_bytes())
|
||||
(avatar_dir / "reference.png").write_bytes(_png_bytes())
|
||||
|
||||
app = FastAPI()
|
||||
app.state.settings = SimpleNamespace(avatars_dir=str(root))
|
||||
app.include_router(avatars.router)
|
||||
response = TestClient(app).get("/avatars")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()[0]["matting_status"] == "transparent_ready"
|
||||
|
||||
|
||||
def test_builtin_female_host_transparent_avatar_asset_is_ready() -> None:
|
||||
avatar_dir = REPO_ROOT / "examples" / "avatars" / "female-host-transparent"
|
||||
manifest = json.loads((avatar_dir / "manifest.json").read_text(encoding="utf-8"))
|
||||
reference = avatar_dir / "reference.png"
|
||||
preview = avatar_dir / "preview.png"
|
||||
source = avatar_dir / "source" / "source.png"
|
||||
|
||||
assert manifest["id"] == "female-host-transparent"
|
||||
assert manifest["name"] == "女主持(透明背景)"
|
||||
assert manifest["metadata"]["matting_status"] == "transparent_ready"
|
||||
assert manifest["metadata"]["source_image"] == "source/source.png"
|
||||
assert manifest["metadata"]["source_image_path"] == "reference.png"
|
||||
assert reference.is_file()
|
||||
assert preview.is_file()
|
||||
assert source.is_file()
|
||||
|
||||
image = Image.open(reference)
|
||||
assert image.mode == "RGBA"
|
||||
assert image.size == (manifest["width"], manifest["height"])
|
||||
assert image.getchannel("A").getextrema() == (0, 255)
|
||||
assert hashlib.sha256(reference.read_bytes()).hexdigest() == manifest["metadata"]["source_image_hash"]
|
||||
|
||||
|
||||
def test_create_custom_avatar_preserves_uploaded_png_alpha(tmp_path, monkeypatch):
|
||||
base = tmp_path / "base-avatar"
|
||||
base.mkdir()
|
||||
(base / "preview.png").write_bytes(_png_bytes())
|
||||
(base / "reference.png").write_bytes(_png_bytes())
|
||||
(base / "manifest.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"id": "base-avatar",
|
||||
"name": "Base Avatar",
|
||||
"model_type": "mock",
|
||||
"fps": 25,
|
||||
"sample_rate": 16000,
|
||||
"width": 8,
|
||||
"height": 8,
|
||||
"version": "1.0",
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(avatars.mouth_metadata, "detect_mouth_landmarks", lambda frame: None)
|
||||
|
||||
app = FastAPI()
|
||||
app.state.settings = SimpleNamespace(avatars_dir=str(tmp_path))
|
||||
app.include_router(avatars.router)
|
||||
client = TestClient(app)
|
||||
|
||||
response = client.post(
|
||||
"/avatars/custom",
|
||||
data={"base_avatar_id": "base-avatar", "name": "透明形象"},
|
||||
files={"image": ("avatar.png", _transparent_png_bytes(), "image/png")},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
created = response.json()
|
||||
assert created["matting_status"] == "transparent_ready"
|
||||
custom_dir = tmp_path / created["id"]
|
||||
manifest = json.loads((custom_dir / "manifest.json").read_text(encoding="utf-8"))
|
||||
assert manifest["metadata"]["matting_status"] == "transparent_ready"
|
||||
for rel in ("reference.png", "preview.png", "source/source.png"):
|
||||
image = Image.open(custom_dir / rel)
|
||||
assert image.mode == "RGBA"
|
||||
assert image.getchannel("A").getextrema()[0] == 0
|
||||
|
||||
|
||||
def test_quicktalk_model_root_falls_back_to_omnirt_model_root(tmp_path, monkeypatch):
|
||||
monkeypatch.delenv("OPENTALKING_QUICKTALK_ASSET_ROOT", raising=False)
|
||||
monkeypatch.delenv("OPENTALKING_QUICKTALK_MODEL_ROOT", raising=False)
|
||||
|
||||
290
apps/api/tests/test_scene_assets.py
Normal file
290
apps/api/tests/test_scene_assets.py
Normal file
@@ -0,0 +1,290 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import apps.unified.main as unified_main
|
||||
from apps.api.routes import scene_assets
|
||||
from opentalking.scene_assets import SceneAssetStore
|
||||
|
||||
|
||||
PNG_BYTES = (
|
||||
b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR"
|
||||
b"\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15\xc4\x89"
|
||||
b"\x00\x00\x00\rIDATx\x9cc\xf8\xff\xff?\x00\x05\xfe\x02\xfeA\xe2!\xbc"
|
||||
b"\x00\x00\x00\x00IEND\xaeB`\x82"
|
||||
)
|
||||
|
||||
|
||||
def test_scene_asset_store_creates_lists_and_deletes_background(tmp_path: Path) -> None:
|
||||
store = SceneAssetStore(tmp_path)
|
||||
|
||||
created = store.create_background(
|
||||
content=PNG_BYTES,
|
||||
filename="studio.png",
|
||||
mime_type="image/png",
|
||||
name="Studio",
|
||||
)
|
||||
|
||||
assert created["id"].startswith("bg-")
|
||||
assert created["name"] == "Studio"
|
||||
assert created["kind"] == "image"
|
||||
assert created["mime_type"] == "image/png"
|
||||
assert created["url"] == f"/scene-assets/backgrounds/{created['id']}/file"
|
||||
assert store.background_file_path(str(created["id"])).is_file()
|
||||
assert store.list_backgrounds()[0]["id"] == created["id"]
|
||||
assert store.delete_background(str(created["id"])) is True
|
||||
assert store.list_backgrounds() == []
|
||||
|
||||
|
||||
def test_scene_asset_store_rejects_unsupported_background_type(tmp_path: Path) -> None:
|
||||
store = SceneAssetStore(tmp_path)
|
||||
|
||||
with pytest.raises(ValueError, match="unsupported background media type"):
|
||||
store.create_background(
|
||||
content=b"not media",
|
||||
filename="notes.txt",
|
||||
mime_type="text/plain",
|
||||
name="Notes",
|
||||
)
|
||||
|
||||
|
||||
def test_scene_asset_store_rejects_spoofed_background_content(tmp_path: Path) -> None:
|
||||
store = SceneAssetStore(tmp_path)
|
||||
|
||||
with pytest.raises(ValueError, match="unsupported background media type"):
|
||||
store.create_background(
|
||||
content=b"plain text pretending to be an image",
|
||||
filename="studio.png",
|
||||
mime_type="image/png",
|
||||
name="Spoof",
|
||||
)
|
||||
|
||||
|
||||
def test_scene_asset_store_rejects_zero_avatar_scale(tmp_path: Path) -> None:
|
||||
store = SceneAssetStore(tmp_path)
|
||||
|
||||
with pytest.raises(ValueError, match="avatar_scale must be between 0.5 and 2.0"):
|
||||
store.create_composition(
|
||||
{
|
||||
"name": "Zero Scale",
|
||||
"avatar_id": "anchor",
|
||||
"avatar_scale": 0,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_scene_asset_store_rejects_unsafe_background_delete_id(tmp_path: Path) -> None:
|
||||
store = SceneAssetStore(tmp_path / "assets")
|
||||
outside_dir = tmp_path / "outside"
|
||||
outside_dir.mkdir()
|
||||
(outside_dir / "keep.txt").write_text("keep", encoding="utf-8")
|
||||
|
||||
unsafe_id = f"bg-valid/../../{outside_dir.name}"
|
||||
store.backgrounds_dir.mkdir(parents=True)
|
||||
store.background_index_path.write_text(
|
||||
f'[{{"id": "{unsafe_id}", "mime_type": "image/png"}}]\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
assert store.delete_background(unsafe_id) is False
|
||||
assert (outside_dir / "keep.txt").is_file()
|
||||
|
||||
|
||||
def test_scene_asset_store_creates_and_updates_composition(tmp_path: Path) -> None:
|
||||
store = SceneAssetStore(tmp_path)
|
||||
background = store.create_background(
|
||||
content=PNG_BYTES,
|
||||
filename="studio.png",
|
||||
mime_type="image/png",
|
||||
name="Studio",
|
||||
)
|
||||
|
||||
created = store.create_composition(
|
||||
{
|
||||
"name": "Demo Scene",
|
||||
"avatar_id": "anchor",
|
||||
"background_id": background["id"],
|
||||
"avatar_fit": "contain",
|
||||
"avatar_scale": 1.0,
|
||||
"avatar_anchor": "center",
|
||||
"matting_required": True,
|
||||
"subtitle_style": "lower-third",
|
||||
}
|
||||
)
|
||||
|
||||
assert created["id"].startswith("scene-")
|
||||
assert created["background_id"] == background["id"]
|
||||
assert created["matting_required"] is True
|
||||
assert store.list_compositions()[0]["id"] == created["id"]
|
||||
|
||||
updated = store.update_composition(str(created["id"]), {"name": "Updated", "avatar_scale": 1.25})
|
||||
assert updated is not None
|
||||
assert updated["name"] == "Updated"
|
||||
assert updated["avatar_scale"] == 1.25
|
||||
assert updated["avatar_id"] == "anchor"
|
||||
|
||||
assert store.delete_composition(str(created["id"])) is True
|
||||
assert store.list_compositions() == []
|
||||
|
||||
|
||||
def test_scene_asset_store_rejects_unknown_background_id(tmp_path: Path) -> None:
|
||||
store = SceneAssetStore(tmp_path)
|
||||
|
||||
with pytest.raises(ValueError, match="background_id not found"):
|
||||
store.create_composition(
|
||||
{
|
||||
"name": "Missing Background",
|
||||
"avatar_id": "anchor",
|
||||
"background_id": "bg-missing",
|
||||
}
|
||||
)
|
||||
|
||||
background = store.create_background(
|
||||
content=PNG_BYTES,
|
||||
filename="studio.png",
|
||||
mime_type="image/png",
|
||||
name="Studio",
|
||||
)
|
||||
created = store.create_composition(
|
||||
{
|
||||
"name": "Demo Scene",
|
||||
"avatar_id": "anchor",
|
||||
"background_id": background["id"],
|
||||
}
|
||||
)
|
||||
with pytest.raises(ValueError, match="background_id not found"):
|
||||
store.update_composition(str(created["id"]), {"background_id": "bg-missing"})
|
||||
|
||||
|
||||
def _client(tmp_path: Path) -> TestClient:
|
||||
app = FastAPI()
|
||||
app.state.settings = SimpleNamespace(
|
||||
scene_assets_dir=str(tmp_path / "scene-assets"),
|
||||
scene_asset_max_bytes=len(PNG_BYTES),
|
||||
)
|
||||
app.include_router(scene_assets.router)
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def test_scene_asset_api_uploads_lists_downloads_and_deletes_background(tmp_path: Path) -> None:
|
||||
with _client(tmp_path) as client:
|
||||
upload = client.post(
|
||||
"/scene-assets/backgrounds",
|
||||
data={"name": "Studio"},
|
||||
files={"file": ("studio.png", PNG_BYTES, "image/png")},
|
||||
)
|
||||
assert upload.status_code == 200
|
||||
item = upload.json()
|
||||
assert item["name"] == "Studio"
|
||||
assert item["url"] == f"/scene-assets/backgrounds/{item['id']}/file"
|
||||
|
||||
listed = client.get("/scene-assets/backgrounds")
|
||||
assert listed.status_code == 200
|
||||
assert listed.json()["items"][0]["id"] == item["id"]
|
||||
|
||||
downloaded = client.get(item["url"])
|
||||
assert downloaded.status_code == 200
|
||||
assert downloaded.headers["content-type"].startswith("image/png")
|
||||
|
||||
deleted = client.delete(f"/scene-assets/backgrounds/{item['id']}")
|
||||
assert deleted.status_code == 200
|
||||
assert deleted.json()["deleted"] is True
|
||||
|
||||
|
||||
def test_scene_asset_api_rejects_oversized_background_upload(tmp_path: Path) -> None:
|
||||
with _client(tmp_path) as client:
|
||||
upload = client.post(
|
||||
"/scene-assets/backgrounds",
|
||||
data={"name": "Too Big"},
|
||||
files={"file": ("studio.png", PNG_BYTES + b"x", "image/png")},
|
||||
)
|
||||
assert upload.status_code == 413
|
||||
assert upload.json()["detail"] == "background asset exceeds size limit"
|
||||
|
||||
|
||||
def test_scene_asset_api_rejects_spoofed_content_type(tmp_path: Path) -> None:
|
||||
with _client(tmp_path) as client:
|
||||
upload = client.post(
|
||||
"/scene-assets/backgrounds",
|
||||
data={"name": "Spoof"},
|
||||
files={"file": ("studio.png", b"this is plain text", "image/png")},
|
||||
)
|
||||
assert upload.status_code == 400
|
||||
assert upload.json()["detail"] == "unsupported background media type"
|
||||
|
||||
|
||||
def test_scene_asset_api_manages_compositions(tmp_path: Path) -> None:
|
||||
with _client(tmp_path) as client:
|
||||
bg = client.post(
|
||||
"/scene-assets/backgrounds",
|
||||
data={"name": "Studio"},
|
||||
files={"file": ("studio.png", PNG_BYTES, "image/png")},
|
||||
).json()
|
||||
|
||||
created = client.post(
|
||||
"/scene-assets/compositions",
|
||||
json={
|
||||
"name": "Demo Scene",
|
||||
"avatar_id": "anchor",
|
||||
"background_id": bg["id"],
|
||||
"avatar_fit": "contain",
|
||||
"avatar_scale": 1.0,
|
||||
"avatar_anchor": "center",
|
||||
"matting_required": True,
|
||||
"subtitle_style": "lower-third",
|
||||
},
|
||||
)
|
||||
assert created.status_code == 200
|
||||
scene = created.json()
|
||||
assert scene["background_id"] == bg["id"]
|
||||
|
||||
patched = client.patch(
|
||||
f"/scene-assets/compositions/{scene['id']}",
|
||||
json={"name": "Updated Scene", "avatar_scale": 1.2},
|
||||
)
|
||||
assert patched.status_code == 200
|
||||
assert patched.json()["name"] == "Updated Scene"
|
||||
assert patched.json()["avatar_scale"] == 1.2
|
||||
|
||||
listed = client.get("/scene-assets/compositions")
|
||||
assert listed.status_code == 200
|
||||
assert listed.json()["items"][0]["id"] == scene["id"]
|
||||
|
||||
deleted = client.delete(f"/scene-assets/compositions/{scene['id']}")
|
||||
assert deleted.status_code == 200
|
||||
assert deleted.json() == {"id": scene["id"], "deleted": True}
|
||||
|
||||
listed_after_delete = client.get("/scene-assets/compositions")
|
||||
assert listed_after_delete.status_code == 200
|
||||
assert listed_after_delete.json()["items"] == []
|
||||
|
||||
deleted_again = client.delete(f"/scene-assets/compositions/{scene['id']}")
|
||||
assert deleted_again.status_code == 404
|
||||
|
||||
|
||||
def test_scene_asset_api_rejects_unknown_background_id(tmp_path: Path) -> None:
|
||||
with _client(tmp_path) as client:
|
||||
created = client.post(
|
||||
"/scene-assets/compositions",
|
||||
json={
|
||||
"name": "Missing Background",
|
||||
"avatar_id": "anchor",
|
||||
"background_id": "bg-missing",
|
||||
},
|
||||
)
|
||||
assert created.status_code == 400
|
||||
assert created.json()["detail"] == "background_id not found"
|
||||
|
||||
|
||||
def test_unified_app_registers_scene_asset_routes() -> None:
|
||||
app = unified_main.create_app()
|
||||
|
||||
paths = set(app.openapi()["paths"])
|
||||
|
||||
assert "/scene-assets/backgrounds" in paths
|
||||
assert "/scene-assets/compositions" in paths
|
||||
@@ -19,7 +19,7 @@ from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from apps.api.core.config import get_settings
|
||||
from apps.api.routes.avatars import _call_adapter_warmup
|
||||
from apps.api.routes import agent, avatars, events, exports, health, memory, models, personas, runtime_config, sessions, tts_preview, video_clone, video_creation, voices
|
||||
from apps.api.routes import agent, avatars, events, exports, health, memory, models, personas, runtime_config, scene_assets, sessions, tts_preview, video_clone, video_creation, voices
|
||||
from opentalking.voice.store import init_voice_store
|
||||
from opentalking.core.in_memory_redis import InMemoryRedis
|
||||
from opentalking.pipeline.session.runner import SessionRunner
|
||||
@@ -219,6 +219,7 @@ def create_app() -> FastAPI:
|
||||
app.include_router(personas.router)
|
||||
app.include_router(events.router)
|
||||
app.include_router(exports.router)
|
||||
app.include_router(scene_assets.router)
|
||||
app.include_router(runtime_config.router)
|
||||
app.include_router(tts_preview.router)
|
||||
app.include_router(video_clone.router)
|
||||
|
||||
@@ -11,11 +11,12 @@ import {
|
||||
type Wav2LipPostprocessMode,
|
||||
} from "./components/SettingsPanel";
|
||||
import { RuntimeConfigWorkspace } from "./components/RuntimeConfigWorkspace";
|
||||
import { TopBar, type StudioWorkflow } from "./components/TopBar";
|
||||
import { SceneStage } from "./components/SceneStage";
|
||||
import { TopBar, type ConversationViewMode, type StudioWorkflow } from "./components/TopBar";
|
||||
import { ToastStack, type ToastMessage, type ToastTone } from "./components/ToastStack";
|
||||
import { VideoBackground } from "./components/VideoBackground";
|
||||
import { AssetLibraryWorkspace, type AssetLibraryTab } from "./components/AssetLibraryWorkspace";
|
||||
import { VideoCloneWorkspace } from "./components/VideoCloneWorkspace";
|
||||
import { playWithMutedFallback } from "./components/VideoBackground";
|
||||
import {
|
||||
DEFAULT_VIDEO_CREATION_FASTLIVEPORTRAIT_CONFIG,
|
||||
VideoCreationWorkspace,
|
||||
@@ -31,6 +32,8 @@ import {
|
||||
apiUploadFile,
|
||||
buildApiUrl,
|
||||
getMemoryLibraries,
|
||||
listSceneBackgrounds,
|
||||
listSceneCompositions,
|
||||
loadRuntimeConfig,
|
||||
uploadExportVideo,
|
||||
type AvatarKnowledgeBasesResponse,
|
||||
@@ -43,6 +46,8 @@ import {
|
||||
type PersonasResponse,
|
||||
type RuntimeConfigApplyInput,
|
||||
type RuntimeConfigResponse,
|
||||
type SceneBackgroundAsset,
|
||||
type SceneComposition,
|
||||
type SessionKnowledgeBasesRequest,
|
||||
type SessionKnowledgeBasesResponse,
|
||||
type VoiceCatalogItem,
|
||||
@@ -186,6 +191,9 @@ const ASR_PROVIDER_STORAGE_KEY = "opentalking-asr-provider-v1";
|
||||
const CLIENT_USER_ID_KEY = "opentalking-client-user-id";
|
||||
const AGENT_CONFIG_STORAGE_KEY = "opentalking-agent-config-v1";
|
||||
const SELECTED_PERSONA_STORAGE_KEY = "opentalking-selected-persona-id-v1";
|
||||
const SELECTED_SCENE_STORAGE_KEY = "opentalking-selected-scene-id-v2";
|
||||
const SELECTED_SCENE_BY_AVATAR_STORAGE_KEY = "opentalking-selected-scene-by-avatar-v1";
|
||||
const CONVERSATION_VIEW_MODE_KEY = "opentalking-conversation-view-mode-v1";
|
||||
const LEGACY_FASTLIVEPORTRAIT_DEFAULT_CONFIG: FasterLivePortraitConfig = {
|
||||
head_motion_multiplier: 1.0,
|
||||
pose_motion_multiplier: 0.35,
|
||||
@@ -347,6 +355,22 @@ function writeStoredAvatarId(avatarId: string, source: "auto" | "explicit" = "ex
|
||||
}
|
||||
}
|
||||
|
||||
function readStoredSceneIdsByAvatar(): Record<string, string> {
|
||||
try {
|
||||
const raw = window.localStorage.getItem(SELECTED_SCENE_BY_AVATAR_STORAGE_KEY);
|
||||
if (!raw) return {};
|
||||
const parsed = JSON.parse(raw);
|
||||
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return {};
|
||||
return Object.fromEntries(
|
||||
Object.entries(parsed).filter((entry): entry is [string, string] => (
|
||||
typeof entry[0] === "string" && typeof entry[1] === "string" && entry[0].length > 0 && entry[1].length > 0
|
||||
)),
|
||||
);
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function readOrCreateClientUserId(): string {
|
||||
try {
|
||||
const existing = window.localStorage.getItem(CLIENT_USER_ID_KEY)?.trim();
|
||||
@@ -828,6 +852,7 @@ function realtimeRecordingStartErrorMessage(error: unknown): string {
|
||||
|
||||
export default function App() {
|
||||
const videoRef = useRef<HTMLVideoElement>(null);
|
||||
const audioRef = useRef<HTMLAudioElement>(null);
|
||||
const pcRef = useRef<RTCPeerConnection | null>(null);
|
||||
const remoteStreamRef = useRef<MediaStream | null>(null);
|
||||
const realtimeRecorderRef = useRef<MediaRecorder | null>(null);
|
||||
@@ -886,6 +911,7 @@ export default function App() {
|
||||
// Connection
|
||||
const [connection, setConnection] = useState<ConnectionStatus>("idle");
|
||||
const [sessionId, setSessionId] = useState<string | null>(null);
|
||||
const [remoteStream, setRemoteStream] = useState<MediaStream | null>(null);
|
||||
const [queueInfo, setQueueInfo] = useState<QueueInfo | null>(null);
|
||||
const [expiringCountdown, setExpiringCountdown] = useState<number | null>(null);
|
||||
|
||||
@@ -972,6 +998,14 @@ export default function App() {
|
||||
const [ftRecordPhase, setFtRecordPhase] = useState<"idle" | "recording" | "stopped">("idle");
|
||||
const [ftRecordBusy, setFtRecordBusy] = useState(false);
|
||||
const [assetLibraryRefreshKey, setAssetLibraryRefreshKey] = useState(0);
|
||||
const [conversationViewMode, setConversationViewMode] = useState<ConversationViewMode>(() => {
|
||||
try {
|
||||
return window.localStorage.getItem(CONVERSATION_VIEW_MODE_KEY) === "immersive" ? "immersive" : "studio";
|
||||
} catch {
|
||||
return "studio";
|
||||
}
|
||||
});
|
||||
const [immersiveAvatarAdjust, setImmersiveAvatarAdjust] = useState({ x: 0, y: 0, scale: 1 });
|
||||
const [voiceCatalog, setVoiceCatalog] = useState<VoiceCatalogItem[]>([]);
|
||||
const [voiceApplyNotice, setVoiceApplyNotice] = useState<string | null>(null);
|
||||
const [ttsPreviewText, setTtsPreviewText] = useState(DEFAULT_TTS_PREVIEW_TEXT);
|
||||
@@ -982,6 +1016,9 @@ export default function App() {
|
||||
const [memoryEnabled, setMemoryEnabled] = useState(false);
|
||||
const [memoryLibraryId, setMemoryLibraryId] = useState<string | null>(null);
|
||||
const [memoryLibraries, setMemoryLibraries] = useState<MemoryLibrary[]>([]);
|
||||
const [sceneBackgrounds, setSceneBackgrounds] = useState<SceneBackgroundAsset[]>([]);
|
||||
const [sceneCompositions, setSceneCompositions] = useState<SceneComposition[]>([]);
|
||||
const [selectedSceneIdsByAvatar, setSelectedSceneIdsByAvatar] = useState<Record<string, string>>(readStoredSceneIdsByAvatar);
|
||||
const avatarKnowledgeBasesSyncReadyRef = useRef(false);
|
||||
const lastPersistedAvatarKnowledgeBasesRef = useRef<Map<string, string[]>>(new Map());
|
||||
const avatarKnowledgeBasesLoadSeqRef = useRef(0);
|
||||
@@ -1041,6 +1078,17 @@ export default function App() {
|
||||
});
|
||||
|
||||
const compactSquareStage = usesCompactSquareStage(model);
|
||||
const selectedScene = useMemo(
|
||||
() => {
|
||||
const selectedSceneId = selectedSceneIdsByAvatar[avatarId];
|
||||
const matchingScenes = sceneCompositions.filter((scene) => scene.avatar_id === avatarId);
|
||||
if (selectedSceneId) {
|
||||
return matchingScenes.find((scene) => scene.id === selectedSceneId) ?? null;
|
||||
}
|
||||
return matchingScenes.length === 1 ? matchingScenes[0] : null;
|
||||
},
|
||||
[avatarId, sceneCompositions, selectedSceneIdsByAvatar],
|
||||
);
|
||||
|
||||
const dismissToast = useCallback((id: string) => {
|
||||
setToasts((prev) => prev.filter((toast) => toast.id !== id));
|
||||
@@ -1177,6 +1225,63 @@ export default function App() {
|
||||
}
|
||||
}, [notify]);
|
||||
|
||||
const refreshScenes = useCallback(async () => {
|
||||
try {
|
||||
const [backgrounds, scenes] = await Promise.all([
|
||||
listSceneBackgrounds(),
|
||||
listSceneCompositions(),
|
||||
]);
|
||||
setSceneBackgrounds(backgrounds.items);
|
||||
setSceneCompositions(scenes.items);
|
||||
setSelectedSceneIdsByAvatar((current) => {
|
||||
const next = Object.fromEntries(
|
||||
Object.entries(current).filter(([sceneAvatarId, sceneId]) => (
|
||||
scenes.items.some((scene) => scene.id === sceneId && scene.avatar_id === sceneAvatarId)
|
||||
)),
|
||||
);
|
||||
try {
|
||||
const legacySceneId = window.localStorage.getItem(SELECTED_SCENE_STORAGE_KEY);
|
||||
const legacyScene = legacySceneId ? scenes.items.find((scene) => scene.id === legacySceneId) : null;
|
||||
if (legacyScene && !next[legacyScene.avatar_id]) {
|
||||
next[legacyScene.avatar_id] = legacyScene.id;
|
||||
}
|
||||
window.localStorage.removeItem(SELECTED_SCENE_STORAGE_KEY);
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
return next;
|
||||
});
|
||||
} catch (error) {
|
||||
console.warn("load scene assets failed", error);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const handleSceneCompositionsChange = useCallback((scenes: SceneComposition[]) => {
|
||||
setSceneCompositions(scenes);
|
||||
setSelectedSceneIdsByAvatar((current) => {
|
||||
return Object.fromEntries(
|
||||
Object.entries(current).filter(([sceneAvatarId, sceneId]) => (
|
||||
scenes.some((scene) => scene.id === sceneId && scene.avatar_id === sceneAvatarId)
|
||||
)),
|
||||
);
|
||||
});
|
||||
}, []);
|
||||
|
||||
const handleSceneSelect = useCallback((scene: SceneComposition) => {
|
||||
setSelectedSceneIdsByAvatar((current) => ({
|
||||
...current,
|
||||
[scene.avatar_id]: scene.id,
|
||||
}));
|
||||
}, []);
|
||||
|
||||
const handleSceneClear = useCallback((sceneAvatarId: string) => {
|
||||
setSelectedSceneIdsByAvatar((current) => {
|
||||
const next = { ...current };
|
||||
delete next[sceneAvatarId];
|
||||
return next;
|
||||
});
|
||||
}, []);
|
||||
|
||||
const refreshAvatarKnowledgeBases = useCallback(async (targetAvatarId: string) => {
|
||||
if (!targetAvatarId) return;
|
||||
const seq = ++avatarKnowledgeBasesLoadSeqRef.current;
|
||||
@@ -1259,6 +1364,46 @@ export default function App() {
|
||||
if (workflow === "realtime") void refreshPersonas();
|
||||
}, [refreshPersonas, workflow]);
|
||||
|
||||
useEffect(() => {
|
||||
void refreshScenes();
|
||||
}, [refreshScenes]);
|
||||
|
||||
useEffect(() => {
|
||||
try {
|
||||
if (Object.keys(selectedSceneIdsByAvatar).length) {
|
||||
window.localStorage.setItem(SELECTED_SCENE_BY_AVATAR_STORAGE_KEY, JSON.stringify(selectedSceneIdsByAvatar));
|
||||
} else {
|
||||
window.localStorage.removeItem(SELECTED_SCENE_BY_AVATAR_STORAGE_KEY);
|
||||
}
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}, [selectedSceneIdsByAvatar]);
|
||||
|
||||
useEffect(() => {
|
||||
try {
|
||||
window.localStorage.setItem(CONVERSATION_VIEW_MODE_KEY, conversationViewMode);
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}, [conversationViewMode]);
|
||||
|
||||
useEffect(() => {
|
||||
const startupVisible = connection === "idle" || connection === "error" || connection === "connecting" || connection === "queued";
|
||||
if (startupVisible && conversationViewMode === "immersive") setConversationViewMode("studio");
|
||||
}, [connection, conversationViewMode]);
|
||||
|
||||
useEffect(() => {
|
||||
if (workflow !== "realtime" || conversationViewMode !== "immersive") return;
|
||||
const handleKeyDown = (event: KeyboardEvent) => {
|
||||
if (event.key !== "Escape") return;
|
||||
event.preventDefault();
|
||||
setConversationViewMode("studio");
|
||||
};
|
||||
window.addEventListener("keydown", handleKeyDown);
|
||||
return () => window.removeEventListener("keydown", handleKeyDown);
|
||||
}, [conversationViewMode, workflow]);
|
||||
|
||||
useEffect(() => {
|
||||
if (selectedPersonaId) return;
|
||||
void refreshAvatarKnowledgeBases(avatarId);
|
||||
@@ -1560,6 +1705,37 @@ export default function App() {
|
||||
sessionIdRef.current = sessionId;
|
||||
}, [sessionId]);
|
||||
|
||||
useEffect(() => {
|
||||
const video = videoRef.current;
|
||||
if (!video) return;
|
||||
if (video.srcObject !== remoteStream) {
|
||||
video.srcObject = remoteStream;
|
||||
}
|
||||
if (remoteStream) {
|
||||
video.muted = false;
|
||||
video.volume = 1;
|
||||
playWithMutedFallback(video);
|
||||
} else {
|
||||
video.muted = true;
|
||||
}
|
||||
}, [conversationViewMode, remoteStream, workflow]);
|
||||
|
||||
useEffect(() => {
|
||||
const audio = audioRef.current;
|
||||
if (!audio) return;
|
||||
if (!remoteStream) {
|
||||
audio.srcObject = null;
|
||||
return;
|
||||
}
|
||||
audio.srcObject = remoteStream;
|
||||
audio.muted = false;
|
||||
audio.volume = 1;
|
||||
void audio.play().catch(() => {});
|
||||
return () => {
|
||||
audio.srcObject = null;
|
||||
};
|
||||
}, [remoteStream]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (ttsPreviewUrlRef.current) {
|
||||
@@ -1656,6 +1832,7 @@ export default function App() {
|
||||
for (const track of remoteStreamRef.current.getTracks()) track.stop();
|
||||
remoteStreamRef.current = null;
|
||||
}
|
||||
setRemoteStream(null);
|
||||
}, []);
|
||||
|
||||
const releaseSession = useCallback(async (sid: string, keepalive = false) => {
|
||||
@@ -2047,10 +2224,12 @@ export default function App() {
|
||||
const playback = await startPlayback(created.session_id, videoRef.current!, {
|
||||
onRemoteStream: (remoteStream) => {
|
||||
remoteStreamRef.current = remoteStream;
|
||||
setRemoteStream(remoteStream);
|
||||
},
|
||||
});
|
||||
pcRef.current = playback.pc;
|
||||
remoteStreamRef.current = playback.remoteStream;
|
||||
setRemoteStream(playback.remoteStream);
|
||||
setActiveAsrProvider(lockedAsrProvider);
|
||||
videoRef.current!.muted = false;
|
||||
setConnection("live");
|
||||
@@ -2605,9 +2784,14 @@ export default function App() {
|
||||
}, [closePeerConnection, releaseSession]);
|
||||
|
||||
const currentAvatar = avatars.find((a) => a.id === avatarId) ?? null;
|
||||
const selectedAvatarMaskUrl = selectedScene && currentAvatar?.matting_status === "transparent_ready"
|
||||
? buildApiUrl(`/avatars/${encodeURIComponent(currentAvatar.id)}/preview`)
|
||||
: null;
|
||||
const sessionConfigLocked = connection === "connecting" || connection === "queued" || connection === "live" || connection === "expiring";
|
||||
const effectiveAsrProvider = activeAsrProvider || normalizeAsrProvider(asrProvider, "dashscope");
|
||||
const showStart = connection === "idle" || connection === "error" || connection === "connecting" || connection === "queued";
|
||||
const effectiveConversationViewMode = showStart ? "studio" : conversationViewMode;
|
||||
const immersiveActive = workflow === "realtime" && effectiveConversationViewMode === "immersive";
|
||||
const chatMaxVisible = readChatMaxVisible();
|
||||
const selectedModelLabel = MODEL_LABELS_FOR_STAGE[model] ?? model;
|
||||
const wav2lipPostprocessModeLocked = sessionId !== null && connection !== "idle" && connection !== "error";
|
||||
@@ -2640,9 +2824,12 @@ export default function App() {
|
||||
};
|
||||
return (
|
||||
<div className="min-h-screen bg-slate-100 text-slate-900 lg:h-screen lg:overflow-hidden">
|
||||
<audio ref={audioRef} autoPlay playsInline className="hidden" />
|
||||
<TopBar
|
||||
connection={connection}
|
||||
workflow={workflow}
|
||||
conversationViewMode={effectiveConversationViewMode}
|
||||
immersiveChrome={immersiveActive}
|
||||
flashtalkRecording={!!sessionId && (connection === "live" || connection === "expiring")}
|
||||
flashtalkRecordPhase={ftRecordPhase}
|
||||
flashtalkRecordBusy={ftRecordBusy}
|
||||
@@ -2650,6 +2837,7 @@ export default function App() {
|
||||
runtimeConfigReady={runtimeConfigReady}
|
||||
runtimeConfigLoading={runtimeConfigLoading}
|
||||
onInactiveModuleClick={(label) => notify(`${label}模块规划中。当前可用的是实时对话、视频克隆、数字人配置、语音驱动和导出能力。`, "info")}
|
||||
onConversationViewModeChange={setConversationViewMode}
|
||||
onWorkflowChange={(next) => {
|
||||
setWorkflow(next);
|
||||
if (next === "videoClone" && sessionIdRef.current) {
|
||||
@@ -2708,6 +2896,12 @@ export default function App() {
|
||||
onMemoryEnabledChange={setMemoryEnabled}
|
||||
onMemoryLibrariesChange={setMemoryLibraries}
|
||||
onRefreshMemoryLibraries={() => void refreshMemoryLibraries()}
|
||||
avatars={avatars}
|
||||
selectedSceneIdsByAvatar={selectedSceneIdsByAvatar}
|
||||
onSceneSelect={handleSceneSelect}
|
||||
onSceneClear={handleSceneClear}
|
||||
onSceneBackgroundsChange={setSceneBackgrounds}
|
||||
onSceneCompositionsChange={handleSceneCompositionsChange}
|
||||
/>
|
||||
</div>
|
||||
) : workflow === "videoCreation" ? (
|
||||
@@ -2761,8 +2955,14 @@ export default function App() {
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
<div className="flex min-h-0 flex-col lg:h-[calc(100vh-3.5rem)] lg:flex-row">
|
||||
<div className="order-2 min-h-0 lg:order-none lg:h-full lg:shrink-0">
|
||||
<div
|
||||
className={
|
||||
immersiveActive
|
||||
? "relative flex h-dvh min-h-0 flex-col bg-slate-950"
|
||||
: "flex min-h-0 flex-col lg:h-[calc(100vh-3.5rem)] lg:flex-row"
|
||||
}
|
||||
>
|
||||
<div className={`${immersiveActive ? "hidden" : "order-2 min-h-0 lg:order-none lg:h-full lg:shrink-0"}`}>
|
||||
<SettingsPanel
|
||||
expanded={settingsExpanded}
|
||||
onExpandedChange={setSettingsExpanded}
|
||||
@@ -2826,55 +3026,141 @@ export default function App() {
|
||||
/>
|
||||
</div>
|
||||
|
||||
<main className="order-1 flex min-h-0 flex-1 flex-col bg-slate-100 lg:order-none">
|
||||
<div className="flex min-h-0 flex-1 flex-col p-4">
|
||||
<div className="relative min-h-[360px] flex-1 overflow-hidden rounded-lg border border-slate-200 bg-white shadow-sm shadow-slate-200/70 lg:min-h-[420px]">
|
||||
<div className="absolute inset-0 bg-slate-50" />
|
||||
<div className="absolute inset-3 rounded-lg border border-slate-200 bg-white shadow-inner shadow-slate-200/60" />
|
||||
<div className="absolute inset-0 flex items-center justify-center p-4 sm:p-6 lg:p-8">
|
||||
<div
|
||||
className={
|
||||
compactSquareStage
|
||||
? "relative aspect-square w-full max-w-[42rem] max-h-full"
|
||||
: "relative h-full w-full"
|
||||
}
|
||||
>
|
||||
<VideoBackground ref={videoRef} className="absolute inset-0 h-full w-full object-contain" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="absolute left-4 right-4 top-4 z-30 flex flex-col gap-2 sm:flex-row sm:items-start sm:justify-between">
|
||||
<div className="flex min-w-0 flex-wrap gap-2">
|
||||
<span className="inline-flex items-center gap-1 rounded-full border border-white/15 bg-white/90 px-2.5 py-1 text-xs font-medium text-slate-700 shadow-sm">
|
||||
<span className={`h-1.5 w-1.5 rounded-full ${
|
||||
connection === "live" || connection === "expiring" ? "bg-emerald-500" : "bg-slate-400"
|
||||
}`} />
|
||||
{connection === "live" || connection === "expiring" ? "已连接" : "待启动"}
|
||||
</span>
|
||||
<span className="inline-flex items-center gap-1 rounded-full border border-cyan-200 bg-cyan-50 px-2.5 py-1 text-xs font-medium text-cyan-700 shadow-sm">
|
||||
WebRTC 舞台
|
||||
</span>
|
||||
<span className="inline-flex items-center gap-1 rounded-full border border-slate-200 bg-slate-50 px-2.5 py-1 text-xs font-medium text-slate-600 shadow-sm">
|
||||
{MODEL_LABELS_FOR_STAGE[model] ?? model}
|
||||
</span>
|
||||
<span className="inline-flex max-w-[14rem] items-center gap-1 truncate rounded-full border border-slate-200 bg-white/90 px-2.5 py-1 text-xs font-medium text-slate-600 shadow-sm">
|
||||
{currentAvatar?.name ?? currentAvatar?.id ?? "未选形象"}
|
||||
</span>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleReturnToAvatarSelection}
|
||||
className="shrink-0 rounded-lg border border-slate-200 bg-white/95 px-3 py-2 text-xs font-semibold text-slate-700 shadow-sm transition hover:border-cyan-200 hover:text-cyan-700"
|
||||
>
|
||||
更换形象
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{currentSubtitle && !showStart ? (
|
||||
<div className="absolute inset-x-4 bottom-4 z-10 mx-auto max-w-xl rounded-lg border border-slate-200 bg-white/95 px-4 py-3 text-center text-sm font-medium leading-relaxed text-slate-900 shadow-lg shadow-slate-200/80 backdrop-blur">
|
||||
{currentSubtitle}
|
||||
</div>
|
||||
) : null}
|
||||
<main
|
||||
className={
|
||||
immersiveActive
|
||||
? "order-1 flex min-h-0 flex-1 flex-col overflow-hidden bg-slate-950 lg:order-none"
|
||||
: "order-1 flex min-h-0 flex-1 flex-col bg-slate-100 lg:order-none"
|
||||
}
|
||||
>
|
||||
<div
|
||||
className={
|
||||
immersiveActive
|
||||
? "relative flex min-h-0 flex-1 flex-col overflow-hidden"
|
||||
: "flex min-h-0 flex-1 flex-col p-4"
|
||||
}
|
||||
>
|
||||
<div
|
||||
className={
|
||||
immersiveActive
|
||||
? "absolute inset-x-0 top-0 bottom-12 overflow-hidden bg-black sm:bottom-14"
|
||||
: "relative min-h-[360px] flex-1 overflow-hidden rounded-lg border border-slate-200 bg-white shadow-sm shadow-slate-200/70 lg:min-h-[420px]"
|
||||
}
|
||||
>
|
||||
<SceneStage
|
||||
videoRef={videoRef}
|
||||
videoStream={remoteStream}
|
||||
scene={showStart ? null : selectedScene}
|
||||
backgrounds={sceneBackgrounds}
|
||||
subtitle={!showStart ? currentSubtitle : null}
|
||||
avatarMaskUrl={showStart ? null : selectedAvatarMaskUrl}
|
||||
avatarAdjust={immersiveActive ? immersiveAvatarAdjust : undefined}
|
||||
compactSquareStage={compactSquareStage}
|
||||
className="h-full w-full"
|
||||
>
|
||||
{immersiveActive ? (
|
||||
<>
|
||||
<div className="absolute right-0 top-0 z-30 flex h-24 w-48 items-start justify-end p-4">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setConversationViewMode("studio")}
|
||||
className="rounded-lg border border-white/15 bg-slate-950/45 px-3 py-2 text-xs font-semibold text-white opacity-0 shadow-lg backdrop-blur transition hover:bg-slate-950/65 hover:opacity-100 focus:opacity-100"
|
||||
>
|
||||
返回工作台
|
||||
</button>
|
||||
</div>
|
||||
<div className="group absolute right-0 top-1/2 z-30 flex -translate-y-1/2 translate-x-[calc(100%-1.25rem)] items-center transition-transform duration-200 hover:translate-x-0 focus-within:translate-x-0">
|
||||
<div className="flex h-20 w-5 items-center justify-center rounded-l-lg border border-r-0 border-slate-700 bg-slate-950 text-[10px] font-semibold text-white shadow-lg">
|
||||
微调
|
||||
</div>
|
||||
<div className="w-64 rounded-l-xl border border-slate-700 bg-slate-950 p-4 text-white shadow-2xl shadow-slate-950/30">
|
||||
<div className="mb-3 flex items-center justify-between gap-3">
|
||||
<p className="text-sm font-semibold">画面微调</p>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setImmersiveAvatarAdjust({ x: 0, y: 0, scale: 1 })}
|
||||
className="rounded-md border border-white/15 px-2 py-1 text-xs font-semibold text-white/80 transition hover:bg-white/10 hover:text-white"
|
||||
>
|
||||
重置
|
||||
</button>
|
||||
</div>
|
||||
<label className="mb-3 block text-xs font-medium text-white/80">
|
||||
<span className="mb-1 flex justify-between">
|
||||
<span>水平</span>
|
||||
<span className="tabular-nums">{immersiveAvatarAdjust.x}px</span>
|
||||
</span>
|
||||
<input
|
||||
type="range"
|
||||
min="-480"
|
||||
max="480"
|
||||
step="4"
|
||||
value={immersiveAvatarAdjust.x}
|
||||
onChange={(event) => setImmersiveAvatarAdjust((prev) => ({ ...prev, x: Number(event.target.value) }))}
|
||||
className="w-full accent-cyan-300"
|
||||
/>
|
||||
</label>
|
||||
<label className="mb-3 block text-xs font-medium text-white/80">
|
||||
<span className="mb-1 flex justify-between">
|
||||
<span>垂直</span>
|
||||
<span className="tabular-nums">{immersiveAvatarAdjust.y}px</span>
|
||||
</span>
|
||||
<input
|
||||
type="range"
|
||||
min="-320"
|
||||
max="320"
|
||||
step="4"
|
||||
value={immersiveAvatarAdjust.y}
|
||||
onChange={(event) => setImmersiveAvatarAdjust((prev) => ({ ...prev, y: Number(event.target.value) }))}
|
||||
className="w-full accent-cyan-300"
|
||||
/>
|
||||
</label>
|
||||
<label className="block text-xs font-medium text-white/80">
|
||||
<span className="mb-1 flex justify-between">
|
||||
<span>缩放</span>
|
||||
<span className="tabular-nums">{immersiveAvatarAdjust.scale.toFixed(2)}x</span>
|
||||
</span>
|
||||
<input
|
||||
type="range"
|
||||
min="0.4"
|
||||
max="2.2"
|
||||
step="0.02"
|
||||
value={immersiveAvatarAdjust.scale}
|
||||
onChange={(event) => setImmersiveAvatarAdjust((prev) => ({ ...prev, scale: Number(event.target.value) }))}
|
||||
className="w-full accent-cyan-300"
|
||||
/>
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
) : (
|
||||
<div className="absolute left-4 right-4 top-4 z-30 flex flex-col gap-2 sm:flex-row sm:items-start sm:justify-between">
|
||||
<div className="flex min-w-0 flex-wrap gap-2">
|
||||
<span className="inline-flex items-center gap-1 rounded-full border border-white/15 bg-white/90 px-2.5 py-1 text-xs font-medium text-slate-700 shadow-sm">
|
||||
<span className={`h-1.5 w-1.5 rounded-full ${
|
||||
connection === "live" || connection === "expiring" ? "bg-emerald-500" : "bg-slate-400"
|
||||
}`} />
|
||||
{connection === "live" || connection === "expiring" ? "已连接" : "待启动"}
|
||||
</span>
|
||||
<span className="inline-flex items-center gap-1 rounded-full border border-cyan-200 bg-cyan-50 px-2.5 py-1 text-xs font-medium text-cyan-700 shadow-sm">
|
||||
WebRTC 舞台
|
||||
</span>
|
||||
<span className="inline-flex items-center gap-1 rounded-full border border-slate-200 bg-slate-50 px-2.5 py-1 text-xs font-medium text-slate-600 shadow-sm">
|
||||
{MODEL_LABELS_FOR_STAGE[model] ?? model}
|
||||
</span>
|
||||
<span className="inline-flex max-w-[14rem] items-center gap-1 truncate rounded-full border border-slate-200 bg-white/90 px-2.5 py-1 text-xs font-medium text-slate-600 shadow-sm">
|
||||
{currentAvatar?.name ?? currentAvatar?.id ?? "未选形象"}
|
||||
</span>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleReturnToAvatarSelection}
|
||||
className="shrink-0 rounded-lg border border-slate-200 bg-white/95 px-3 py-2 text-xs font-semibold text-slate-700 shadow-sm transition hover:border-cyan-200 hover:text-cyan-700"
|
||||
>
|
||||
更换形象
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</SceneStage>
|
||||
|
||||
{showStart ? (
|
||||
<div className="absolute inset-0 z-40 bg-white">
|
||||
@@ -2909,7 +3195,18 @@ export default function App() {
|
||||
</div>
|
||||
|
||||
{connection === "live" || connection === "expiring" ? (
|
||||
<div className="mt-4">
|
||||
<div
|
||||
className={
|
||||
immersiveActive
|
||||
? "group absolute inset-x-3 bottom-0 z-30 mx-auto max-w-4xl translate-y-[calc(100%-1.25rem)] pb-3 transition-transform duration-200 hover:translate-y-0 focus-within:translate-y-0 sm:pb-5"
|
||||
: "mt-4"
|
||||
}
|
||||
>
|
||||
{immersiveActive ? (
|
||||
<div className="pointer-events-none mb-2 flex justify-center opacity-100 transition-opacity duration-150 group-hover:opacity-0 group-focus-within:opacity-0">
|
||||
<div className="h-1.5 w-16 rounded-full bg-white/80 shadow-sm" />
|
||||
</div>
|
||||
) : null}
|
||||
<ChatInput
|
||||
onSend={handleSend}
|
||||
onSpeakAudio={handleSpeakAudio}
|
||||
@@ -2936,9 +3233,13 @@ export default function App() {
|
||||
</main>
|
||||
|
||||
<aside
|
||||
className={`order-3 min-h-0 overflow-hidden border-l border-slate-200 bg-white transition-[width] duration-200 lg:shrink-0 ${
|
||||
sessionPanelCollapsed ? "lg:w-12" : "lg:w-[360px]"
|
||||
}`}
|
||||
className={
|
||||
immersiveActive
|
||||
? "hidden"
|
||||
: `order-3 min-h-0 overflow-hidden border-l border-slate-200 bg-white transition-[width] duration-200 lg:shrink-0 ${
|
||||
sessionPanelCollapsed ? "lg:w-12" : "lg:w-[360px]"
|
||||
}`
|
||||
}
|
||||
>
|
||||
<div className="flex h-full min-h-0">
|
||||
<div className="flex w-12 shrink-0 flex-col items-center justify-center gap-4 border-r border-slate-100 bg-slate-50 py-3">
|
||||
|
||||
@@ -7,16 +7,25 @@ import {
|
||||
apiPostForm,
|
||||
buildApiDownloadUrl,
|
||||
buildApiUrl,
|
||||
createSceneComposition,
|
||||
deleteSceneBackground,
|
||||
deleteSceneComposition,
|
||||
listSceneBackgrounds,
|
||||
listSceneCompositions,
|
||||
uploadSceneBackground,
|
||||
type AvatarSummary,
|
||||
type ExportVideoItem,
|
||||
type KnowledgeBaseSummary,
|
||||
type KnowledgeBasesResponse,
|
||||
type KnowledgeDocument,
|
||||
type KnowledgeDocumentsResponse,
|
||||
type SceneBackgroundAsset,
|
||||
type SceneComposition,
|
||||
} from "../lib/api";
|
||||
import { MemoryPanel } from "./MemoryPanel";
|
||||
import type { MemoryLibrary } from "../types";
|
||||
|
||||
type AssetTab = "exports" | "knowledge" | "memory" | "avatars" | "voices";
|
||||
type AssetTab = "exports" | "knowledge" | "memory" | "scenes" | "voices";
|
||||
export type AssetLibraryTab = AssetTab;
|
||||
|
||||
type AssetLibraryWorkspaceProps = {
|
||||
@@ -34,13 +43,19 @@ type AssetLibraryWorkspaceProps = {
|
||||
onMemoryEnabledChange?: (enabled: boolean) => void;
|
||||
onMemoryLibrariesChange?: (libraries: MemoryLibrary[]) => void;
|
||||
onRefreshMemoryLibraries?: () => void;
|
||||
avatars?: AvatarSummary[];
|
||||
onSceneCompositionsChange?: (scenes: SceneComposition[]) => void;
|
||||
selectedSceneIdsByAvatar?: Record<string, string>;
|
||||
onSceneSelect?: (scene: SceneComposition) => void;
|
||||
onSceneClear?: (avatarId: string) => void;
|
||||
onSceneBackgroundsChange?: (backgrounds: SceneBackgroundAsset[]) => void;
|
||||
};
|
||||
|
||||
const ASSET_TABS: { id: AssetTab; label: string; disabled?: boolean }[] = [
|
||||
{ id: "exports", label: "导出视频" },
|
||||
{ id: "knowledge", label: "知识库" },
|
||||
{ id: "memory", label: "记忆库" },
|
||||
{ id: "avatars", label: "Avatar资产", disabled: true },
|
||||
{ id: "scenes", label: "场景资产" },
|
||||
{ id: "voices", label: "声音资产", disabled: true },
|
||||
];
|
||||
|
||||
@@ -224,6 +239,12 @@ export function AssetLibraryWorkspace({
|
||||
onMemoryEnabledChange,
|
||||
onMemoryLibrariesChange,
|
||||
onRefreshMemoryLibraries,
|
||||
avatars,
|
||||
onSceneCompositionsChange,
|
||||
selectedSceneIdsByAvatar = {},
|
||||
onSceneSelect,
|
||||
onSceneClear,
|
||||
onSceneBackgroundsChange,
|
||||
}: AssetLibraryWorkspaceProps) {
|
||||
const [activeTab, setActiveTab] = useState<AssetTab>(initialTab);
|
||||
const [items, setItems] = useState<ExportVideoItem[]>([]);
|
||||
@@ -253,6 +274,14 @@ export function AssetLibraryWorkspace({
|
||||
const [creatingKnowledge, setCreatingKnowledge] = useState(false);
|
||||
const [filePoolUploading, setFilePoolUploading] = useState(false);
|
||||
const [memoryRefreshToken, setMemoryRefreshToken] = useState(0);
|
||||
const [sceneBackgrounds, setSceneBackgrounds] = useState<SceneBackgroundAsset[]>([]);
|
||||
const [sceneCompositions, setSceneCompositions] = useState<SceneComposition[]>([]);
|
||||
const [sceneLoading, setSceneLoading] = useState(false);
|
||||
const [sceneName, setSceneName] = useState("");
|
||||
const [sceneAvatarId, setSceneAvatarId] = useState("");
|
||||
const [sceneBackgroundId, setSceneBackgroundId] = useState("");
|
||||
const [backgroundName, setBackgroundName] = useState("");
|
||||
const [backgroundFile, setBackgroundFile] = useState<File | null>(null);
|
||||
const newKnowledgeFileInputRef = useRef<HTMLInputElement>(null);
|
||||
const uploadKnowledgeFileInputRef = useRef<HTMLInputElement>(null);
|
||||
const filePoolUploadInputRef = useRef<HTMLInputElement>(null);
|
||||
@@ -324,6 +353,27 @@ export function AssetLibraryWorkspace({
|
||||
}
|
||||
}, [onNotify]);
|
||||
|
||||
const loadScenes = useCallback(async () => {
|
||||
setSceneLoading(true);
|
||||
try {
|
||||
const [backgrounds, scenes] = await Promise.all([
|
||||
listSceneBackgrounds(),
|
||||
listSceneCompositions(),
|
||||
]);
|
||||
setSceneBackgrounds(backgrounds.items);
|
||||
setSceneCompositions(scenes.items);
|
||||
onSceneBackgroundsChange?.(backgrounds.items);
|
||||
onSceneCompositionsChange?.(scenes.items);
|
||||
setSceneAvatarId((current) => current || avatars?.[0]?.id || "");
|
||||
} catch (err) {
|
||||
console.warn("load scene assets failed", err);
|
||||
const detail = err instanceof ApiError ? err.detail : null;
|
||||
onNotify?.(detail ? `场景资产加载失败:${detail}` : "场景资产加载失败。", "error");
|
||||
} finally {
|
||||
setSceneLoading(false);
|
||||
}
|
||||
}, [avatars, onNotify, onSceneBackgroundsChange, onSceneCompositionsChange]);
|
||||
|
||||
useEffect(() => {
|
||||
if (activeTabOverride) setActiveTab(activeTabOverride);
|
||||
}, [activeTabOverride]);
|
||||
@@ -347,6 +397,10 @@ export function AssetLibraryWorkspace({
|
||||
}
|
||||
}, [activeTab, loadKnowledgeDocuments, selectedKnowledgeId]);
|
||||
|
||||
useEffect(() => {
|
||||
if (activeTab === "scenes") void loadScenes();
|
||||
}, [activeTab, loadScenes, refreshToken]);
|
||||
|
||||
const totalSize = useMemo(
|
||||
() => items.reduce((sum, item) => sum + item.size_bytes, 0),
|
||||
[items],
|
||||
@@ -386,8 +440,12 @@ export function AssetLibraryWorkspace({
|
||||
setMemoryRefreshToken((value) => value + 1);
|
||||
return;
|
||||
}
|
||||
if (activeTab === "scenes") {
|
||||
void loadScenes();
|
||||
return;
|
||||
}
|
||||
if (activeTab === "exports") void loadExports();
|
||||
}, [activeTab, loadAllKnowledgeDocuments, loadExports, loadKnowledgeBases, loadKnowledgeDocuments, onRefreshMemoryLibraries, selectedKnowledgeId]);
|
||||
}, [activeTab, loadAllKnowledgeDocuments, loadExports, loadKnowledgeBases, loadKnowledgeDocuments, loadScenes, onRefreshMemoryLibraries, selectedKnowledgeId]);
|
||||
|
||||
const handleCopyPath = useCallback(async (path: string) => {
|
||||
try {
|
||||
@@ -601,6 +659,76 @@ export function AssetLibraryWorkspace({
|
||||
}
|
||||
}, [filePoolFiles, filePoolUploadDisabled, loadAllKnowledgeDocuments, onNotify, uploadFilesToFilePool]);
|
||||
|
||||
const handleUploadSceneBackground = useCallback(async () => {
|
||||
if (!backgroundFile) return;
|
||||
try {
|
||||
const uploaded = await uploadSceneBackground({
|
||||
file: backgroundFile,
|
||||
name: backgroundName.trim() || backgroundFile.name,
|
||||
});
|
||||
const nextBackgrounds = [uploaded, ...sceneBackgrounds.filter((item) => item.id !== uploaded.id)];
|
||||
setSceneBackgrounds(nextBackgrounds);
|
||||
onSceneBackgroundsChange?.(nextBackgrounds);
|
||||
setBackgroundFile(null);
|
||||
setBackgroundName("");
|
||||
onNotify?.("背景资产已上传。", "success");
|
||||
} catch (err) {
|
||||
console.warn("upload scene background failed", err);
|
||||
const detail = err instanceof ApiError ? err.detail : null;
|
||||
onNotify?.(detail ? `上传失败:${detail}` : "上传失败,请稍后重试。", "error");
|
||||
}
|
||||
}, [backgroundFile, backgroundName, onNotify, onSceneBackgroundsChange, sceneBackgrounds]);
|
||||
|
||||
const handleCreateSceneComposition = useCallback(async () => {
|
||||
if (!sceneName.trim() || !sceneAvatarId) return;
|
||||
try {
|
||||
const created = await createSceneComposition({
|
||||
name: sceneName.trim(),
|
||||
avatar_id: sceneAvatarId,
|
||||
background_id: sceneBackgroundId || null,
|
||||
avatar_fit: "contain",
|
||||
avatar_scale: 1,
|
||||
avatar_anchor: "center",
|
||||
matting_required: true,
|
||||
subtitle_style: "lower-third",
|
||||
});
|
||||
const nextScenes = [created, ...sceneCompositions.filter((item) => item.id !== created.id)];
|
||||
setSceneCompositions(nextScenes);
|
||||
onSceneCompositionsChange?.(nextScenes);
|
||||
onSceneSelect?.(created);
|
||||
setSceneName("");
|
||||
onNotify?.("场景组合已创建。", "success");
|
||||
} catch (err) {
|
||||
console.warn("create scene composition failed", err);
|
||||
const detail = err instanceof ApiError ? err.detail : null;
|
||||
onNotify?.(detail ? `创建失败:${detail}` : "创建失败,请稍后重试。", "error");
|
||||
}
|
||||
}, [onNotify, onSceneCompositionsChange, onSceneSelect, sceneAvatarId, sceneBackgroundId, sceneCompositions, sceneName]);
|
||||
|
||||
const handleDeleteSceneBackground = useCallback(async (background: SceneBackgroundAsset) => {
|
||||
try {
|
||||
await deleteSceneBackground(background.id);
|
||||
await loadScenes();
|
||||
onNotify?.("背景资产已删除。", "success");
|
||||
} catch (err) {
|
||||
console.warn("delete scene background failed", err);
|
||||
const detail = err instanceof ApiError ? err.detail : null;
|
||||
onNotify?.(detail ? `删除失败:${detail}` : "删除失败,请稍后重试。", "error");
|
||||
}
|
||||
}, [loadScenes, onNotify]);
|
||||
|
||||
const handleDeleteSceneComposition = useCallback(async (scene: SceneComposition) => {
|
||||
try {
|
||||
await deleteSceneComposition(scene.id);
|
||||
await loadScenes();
|
||||
onNotify?.("场景组合已删除。", "success");
|
||||
} catch (err) {
|
||||
console.warn("delete scene composition failed", err);
|
||||
const detail = err instanceof ApiError ? err.detail : null;
|
||||
onNotify?.(detail ? `删除失败:${detail}` : "删除失败,请稍后重试。", "error");
|
||||
}
|
||||
}, [loadScenes, onNotify]);
|
||||
|
||||
const handleUploadKnowledgeDocuments = useCallback(async () => {
|
||||
if (!selectedKnowledgeId || uploadDisabled) return;
|
||||
setDocumentActionId("__upload__");
|
||||
@@ -911,6 +1039,195 @@ export function AssetLibraryWorkspace({
|
||||
/>
|
||||
);
|
||||
|
||||
const avatarById = useMemo(() => new Map((avatars ?? []).map((avatar) => [avatar.id, avatar])), [avatars]);
|
||||
const sceneGroups = useMemo(() => {
|
||||
const avatarGroups = (avatars ?? [])
|
||||
.map((avatar) => ({
|
||||
avatar,
|
||||
scenes: sceneCompositions.filter((scene) => scene.avatar_id === avatar.id),
|
||||
}))
|
||||
.filter((group) => group.scenes.length > 0);
|
||||
const knownAvatarIds = new Set((avatars ?? []).map((avatar) => avatar.id));
|
||||
const orphanScenes = sceneCompositions.filter((scene) => !knownAvatarIds.has(scene.avatar_id));
|
||||
if (orphanScenes.length > 0) {
|
||||
avatarGroups.push({
|
||||
avatar: { id: "__unknown__", name: "未识别形象" } as AvatarSummary,
|
||||
scenes: orphanScenes,
|
||||
});
|
||||
}
|
||||
return avatarGroups;
|
||||
}, [avatars, sceneCompositions]);
|
||||
|
||||
const renderScenesTab = () => (
|
||||
<div className="grid min-h-[24rem] gap-4 xl:grid-cols-[22rem_minmax(0,1fr)]">
|
||||
<aside className="rounded-lg border border-slate-200 bg-white p-4">
|
||||
<h2 className="text-sm font-semibold text-slate-950">背景资产</h2>
|
||||
<p className="mt-1 text-xs leading-relaxed text-slate-500">
|
||||
背景是资产库能力,可被工作台、沉浸模式和后续视频创作复用。
|
||||
</p>
|
||||
<div className="mt-4 space-y-3">
|
||||
<label className="block text-xs font-semibold text-slate-600" htmlFor="scene-background-name">
|
||||
背景名称
|
||||
</label>
|
||||
<input
|
||||
id="scene-background-name"
|
||||
value={backgroundName}
|
||||
onChange={(event) => setBackgroundName(event.target.value)}
|
||||
className="w-full rounded-lg border border-slate-200 bg-slate-50 px-3 py-2 text-sm outline-none focus:border-cyan-300 focus:bg-white"
|
||||
/>
|
||||
<input
|
||||
type="file"
|
||||
accept="image/png,image/jpeg,image/webp,video/mp4,video/webm,video/quicktime"
|
||||
onChange={(event) => setBackgroundFile(event.currentTarget.files?.[0] ?? null)}
|
||||
className="block w-full text-xs text-slate-600"
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
disabled={!backgroundFile}
|
||||
onClick={() => void handleUploadSceneBackground()}
|
||||
className="w-full rounded-lg bg-cyan-600 px-3 py-2 text-sm font-semibold text-white transition hover:bg-cyan-500 disabled:cursor-not-allowed disabled:opacity-50"
|
||||
>
|
||||
上传背景
|
||||
</button>
|
||||
</div>
|
||||
<div className="mt-5 space-y-2">
|
||||
{sceneLoading ? (
|
||||
<p className="text-sm text-slate-500">背景资产加载中...</p>
|
||||
) : sceneBackgrounds.length ? sceneBackgrounds.map((background) => (
|
||||
<div key={background.id} className="rounded-lg border border-slate-200 bg-slate-50 p-3">
|
||||
<p className="truncate text-sm font-semibold text-slate-900">{background.name}</p>
|
||||
<p className="mt-1 text-xs text-slate-500">{background.kind} · {formatSize(background.size_bytes)}</p>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => void handleDeleteSceneBackground(background)}
|
||||
className="mt-2 text-xs font-semibold text-rose-600 hover:text-rose-500"
|
||||
>
|
||||
删除
|
||||
</button>
|
||||
</div>
|
||||
)) : (
|
||||
<p className="rounded-lg border border-dashed border-slate-200 bg-slate-50 p-4 text-center text-sm text-slate-500">
|
||||
暂无背景资产
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</aside>
|
||||
<section className="rounded-lg border border-slate-200 bg-white p-4">
|
||||
<h2 className="text-sm font-semibold text-slate-950">场景组合</h2>
|
||||
<p className="mt-1 text-xs leading-relaxed text-slate-500">
|
||||
场景组合保存数字人、背景、构图和字幕样式,可在实时对话和沉浸模式中复用。
|
||||
</p>
|
||||
<div className="mt-4 grid gap-3 md:grid-cols-[minmax(0,1fr)_12rem_12rem_auto] md:items-end">
|
||||
<label className="block text-xs font-semibold text-slate-600">
|
||||
场景名称
|
||||
<input
|
||||
value={sceneName}
|
||||
onChange={(event) => setSceneName(event.target.value)}
|
||||
className="mt-1 w-full rounded-lg border border-slate-200 bg-slate-50 px-3 py-2 text-sm outline-none focus:border-cyan-300 focus:bg-white"
|
||||
/>
|
||||
</label>
|
||||
<label className="block text-xs font-semibold text-slate-600">
|
||||
数字人
|
||||
<select
|
||||
value={sceneAvatarId}
|
||||
onChange={(event) => setSceneAvatarId(event.target.value)}
|
||||
className="mt-1 w-full rounded-lg border border-slate-200 bg-white px-3 py-2 text-sm"
|
||||
>
|
||||
{(avatars ?? []).map((avatar) => (
|
||||
<option key={avatar.id} value={avatar.id}>{avatar.name ?? avatar.id}</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
<label className="block text-xs font-semibold text-slate-600">
|
||||
背景
|
||||
<select
|
||||
value={sceneBackgroundId}
|
||||
onChange={(event) => setSceneBackgroundId(event.target.value)}
|
||||
className="mt-1 w-full rounded-lg border border-slate-200 bg-white px-3 py-2 text-sm"
|
||||
>
|
||||
<option value="">纯色背景</option>
|
||||
{sceneBackgrounds.map((background) => (
|
||||
<option key={background.id} value={background.id}>{background.name}</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
<button
|
||||
type="button"
|
||||
disabled={!sceneName.trim() || !sceneAvatarId}
|
||||
onClick={() => void handleCreateSceneComposition()}
|
||||
className="rounded-lg bg-slate-950 px-4 py-2 text-sm font-semibold text-white transition hover:bg-slate-800 disabled:cursor-not-allowed disabled:opacity-50"
|
||||
>
|
||||
创建
|
||||
</button>
|
||||
</div>
|
||||
<div className="mt-5 space-y-4">
|
||||
{sceneGroups.length ? sceneGroups.map(({ avatar, scenes }) => (
|
||||
<section key={avatar.id} className="rounded-lg border border-slate-200 bg-slate-50 p-3">
|
||||
<div className="mb-3 flex flex-wrap items-center justify-between gap-2">
|
||||
<div>
|
||||
<p className="text-sm font-semibold text-slate-950">{avatar.name ?? avatar.id}</p>
|
||||
<p className="text-xs text-slate-500">{scenes.length} 个场景组合</p>
|
||||
</div>
|
||||
{selectedSceneIdsByAvatar[avatar.id] ? (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => onSceneClear?.(avatar.id)}
|
||||
className="text-xs font-semibold text-slate-500 hover:text-slate-800"
|
||||
>
|
||||
取消默认
|
||||
</button>
|
||||
) : null}
|
||||
</div>
|
||||
<div className="grid gap-3 md:grid-cols-2 xl:grid-cols-3">
|
||||
{scenes.map((scene) => {
|
||||
const selected = selectedSceneIdsByAvatar[scene.avatar_id] === scene.id;
|
||||
const sceneAvatar = avatarById.get(scene.avatar_id);
|
||||
return (
|
||||
<article
|
||||
key={scene.id}
|
||||
className={`rounded-lg border p-3 transition ${
|
||||
selected
|
||||
? "border-cyan-300 bg-cyan-50"
|
||||
: "border-slate-200 bg-white"
|
||||
}`}
|
||||
>
|
||||
<p className="truncate text-sm font-semibold text-slate-950">{scene.name}</p>
|
||||
<p className="mt-1 truncate text-xs text-slate-500">Avatar {sceneAvatar?.name ?? scene.avatar_id}</p>
|
||||
<p className="mt-1 truncate text-xs text-slate-500">Background {scene.background_id ?? scene.background_color}</p>
|
||||
<div className="mt-3 flex items-center gap-3">
|
||||
<button
|
||||
type="button"
|
||||
aria-pressed={selected}
|
||||
onClick={() => onSceneSelect?.(scene)}
|
||||
className={`text-xs font-semibold ${
|
||||
selected ? "text-cyan-800" : "text-cyan-700 hover:text-cyan-600"
|
||||
}`}
|
||||
>
|
||||
{selected ? "当前默认" : "设为默认"}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => void handleDeleteSceneComposition(scene)}
|
||||
className="text-xs font-semibold text-rose-600 hover:text-rose-500"
|
||||
>
|
||||
删除
|
||||
</button>
|
||||
</div>
|
||||
</article>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</section>
|
||||
)) : (
|
||||
<div className="rounded-lg border border-dashed border-slate-200 bg-slate-50 p-6 text-center text-sm text-slate-500">
|
||||
暂无场景组合
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
);
|
||||
|
||||
return (
|
||||
<main className="flex min-h-0 flex-1 flex-col bg-slate-100 p-4">
|
||||
<section className="flex min-h-0 flex-1 flex-col rounded-lg border border-slate-200 bg-white shadow-sm">
|
||||
@@ -930,6 +1247,11 @@ export function AssetLibraryWorkspace({
|
||||
<span>{memoryLibraries.length} 个记忆库</span>
|
||||
<span>{memoryLibraries.reduce((sum, library) => sum + library.memory_count, 0)} 条记忆</span>
|
||||
</>
|
||||
) : activeTab === "scenes" ? (
|
||||
<>
|
||||
<span>{sceneBackgrounds.length} 个背景</span>
|
||||
<span>{sceneCompositions.length} 个组合</span>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<span>{items.length} 个导出</span>
|
||||
@@ -939,10 +1261,10 @@ export function AssetLibraryWorkspace({
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleRefresh}
|
||||
disabled={activeTab === "knowledge" ? knowledgeLoading : loading}
|
||||
disabled={activeTab === "knowledge" ? knowledgeLoading : activeTab === "scenes" ? sceneLoading : loading}
|
||||
className="rounded-lg border border-slate-200 bg-white px-3 py-1.5 font-semibold text-slate-700 transition hover:border-cyan-200 hover:text-cyan-700 disabled:cursor-not-allowed disabled:opacity-50"
|
||||
>
|
||||
{(activeTab === "knowledge" ? knowledgeLoading : loading) ? "刷新中..." : "刷新"}
|
||||
{(activeTab === "knowledge" ? knowledgeLoading : activeTab === "scenes" ? sceneLoading : loading) ? "刷新中..." : "刷新"}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
@@ -1036,9 +1358,11 @@ export function AssetLibraryWorkspace({
|
||||
renderKnowledgeTab()
|
||||
) : activeTab === "memory" ? (
|
||||
renderMemoryTab()
|
||||
) : activeTab === "scenes" ? (
|
||||
renderScenesTab()
|
||||
) : (
|
||||
<div className="flex min-h-[18rem] items-center justify-center rounded-lg border border-dashed border-slate-300 bg-slate-50 text-sm font-medium text-slate-500">
|
||||
{activeTab === "avatars" ? "Avatar资产管理规划中" : "声音资产管理规划中"}
|
||||
声音资产管理规划中
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -271,7 +271,7 @@ export function AvatarSelectionStage({
|
||||
/>
|
||||
</div>
|
||||
<div className="px-3 py-2">
|
||||
<span className="truncate text-xs font-medium text-slate-500">
|
||||
<span className="block truncate text-xs font-medium text-slate-500">
|
||||
{avatar.is_custom ? "自定义形象" : "数字人形象"}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
114
apps/web/src/components/ImmersiveConversation.tsx
Normal file
114
apps/web/src/components/ImmersiveConversation.tsx
Normal file
@@ -0,0 +1,114 @@
|
||||
import { useEffect, type RefObject } from "react";
|
||||
import type { TtsProviderExtended } from "../constants/ttsBailian";
|
||||
import type { SceneBackgroundAsset, SceneComposition } from "../lib/api";
|
||||
import type { ConnectionStatus } from "../types";
|
||||
import { ChatInput } from "./ChatInput";
|
||||
import { SceneStage } from "./SceneStage";
|
||||
|
||||
type ImmersiveConversationProps = {
|
||||
videoRef: RefObject<HTMLVideoElement>;
|
||||
videoStream?: MediaStream | null;
|
||||
scene: SceneComposition | null;
|
||||
backgrounds: SceneBackgroundAsset[];
|
||||
connection: ConnectionStatus;
|
||||
sessionId: string | null;
|
||||
subtitle: string | null;
|
||||
isSpeaking: boolean;
|
||||
ttsProvider: TtsProviderExtended;
|
||||
sttProvider: string;
|
||||
edgeVoice: string;
|
||||
qwenModel: string;
|
||||
qwenVoice: string;
|
||||
onExit: () => void;
|
||||
onSend: (text: string) => void;
|
||||
onSpeakAudio: (blob: Blob) => void | Promise<void>;
|
||||
onSpeakFlashtalkAudioFile?: (file: File) => void | Promise<void>;
|
||||
onSpeakAudioStreamResult: (result: { text: string }) => void | Promise<void>;
|
||||
onSpeakAudioStreamError: (message: string) => void;
|
||||
onInterrupt: () => void;
|
||||
onNotify?: (message: string, tone?: "info" | "success" | "error") => void;
|
||||
};
|
||||
|
||||
export function ImmersiveConversation({
|
||||
videoRef,
|
||||
videoStream = null,
|
||||
scene,
|
||||
backgrounds,
|
||||
connection,
|
||||
sessionId,
|
||||
subtitle,
|
||||
isSpeaking,
|
||||
ttsProvider,
|
||||
sttProvider,
|
||||
edgeVoice,
|
||||
qwenModel,
|
||||
qwenVoice,
|
||||
onExit,
|
||||
onSend,
|
||||
onSpeakAudio,
|
||||
onSpeakFlashtalkAudioFile,
|
||||
onSpeakAudioStreamResult,
|
||||
onSpeakAudioStreamError,
|
||||
onInterrupt,
|
||||
onNotify,
|
||||
}: ImmersiveConversationProps) {
|
||||
useEffect(() => {
|
||||
const handleKeyDown = (event: KeyboardEvent) => {
|
||||
const target = event.target as HTMLElement | null;
|
||||
const editing = target
|
||||
? ["INPUT", "TEXTAREA", "SELECT"].includes(target.tagName) || target.isContentEditable
|
||||
: false;
|
||||
if (editing) return;
|
||||
if (event.key === "Escape") onExit();
|
||||
};
|
||||
window.addEventListener("keydown", handleKeyDown);
|
||||
return () => window.removeEventListener("keydown", handleKeyDown);
|
||||
}, [onExit]);
|
||||
|
||||
const live = connection === "live" || connection === "expiring";
|
||||
|
||||
return (
|
||||
<main className="relative h-dvh min-h-0 overflow-hidden bg-slate-950 text-white">
|
||||
<SceneStage
|
||||
videoRef={videoRef}
|
||||
videoStream={videoStream}
|
||||
scene={scene}
|
||||
backgrounds={backgrounds}
|
||||
subtitle={subtitle}
|
||||
className="h-full w-full"
|
||||
>
|
||||
<div className="absolute right-0 top-0 z-30 flex h-24 w-48 items-start justify-end p-4">
|
||||
<button
|
||||
type="button"
|
||||
onClick={onExit}
|
||||
className="rounded-lg border border-white/15 bg-slate-950/45 px-3 py-2 text-xs font-semibold text-white opacity-0 shadow-lg backdrop-blur transition hover:bg-slate-950/65 hover:opacity-100 focus:opacity-100"
|
||||
>
|
||||
返回工作台
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{live ? (
|
||||
<div className="absolute inset-x-3 bottom-3 z-30 mx-auto max-w-4xl sm:bottom-5">
|
||||
<ChatInput
|
||||
onSend={onSend}
|
||||
onSpeakAudio={onSpeakAudio}
|
||||
onSpeakFlashtalkAudioFile={onSpeakFlashtalkAudioFile}
|
||||
streamingAsrSessionId={sessionId}
|
||||
onSpeakAudioStreamResult={onSpeakAudioStreamResult}
|
||||
onSpeakAudioStreamError={onSpeakAudioStreamError}
|
||||
onInterrupt={onInterrupt}
|
||||
isSpeaking={isSpeaking}
|
||||
disabled={!live}
|
||||
onNotify={onNotify}
|
||||
ttsProvider={ttsProvider}
|
||||
sttProvider={sttProvider}
|
||||
edgeVoice={edgeVoice}
|
||||
qwenModel={qwenModel}
|
||||
qwenVoice={qwenVoice}
|
||||
/>
|
||||
</div>
|
||||
) : null}
|
||||
</SceneStage>
|
||||
</main>
|
||||
);
|
||||
}
|
||||
138
apps/web/src/components/SceneStage.tsx
Normal file
138
apps/web/src/components/SceneStage.tsx
Normal file
@@ -0,0 +1,138 @@
|
||||
import type { CSSProperties, ReactNode, RefObject } from "react";
|
||||
import type { SceneBackgroundAsset, SceneComposition } from "../lib/api";
|
||||
import { buildApiUrl } from "../lib/api";
|
||||
import { VideoBackground } from "./VideoBackground";
|
||||
|
||||
type SceneStageProps = {
|
||||
videoRef: RefObject<HTMLVideoElement>;
|
||||
videoStream?: MediaStream | null;
|
||||
scene: SceneComposition | null;
|
||||
backgrounds: SceneBackgroundAsset[];
|
||||
subtitle?: string | null;
|
||||
avatarMaskUrl?: string | null;
|
||||
avatarAdjust?: {
|
||||
x: number;
|
||||
y: number;
|
||||
scale: number;
|
||||
};
|
||||
children?: ReactNode;
|
||||
className?: string;
|
||||
compactSquareStage?: boolean;
|
||||
};
|
||||
|
||||
function backgroundUrl(background: SceneBackgroundAsset): string {
|
||||
return buildApiUrl(background.url);
|
||||
}
|
||||
|
||||
const AVATAR_ANCHOR_CLASSES = {
|
||||
center: "items-center justify-center",
|
||||
bottom: "items-end justify-center",
|
||||
left: "items-center justify-start",
|
||||
right: "items-center justify-end",
|
||||
} as const;
|
||||
|
||||
const AVATAR_ANCHOR_OBJECT_POSITIONS = {
|
||||
center: "object-center",
|
||||
bottom: "object-[center_bottom]",
|
||||
left: "object-[left_center]",
|
||||
right: "object-[right_center]",
|
||||
} as const;
|
||||
|
||||
const AVATAR_ANCHOR_TRANSFORM_ORIGINS = {
|
||||
center: "center",
|
||||
bottom: "center bottom",
|
||||
left: "left center",
|
||||
right: "right center",
|
||||
} as const;
|
||||
|
||||
export function SceneStage({
|
||||
videoRef,
|
||||
videoStream = null,
|
||||
scene,
|
||||
backgrounds,
|
||||
subtitle,
|
||||
avatarMaskUrl = null,
|
||||
avatarAdjust,
|
||||
children,
|
||||
className = "",
|
||||
compactSquareStage = false,
|
||||
}: SceneStageProps) {
|
||||
const background = scene?.background_id
|
||||
? backgrounds.find((item) => item.id === scene.background_id) ?? null
|
||||
: null;
|
||||
const subtitleStyle = scene?.subtitle_style ?? "lower-third";
|
||||
const avatarFit = scene?.avatar_fit === "cover" ? "object-cover" : "object-contain";
|
||||
const avatarAnchor = scene?.avatar_anchor ?? "center";
|
||||
const avatarAnchorClass = AVATAR_ANCHOR_CLASSES[avatarAnchor as keyof typeof AVATAR_ANCHOR_CLASSES] ?? AVATAR_ANCHOR_CLASSES.center;
|
||||
const avatarObjectPosition = AVATAR_ANCHOR_OBJECT_POSITIONS[avatarAnchor as keyof typeof AVATAR_ANCHOR_OBJECT_POSITIONS] ?? AVATAR_ANCHOR_OBJECT_POSITIONS.center;
|
||||
const avatarTransformOrigin = AVATAR_ANCHOR_TRANSFORM_ORIGINS[avatarAnchor as keyof typeof AVATAR_ANCHOR_TRANSFORM_ORIGINS] ?? AVATAR_ANCHOR_TRANSFORM_ORIGINS.center;
|
||||
const avatarMaskSize = scene?.avatar_fit === "cover" ? "cover" : "contain";
|
||||
const avatarMaskPosition = avatarAnchor === "bottom"
|
||||
? "center bottom"
|
||||
: avatarAnchor === "left"
|
||||
? "left center"
|
||||
: avatarAnchor === "right"
|
||||
? "right center"
|
||||
: "center";
|
||||
const hasSceneBackground = Boolean(scene);
|
||||
const backgroundColor = scene?.background_color || "#ffffff";
|
||||
const sceneAvatarScale = scene?.avatar_scale ?? 1;
|
||||
const avatarDisplayScale = sceneAvatarScale * (avatarAdjust?.scale ?? 1);
|
||||
const avatarTransform = avatarAdjust
|
||||
? `translate(${avatarAdjust.x}px, ${avatarAdjust.y}px) scale(${avatarDisplayScale})`
|
||||
: `scale(${sceneAvatarScale})`;
|
||||
const avatarMaskStyle: CSSProperties | undefined = avatarMaskUrl
|
||||
? {
|
||||
WebkitMaskImage: `url("${avatarMaskUrl}")`,
|
||||
WebkitMaskRepeat: "no-repeat",
|
||||
WebkitMaskSize: avatarMaskSize,
|
||||
WebkitMaskPosition: avatarMaskPosition,
|
||||
maskImage: `url("${avatarMaskUrl}")`,
|
||||
maskMode: "alpha",
|
||||
maskRepeat: "no-repeat",
|
||||
maskSize: avatarMaskSize,
|
||||
maskPosition: avatarMaskPosition,
|
||||
}
|
||||
: undefined;
|
||||
|
||||
return (
|
||||
<div className={`relative min-h-0 overflow-hidden ${hasSceneBackground ? "bg-slate-950" : "bg-white"} ${className}`}>
|
||||
<div className="scene-background-layer absolute inset-0" style={{ backgroundColor }}>
|
||||
{background?.kind === "image" ? (
|
||||
<img src={backgroundUrl(background)} alt={background.name} className="h-full w-full object-cover" />
|
||||
) : null}
|
||||
{background?.kind === "video" ? (
|
||||
<video src={backgroundUrl(background)} className="h-full w-full object-cover" autoPlay muted loop playsInline />
|
||||
) : null}
|
||||
{hasSceneBackground ? <div className="absolute inset-0 bg-slate-950/10" /> : null}
|
||||
</div>
|
||||
|
||||
<div className={`absolute inset-0 flex p-4 sm:p-6 lg:p-8 ${avatarAnchorClass}`}>
|
||||
<div
|
||||
className={
|
||||
compactSquareStage
|
||||
? "relative aspect-square w-full max-w-[42rem] max-h-full"
|
||||
: "relative h-full w-full"
|
||||
}
|
||||
style={{ transform: avatarTransform, transformOrigin: avatarTransformOrigin }}
|
||||
>
|
||||
<VideoBackground ref={videoRef} stream={videoStream} className={`absolute inset-0 h-full w-full ${avatarFit} ${avatarObjectPosition}`} style={avatarMaskStyle} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{subtitle && subtitleStyle !== "none" ? (
|
||||
<div
|
||||
className={
|
||||
subtitleStyle === "compact"
|
||||
? "absolute inset-x-4 bottom-4 z-20 mx-auto max-w-lg rounded-lg bg-slate-950/72 px-4 py-2 text-center text-sm font-medium leading-relaxed text-white shadow-lg backdrop-blur"
|
||||
: "absolute inset-x-4 bottom-6 z-20 mx-auto max-w-2xl rounded-lg border border-white/15 bg-slate-950/75 px-5 py-3 text-center text-base font-semibold leading-relaxed text-white shadow-xl backdrop-blur"
|
||||
}
|
||||
>
|
||||
{subtitle}
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{children}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -29,10 +29,13 @@ const DOT_LABELS: Record<ConnectionStatus, string> = {
|
||||
|
||||
export type FlashtalkRecordPhase = "idle" | "recording" | "stopped";
|
||||
export type StudioWorkflow = "realtime" | "videoCreation" | "videoClone" | "assetLibrary" | "runtimeConfig";
|
||||
export type ConversationViewMode = "studio" | "immersive";
|
||||
|
||||
interface TopBarProps {
|
||||
connection: ConnectionStatus;
|
||||
workflow?: StudioWorkflow;
|
||||
conversationViewMode?: ConversationViewMode;
|
||||
immersiveChrome?: boolean;
|
||||
flashtalkRecording?: boolean;
|
||||
flashtalkRecordPhase?: FlashtalkRecordPhase;
|
||||
flashtalkRecordBusy?: boolean;
|
||||
@@ -40,6 +43,7 @@ interface TopBarProps {
|
||||
runtimeConfigReady?: boolean;
|
||||
runtimeConfigLoading?: boolean;
|
||||
onInactiveModuleClick?: (label: string) => void;
|
||||
onConversationViewModeChange?: (mode: ConversationViewMode) => void;
|
||||
onFlashtalkRecordStart?: () => void;
|
||||
onFlashtalkRecordStop?: () => void;
|
||||
onFlashtalkRecordSave?: () => void;
|
||||
@@ -49,6 +53,8 @@ interface TopBarProps {
|
||||
export function TopBar({
|
||||
connection,
|
||||
workflow = "realtime",
|
||||
conversationViewMode = "studio",
|
||||
immersiveChrome = false,
|
||||
flashtalkRecording = false,
|
||||
flashtalkRecordPhase = "idle",
|
||||
flashtalkRecordBusy = false,
|
||||
@@ -56,6 +62,7 @@ export function TopBar({
|
||||
runtimeConfigReady = false,
|
||||
runtimeConfigLoading = false,
|
||||
onInactiveModuleClick,
|
||||
onConversationViewModeChange,
|
||||
onFlashtalkRecordStart,
|
||||
onFlashtalkRecordStop,
|
||||
onFlashtalkRecordSave,
|
||||
@@ -64,7 +71,18 @@ export function TopBar({
|
||||
const busy = flashtalkRecordBusy || recordingSaving;
|
||||
|
||||
return (
|
||||
<header className="flex h-14 shrink-0 items-center justify-between border-b border-slate-200 bg-white px-4 shadow-sm">
|
||||
<header
|
||||
className={
|
||||
immersiveChrome
|
||||
? "group fixed inset-x-0 top-0 z-50 flex h-14 -translate-y-12 items-center justify-between border-b border-white/10 bg-white/92 px-4 shadow-lg shadow-slate-950/10 backdrop-blur transition-transform duration-200 hover:translate-y-0"
|
||||
: "flex h-14 shrink-0 items-center justify-between border-b border-slate-200 bg-white px-4 shadow-sm"
|
||||
}
|
||||
>
|
||||
{immersiveChrome ? (
|
||||
<div className="pointer-events-none absolute inset-x-0 bottom-[-0.65rem] flex justify-center opacity-100 transition-opacity duration-150 group-hover:opacity-0 group-focus-within:opacity-0">
|
||||
<div className="h-1.5 w-16 rounded-full bg-white/80 shadow-sm" />
|
||||
</div>
|
||||
) : null}
|
||||
<div className="flex min-w-0 items-center gap-3">
|
||||
<div className="flex h-9 w-9 shrink-0 items-center justify-center rounded-lg bg-slate-950 text-cyan-300">
|
||||
<svg viewBox="0 0 24 24" className="h-4 w-4" fill="currentColor" aria-hidden>
|
||||
@@ -80,7 +98,14 @@ export function TopBar({
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<nav className="hidden items-center gap-1 rounded-lg bg-slate-100 p-1 md:flex" aria-label="工作台模块">
|
||||
<nav
|
||||
className={
|
||||
immersiveChrome
|
||||
? "hidden"
|
||||
: "hidden items-center gap-1 rounded-lg bg-slate-100 p-1 md:flex"
|
||||
}
|
||||
aria-label="工作台模块"
|
||||
>
|
||||
{[
|
||||
["realtime", "实时对话"],
|
||||
["videoCreation", "视频创作"],
|
||||
@@ -114,6 +139,30 @@ export function TopBar({
|
||||
</nav>
|
||||
|
||||
<div className="flex min-w-0 flex-wrap items-center justify-end gap-1.5 sm:gap-2">
|
||||
{workflow === "realtime" ? (
|
||||
<div className="hidden rounded-lg bg-slate-100 p-1 sm:flex" aria-label="实时对话视图">
|
||||
{[
|
||||
["studio", "工作台"],
|
||||
["immersive", "沉浸"],
|
||||
].map(([id, label]) => (
|
||||
<button
|
||||
key={id}
|
||||
type="button"
|
||||
onClick={(event) => {
|
||||
event.currentTarget.blur();
|
||||
onConversationViewModeChange?.(id as ConversationViewMode);
|
||||
}}
|
||||
className={`rounded-md px-2.5 py-1.5 text-xs font-semibold transition ${
|
||||
conversationViewMode === id
|
||||
? "bg-white text-cyan-700 shadow-sm"
|
||||
: "text-slate-500 hover:bg-white/70 hover:text-slate-700"
|
||||
}`}
|
||||
>
|
||||
{label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
) : null}
|
||||
{flashtalkRecording ? (
|
||||
<div className="flex flex-wrap items-center justify-end gap-1.5">
|
||||
{flashtalkRecordPhase === "idle" ? (
|
||||
|
||||
@@ -1,19 +1,50 @@
|
||||
import { forwardRef } from "react";
|
||||
import type { CSSProperties } from "react";
|
||||
import { forwardRef, useEffect, useImperativeHandle, useRef } from "react";
|
||||
|
||||
type VideoBackgroundProps = {
|
||||
className?: string;
|
||||
style?: CSSProperties;
|
||||
stream?: MediaStream | null;
|
||||
};
|
||||
|
||||
export function playWithMutedFallback(video: HTMLVideoElement) {
|
||||
void video.play().catch(() => {
|
||||
video.muted = true;
|
||||
void video.play().catch(() => {});
|
||||
});
|
||||
}
|
||||
|
||||
export const VideoBackground = forwardRef<HTMLVideoElement, VideoBackgroundProps>(
|
||||
({ className }, ref) => (
|
||||
<video
|
||||
ref={ref}
|
||||
className={className ?? "absolute inset-0 h-full w-full object-contain"}
|
||||
autoPlay
|
||||
playsInline
|
||||
muted
|
||||
/>
|
||||
),
|
||||
({ className, style, stream }, ref) => {
|
||||
const videoRef = useRef<HTMLVideoElement>(null);
|
||||
useImperativeHandle(ref, () => videoRef.current as HTMLVideoElement);
|
||||
|
||||
useEffect(() => {
|
||||
const video = videoRef.current;
|
||||
if (!video) return;
|
||||
if (video.srcObject !== stream) {
|
||||
video.srcObject = stream ?? null;
|
||||
}
|
||||
if (stream) {
|
||||
video.muted = false;
|
||||
video.volume = 1;
|
||||
playWithMutedFallback(video);
|
||||
} else {
|
||||
video.muted = true;
|
||||
}
|
||||
}, [stream]);
|
||||
|
||||
return (
|
||||
<video
|
||||
ref={videoRef}
|
||||
className={className ?? "absolute inset-0 h-full w-full object-contain"}
|
||||
style={style}
|
||||
autoPlay
|
||||
playsInline
|
||||
muted={!stream}
|
||||
/>
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
VideoBackground.displayName = "VideoBackground";
|
||||
|
||||
@@ -242,6 +242,71 @@ export async function uploadExportVideo(input: UploadExportVideoInput): Promise<
|
||||
return apiPostForm<ExportVideoItem>("/exports/videos", form);
|
||||
}
|
||||
|
||||
export type SceneBackgroundAsset = {
|
||||
id: string;
|
||||
name: string;
|
||||
kind: "image" | "video";
|
||||
mime_type: string;
|
||||
filename: string;
|
||||
size_bytes: number;
|
||||
url: string;
|
||||
created_at: string;
|
||||
};
|
||||
|
||||
export type SceneComposition = {
|
||||
id: string;
|
||||
name: string;
|
||||
avatar_id: string;
|
||||
background_id: string | null;
|
||||
background_color: string;
|
||||
avatar_fit: "contain" | "cover";
|
||||
avatar_scale: number;
|
||||
avatar_anchor: "center" | "bottom" | "left" | "right";
|
||||
matting_required: boolean;
|
||||
subtitle_style: "none" | "compact" | "lower-third";
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
};
|
||||
|
||||
export type CreateSceneCompositionInput = {
|
||||
name: string;
|
||||
avatar_id: string;
|
||||
background_id?: string | null;
|
||||
background_color?: string;
|
||||
avatar_fit?: "contain" | "cover";
|
||||
avatar_scale?: number;
|
||||
avatar_anchor?: "center" | "bottom" | "left" | "right";
|
||||
matting_required?: boolean;
|
||||
subtitle_style?: "none" | "compact" | "lower-third";
|
||||
};
|
||||
|
||||
export async function listSceneBackgrounds(): Promise<{ items: SceneBackgroundAsset[] }> {
|
||||
return apiGet<{ items: SceneBackgroundAsset[] }>("/scene-assets/backgrounds");
|
||||
}
|
||||
|
||||
export async function uploadSceneBackground(input: { file: File; name: string }): Promise<SceneBackgroundAsset> {
|
||||
const form = new FormData();
|
||||
form.set("file", input.file);
|
||||
form.set("name", input.name);
|
||||
return apiPostForm<SceneBackgroundAsset>("/scene-assets/backgrounds", form);
|
||||
}
|
||||
|
||||
export async function deleteSceneBackground(backgroundId: string): Promise<{ id: string; deleted: boolean }> {
|
||||
return apiDelete<{ id: string; deleted: boolean }>(`/scene-assets/backgrounds/${encodeURIComponent(backgroundId)}`);
|
||||
}
|
||||
|
||||
export async function listSceneCompositions(): Promise<{ items: SceneComposition[] }> {
|
||||
return apiGet<{ items: SceneComposition[] }>("/scene-assets/compositions");
|
||||
}
|
||||
|
||||
export async function createSceneComposition(input: CreateSceneCompositionInput): Promise<SceneComposition> {
|
||||
return apiPost<SceneComposition>("/scene-assets/compositions", input);
|
||||
}
|
||||
|
||||
export async function deleteSceneComposition(compositionId: string): Promise<{ id: string; deleted: boolean }> {
|
||||
return apiDelete<{ id: string; deleted: boolean }>(`/scene-assets/compositions/${encodeURIComponent(compositionId)}`);
|
||||
}
|
||||
|
||||
|
||||
export type VideoCreationAudioSource = "upload" | "tts_text" | "voice_clone" | "reference_video";
|
||||
|
||||
@@ -441,6 +506,7 @@ export type AvatarSummary = {
|
||||
height: number;
|
||||
is_custom: boolean;
|
||||
has_preview_video: boolean;
|
||||
matting_status: "unknown" | "opaque" | "transparent_ready";
|
||||
};
|
||||
|
||||
export type CreateSessionResponse = { session_id: string; status: string };
|
||||
|
||||
20
examples/avatars/female-host-transparent/manifest.json
Normal file
20
examples/avatars/female-host-transparent/manifest.json
Normal file
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"id": "female-host-transparent",
|
||||
"name": "女主持(透明背景)",
|
||||
"model_type": "wav2lip",
|
||||
"fps": 25,
|
||||
"sample_rate": 16000,
|
||||
"width": 1024,
|
||||
"height": 1024,
|
||||
"version": "1.0",
|
||||
"metadata": {
|
||||
"description": "Built-in female host avatar asset with a transparent background.",
|
||||
"idle_mode": "static",
|
||||
"reference_mode": "image",
|
||||
"frame_dir": "frames",
|
||||
"source_image": "source/source.png",
|
||||
"source_image_path": "reference.png",
|
||||
"source_image_hash": "712c1cd47f32a18042a2908893f81128ba5176be6f896c8e31fb6ae4e5321f9b",
|
||||
"matting_status": "transparent_ready"
|
||||
}
|
||||
}
|
||||
BIN
examples/avatars/female-host-transparent/preview.png
Normal file
BIN
examples/avatars/female-host-transparent/preview.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 1.4 MiB |
BIN
examples/avatars/female-host-transparent/reference.png
Normal file
BIN
examples/avatars/female-host-transparent/reference.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 1.4 MiB |
BIN
examples/avatars/female-host-transparent/source/source.png
Normal file
BIN
examples/avatars/female-host-transparent/source/source.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 1.4 MiB |
@@ -328,6 +328,8 @@ class Settings(BaseSettings):
|
||||
models_dir: str = "./models"
|
||||
worker_url: str = "http://127.0.0.1:9001"
|
||||
exports_dir: str = "./data/exports"
|
||||
scene_assets_dir: str = "./data/scene-assets"
|
||||
scene_asset_max_bytes: int = 200 * 1024 * 1024
|
||||
export_max_bytes: int = 1024 * 1024 * 1024
|
||||
video_creation_audio_max_bytes: int = 50 * 1024 * 1024
|
||||
video_creation_fasterliveportrait_preroll_ms: int = 400
|
||||
|
||||
213
opentalking/scene_assets.py
Normal file
213
opentalking/scene_assets.py
Normal file
@@ -0,0 +1,213 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
SUPPORTED_IMAGE_TYPES = {"image/png", "image/jpeg", "image/webp"}
|
||||
SUPPORTED_VIDEO_TYPES = {"video/mp4", "video/webm", "video/quicktime"}
|
||||
SUPPORTED_BACKGROUND_TYPES = SUPPORTED_IMAGE_TYPES | SUPPORTED_VIDEO_TYPES
|
||||
EXT_BY_MIME = {
|
||||
"image/png": ".png",
|
||||
"image/jpeg": ".jpg",
|
||||
"image/webp": ".webp",
|
||||
"video/mp4": ".mp4",
|
||||
"video/webm": ".webm",
|
||||
"video/quicktime": ".mov",
|
||||
}
|
||||
VALID_AVATAR_FITS = {"contain", "cover"}
|
||||
VALID_AVATAR_ANCHORS = {"center", "bottom", "left", "right"}
|
||||
VALID_SUBTITLE_STYLES = {"none", "compact", "lower-third"}
|
||||
|
||||
|
||||
def sniff_background_mime(content: bytes) -> str | None:
|
||||
if content.startswith(b"\x89PNG\r\n\x1a\n"):
|
||||
return "image/png"
|
||||
if content.startswith(b"\xff\xd8\xff"):
|
||||
return "image/jpeg"
|
||||
if len(content) >= 12 and content[:4] == b"RIFF" and content[8:12] == b"WEBP":
|
||||
return "image/webp"
|
||||
if len(content) >= 12 and content[4:8] == b"ftyp":
|
||||
brand = content[8:12]
|
||||
if brand in {b"qt "}:
|
||||
return "video/quicktime"
|
||||
if brand in {b"mp41", b"mp42", b"isom", b"iso2", b"avc1", b"M4V "}:
|
||||
return "video/mp4"
|
||||
return "video/mp4"
|
||||
if content.startswith(b"\x1a\x45\xdf\xa3"):
|
||||
return "video/webm"
|
||||
return None
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _slug(value: str, fallback: str) -> str:
|
||||
normalized = re.sub(r"[^a-zA-Z0-9\u4e00-\u9fff_-]+", "-", value.strip()).strip("-_")
|
||||
return normalized[:48] or fallback
|
||||
|
||||
|
||||
def _read_json(path: Path, fallback: Any) -> Any:
|
||||
if not path.is_file():
|
||||
return fallback
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return fallback
|
||||
|
||||
|
||||
def _write_json(path: Path, payload: Any) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
class SceneAssetStore:
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = root.expanduser().resolve()
|
||||
self.backgrounds_dir = self.root / "backgrounds"
|
||||
self.compositions_dir = self.root / "compositions"
|
||||
self.background_index_path = self.backgrounds_dir / "index.json"
|
||||
self.composition_index_path = self.compositions_dir / "index.json"
|
||||
|
||||
def list_backgrounds(self) -> list[dict[str, object]]:
|
||||
items = _read_json(self.background_index_path, [])
|
||||
return [item for item in items if isinstance(item, dict)]
|
||||
|
||||
def create_background(self, *, content: bytes, filename: str, mime_type: str, name: str) -> dict[str, object]:
|
||||
normalized_mime = (mime_type or "").split(";")[0].strip().lower()
|
||||
if not content:
|
||||
raise ValueError("empty background asset")
|
||||
sniffed_mime = sniff_background_mime(content)
|
||||
if sniffed_mime not in SUPPORTED_BACKGROUND_TYPES:
|
||||
raise ValueError("unsupported background media type")
|
||||
normalized_mime = sniffed_mime
|
||||
ext = EXT_BY_MIME[normalized_mime]
|
||||
background_id = f"bg-{_slug(name or Path(filename).stem, 'background')}-{uuid.uuid4().hex[:10]}"
|
||||
media_path = self.backgrounds_dir / background_id / f"source{ext}"
|
||||
media_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
media_path.write_bytes(content)
|
||||
item: dict[str, object] = {
|
||||
"id": background_id,
|
||||
"name": (name or Path(filename).stem or background_id).strip(),
|
||||
"kind": "video" if normalized_mime.startswith("video/") else "image",
|
||||
"mime_type": normalized_mime,
|
||||
"filename": filename or f"source{ext}",
|
||||
"size_bytes": len(content),
|
||||
"url": f"/scene-assets/backgrounds/{background_id}/file",
|
||||
"created_at": _now(),
|
||||
}
|
||||
items = [entry for entry in self.list_backgrounds() if entry.get("id") != background_id]
|
||||
items.insert(0, item)
|
||||
_write_json(self.background_index_path, items)
|
||||
return item
|
||||
|
||||
def background_file_path(self, background_id: str) -> Path | None:
|
||||
if not re.fullmatch(r"bg-[\w\u4e00-\u9fff-]+", background_id or ""):
|
||||
return None
|
||||
item = next((entry for entry in self.list_backgrounds() if entry.get("id") == background_id), None)
|
||||
if not item:
|
||||
return None
|
||||
ext = EXT_BY_MIME.get(str(item.get("mime_type") or ""))
|
||||
if not ext:
|
||||
return None
|
||||
path = (self.backgrounds_dir / background_id / f"source{ext}").resolve()
|
||||
try:
|
||||
path.relative_to(self.backgrounds_dir.resolve())
|
||||
except ValueError:
|
||||
return None
|
||||
return path if path.is_file() else None
|
||||
|
||||
def delete_background(self, background_id: str) -> bool:
|
||||
if not re.fullmatch(r"bg-[\w\u4e00-\u9fff-]+", background_id or ""):
|
||||
return False
|
||||
items = self.list_backgrounds()
|
||||
next_items = [item for item in items if item.get("id") != background_id]
|
||||
if len(next_items) == len(items):
|
||||
return False
|
||||
_write_json(self.background_index_path, next_items)
|
||||
shutil.rmtree(self.backgrounds_dir / background_id, ignore_errors=True)
|
||||
compositions = [
|
||||
{**item, "background_id": None}
|
||||
if item.get("background_id") == background_id else item
|
||||
for item in self.list_compositions()
|
||||
]
|
||||
_write_json(self.composition_index_path, compositions)
|
||||
return True
|
||||
|
||||
def list_compositions(self) -> list[dict[str, object]]:
|
||||
items = _read_json(self.composition_index_path, [])
|
||||
return [item for item in items if isinstance(item, dict)]
|
||||
|
||||
def create_composition(self, payload: dict[str, object]) -> dict[str, object]:
|
||||
composition_id = f"scene-{_slug(str(payload.get('name') or 'scene'), 'scene')}-{uuid.uuid4().hex[:10]}"
|
||||
now = _now()
|
||||
item = self._normalize_composition({**payload, "id": composition_id, "created_at": now, "updated_at": now})
|
||||
items = [entry for entry in self.list_compositions() if entry.get("id") != composition_id]
|
||||
items.insert(0, item)
|
||||
_write_json(self.composition_index_path, items)
|
||||
return item
|
||||
|
||||
def update_composition(self, composition_id: str, payload: dict[str, object]) -> dict[str, object] | None:
|
||||
items = self.list_compositions()
|
||||
for index, item in enumerate(items):
|
||||
if item.get("id") != composition_id:
|
||||
continue
|
||||
updated = self._normalize_composition({**item, **payload, "id": composition_id, "updated_at": _now()})
|
||||
items[index] = updated
|
||||
_write_json(self.composition_index_path, items)
|
||||
return updated
|
||||
return None
|
||||
|
||||
def delete_composition(self, composition_id: str) -> bool:
|
||||
items = self.list_compositions()
|
||||
next_items = [item for item in items if item.get("id") != composition_id]
|
||||
if len(next_items) == len(items):
|
||||
return False
|
||||
_write_json(self.composition_index_path, next_items)
|
||||
return True
|
||||
|
||||
def _normalize_composition(self, payload: dict[str, object]) -> dict[str, object]:
|
||||
avatar_id = str(payload.get("avatar_id") or "").strip()
|
||||
if not avatar_id:
|
||||
raise ValueError("avatar_id is required")
|
||||
background_id = str(payload.get("background_id") or "").strip() or None
|
||||
if background_id and not any(item.get("id") == background_id for item in self.list_backgrounds()):
|
||||
raise ValueError("background_id not found")
|
||||
avatar_fit = str(payload.get("avatar_fit") or "contain").strip()
|
||||
avatar_anchor = str(payload.get("avatar_anchor") or "center").strip()
|
||||
subtitle_style = str(payload.get("subtitle_style") or "lower-third").strip()
|
||||
raw_avatar_scale = payload.get("avatar_scale")
|
||||
if raw_avatar_scale is None:
|
||||
avatar_scale = 1.0
|
||||
elif isinstance(raw_avatar_scale, str | int | float):
|
||||
avatar_scale = float(raw_avatar_scale)
|
||||
else:
|
||||
raise ValueError("avatar_scale must be a number")
|
||||
if avatar_fit not in VALID_AVATAR_FITS:
|
||||
raise ValueError("invalid avatar_fit")
|
||||
if avatar_anchor not in VALID_AVATAR_ANCHORS:
|
||||
raise ValueError("invalid avatar_anchor")
|
||||
if subtitle_style not in VALID_SUBTITLE_STYLES:
|
||||
raise ValueError("invalid subtitle_style")
|
||||
if not 0.5 <= avatar_scale <= 2.0:
|
||||
raise ValueError("avatar_scale must be between 0.5 and 2.0")
|
||||
return {
|
||||
"id": str(payload["id"]),
|
||||
"name": str(payload.get("name") or payload["id"]).strip(),
|
||||
"avatar_id": avatar_id,
|
||||
"background_id": background_id,
|
||||
"background_color": str(payload.get("background_color") or "#0f172a").strip(),
|
||||
"avatar_fit": avatar_fit,
|
||||
"avatar_scale": avatar_scale,
|
||||
"avatar_anchor": avatar_anchor,
|
||||
"matting_required": bool(payload.get("matting_required", False)),
|
||||
"subtitle_style": subtitle_style,
|
||||
"created_at": str(payload.get("created_at") or _now()),
|
||||
"updated_at": str(payload.get("updated_at") or _now()),
|
||||
}
|
||||
@@ -6,7 +6,7 @@ from pathlib import Path
|
||||
def test_asset_library_exposes_knowledge_base_tab_and_modal() -> None:
|
||||
source = Path("apps/web/src/components/AssetLibraryWorkspace.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert 'type AssetTab = "exports" | "knowledge" | "memory" | "avatars" | "voices"' in source
|
||||
assert 'type AssetTab = "exports" | "knowledge" | "memory" | "scenes" | "voices"' in source
|
||||
assert "知识库" in source
|
||||
assert "新建知识库" in source
|
||||
assert "从本地中间文件导入" in source
|
||||
|
||||
190
tests/frontend/test_immersive_conversation_mode.py
Normal file
190
tests/frontend/test_immersive_conversation_mode.py
Normal file
@@ -0,0 +1,190 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def test_scene_stage_component_exists_and_handles_background_layers() -> None:
|
||||
source = Path("apps/web/src/components/SceneStage.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert "export function SceneStage" in source
|
||||
assert "SceneBackgroundAsset" in source
|
||||
assert "SceneComposition" in source
|
||||
assert "scene-background-layer" in source
|
||||
assert "VideoBackground" in source
|
||||
assert "subtitle_style" in source
|
||||
|
||||
|
||||
def test_scene_stage_applies_avatar_anchor_positioning() -> None:
|
||||
source = Path("apps/web/src/components/SceneStage.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert "scene?.avatar_anchor" in source
|
||||
assert "AVATAR_ANCHOR_OBJECT_POSITIONS" in source
|
||||
assert 'bottom: "object-[center_bottom]"' in source
|
||||
assert 'left: "object-[left_center]"' in source
|
||||
assert 'right: "object-[right_center]"' in source
|
||||
assert "${avatarFit} ${avatarObjectPosition}" in source
|
||||
assert "AVATAR_ANCHOR_TRANSFORM_ORIGINS" in source
|
||||
assert "transformOrigin: avatarTransformOrigin" in source
|
||||
assert 'bottom: "items-end justify-center"' in source
|
||||
assert 'left: "items-center justify-start"' in source
|
||||
assert 'right: "items-center justify-end"' in source
|
||||
|
||||
|
||||
def test_scene_stage_masks_transparent_avatar_video_with_preview_alpha() -> None:
|
||||
source = Path("apps/web/src/components/SceneStage.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert "avatarMaskUrl?: string | null;" in source
|
||||
assert "WebkitMaskImage" in source
|
||||
assert "maskImage" in source
|
||||
assert "maskMode" in source
|
||||
assert "avatarMaskStyle" in source
|
||||
|
||||
|
||||
def test_scene_stage_accepts_immersive_avatar_adjustments() -> None:
|
||||
source = Path("apps/web/src/components/SceneStage.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert "avatarAdjust?: {" in source
|
||||
assert "x: number;" in source
|
||||
assert "y: number;" in source
|
||||
assert "scale: number;" in source
|
||||
assert "translate(${avatarAdjust.x}px, ${avatarAdjust.y}px) scale" in source
|
||||
|
||||
|
||||
def test_app_uses_scene_stage_for_realtime_stage() -> None:
|
||||
source = Path("apps/web/src/App.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert 'import { SceneStage } from "./components/SceneStage";' in source
|
||||
assert "selectedScene" in source
|
||||
assert "sceneBackgrounds" in source
|
||||
assert "<SceneStage" in source
|
||||
assert "selectedAvatarMaskUrl" in source
|
||||
assert "avatarMaskUrl={showStart ? null : selectedAvatarMaskUrl}" in source
|
||||
|
||||
|
||||
def test_app_keeps_webrtc_remote_stream_identity_for_async_tracks() -> None:
|
||||
source = Path("apps/web/src/App.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert "cloneMediaStream" not in source
|
||||
assert "setRemoteStream(remoteStream)" in source
|
||||
assert "setRemoteStream(playback.remoteStream)" in source
|
||||
|
||||
|
||||
def test_video_background_falls_back_to_muted_playback_when_autoplay_blocks_audio() -> None:
|
||||
source = Path("apps/web/src/components/VideoBackground.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert "export function playWithMutedFallback" in source
|
||||
assert "video.muted = true" in source
|
||||
assert "video.play().catch" in source
|
||||
|
||||
|
||||
def test_app_replays_remote_video_with_muted_fallback_when_view_changes() -> None:
|
||||
source = Path("apps/web/src/App.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert 'import { playWithMutedFallback } from "./components/VideoBackground";' in source
|
||||
effect_start = source.index("const video = videoRef.current;")
|
||||
effect_end = source.index("}, [conversationViewMode, remoteStream, workflow]);", effect_start)
|
||||
playback_effect = source[effect_start:effect_end]
|
||||
assert "playWithMutedFallback(video)" in playback_effect
|
||||
|
||||
|
||||
def test_immersive_conversation_component_focuses_stage_and_input() -> None:
|
||||
source = Path("apps/web/src/components/ImmersiveConversation.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert "export function ImmersiveConversation" in source
|
||||
assert "SceneStage" in source
|
||||
assert "ChatInput" in source
|
||||
assert "onExit" in source
|
||||
assert 'event.key === "Escape"' in source
|
||||
assert "bottom-3" in source
|
||||
assert "返回工作台" in source
|
||||
|
||||
|
||||
def test_topbar_exposes_realtime_view_mode_toggle() -> None:
|
||||
source = Path("apps/web/src/components/TopBar.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert 'export type ConversationViewMode = "studio" | "immersive";' in source
|
||||
assert "onConversationViewModeChange" in source
|
||||
assert "沉浸" in source
|
||||
|
||||
|
||||
def test_topbar_hides_main_module_navigation_in_immersive_chrome() -> None:
|
||||
source = Path("apps/web/src/components/TopBar.tsx").read_text(encoding="utf-8")
|
||||
|
||||
nav_label = source.index('aria-label="工作台模块"')
|
||||
nav_start = source.rindex("<nav", 0, nav_label)
|
||||
nav_end = source.index("</nav>", nav_label)
|
||||
nav_block = source[nav_start:nav_end]
|
||||
assert "immersiveChrome" in nav_block
|
||||
assert '? "hidden"' in nav_block
|
||||
|
||||
|
||||
def test_topbar_immersive_chrome_does_not_stay_open_from_button_focus() -> None:
|
||||
source = Path("apps/web/src/components/TopBar.tsx").read_text(encoding="utf-8")
|
||||
|
||||
header_start = source.index("<header")
|
||||
header_end = source.index(">", header_start)
|
||||
header_block = source[header_start:header_end]
|
||||
assert "focus-within:translate-y-0" not in header_block
|
||||
|
||||
|
||||
def test_app_switches_between_studio_and_immersive_realtime_views() -> None:
|
||||
source = Path("apps/web/src/App.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert "conversationViewMode" in source
|
||||
assert "immersiveActive" in source
|
||||
assert 'conversationViewMode === "immersive"' in source
|
||||
|
||||
|
||||
def test_app_exits_immersive_realtime_view_with_escape() -> None:
|
||||
source = Path("apps/web/src/App.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert 'event.key !== "Escape"' in source
|
||||
assert 'setConversationViewMode("studio")' in source
|
||||
|
||||
|
||||
def test_app_offers_immersive_avatar_adjustment_controls() -> None:
|
||||
source = Path("apps/web/src/App.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert "immersiveAvatarAdjust" in source
|
||||
assert "avatarAdjust={immersiveActive ? immersiveAvatarAdjust : undefined}" in source
|
||||
assert "画面微调" in source
|
||||
assert "水平" in source
|
||||
assert "垂直" in source
|
||||
assert "缩放" in source
|
||||
assert "重置" in source
|
||||
|
||||
|
||||
def test_immersive_avatar_adjustment_controls_have_wide_ranges() -> None:
|
||||
source = Path("apps/web/src/App.tsx").read_text(encoding="utf-8")
|
||||
|
||||
x_label = source.index("<span>水平</span>")
|
||||
y_label = source.index("<span>垂直</span>")
|
||||
scale_label = source.index("<span>缩放</span>")
|
||||
x_block = source[x_label:source.index('value={immersiveAvatarAdjust.x}', x_label)]
|
||||
y_block = source[y_label:source.index('value={immersiveAvatarAdjust.y}', y_label)]
|
||||
scale_block = source[scale_label:source.index('value={immersiveAvatarAdjust.scale}', scale_label)]
|
||||
|
||||
assert 'min="-480"' in x_block
|
||||
assert 'max="480"' in x_block
|
||||
assert 'min="-320"' in y_block
|
||||
assert 'max="320"' in y_block
|
||||
assert 'min="0.4"' in scale_block
|
||||
assert 'max="2.2"' in scale_block
|
||||
|
||||
|
||||
def test_immersive_avatar_adjustment_panel_uses_opaque_background() -> None:
|
||||
source = Path("apps/web/src/App.tsx").read_text(encoding="utf-8")
|
||||
|
||||
panel_start = source.index('className="w-64 rounded-l-xl')
|
||||
panel_end = source.index('画面微调', panel_start)
|
||||
panel_block = source[panel_start:panel_end]
|
||||
assert "bg-slate-950 " in panel_block
|
||||
assert "bg-slate-950/" not in panel_block
|
||||
assert "backdrop-blur" not in panel_block
|
||||
|
||||
|
||||
def test_app_gates_persisted_immersive_mode_while_start_is_visible() -> None:
|
||||
source = Path("apps/web/src/App.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert 'const effectiveConversationViewMode = showStart ? "studio" : conversationViewMode;' in source
|
||||
assert 'effectiveConversationViewMode === "immersive"' in source
|
||||
@@ -36,7 +36,7 @@ def test_realtime_settings_memory_does_not_render_editing_tools() -> None:
|
||||
def test_asset_library_exposes_memory_tab_and_management_panel() -> None:
|
||||
source = (ROOT / "apps/web/src/components/AssetLibraryWorkspace.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert 'type AssetTab = "exports" | "knowledge" | "memory" | "avatars" | "voices"' in source
|
||||
assert 'type AssetTab = "exports" | "knowledge" | "memory" | "scenes" | "voices"' in source
|
||||
assert '{ id: "memory", label: "记忆库" }' in source
|
||||
assert "renderMemoryTab" in source
|
||||
assert "<MemoryPanel" in source
|
||||
|
||||
111
tests/frontend/test_scene_assets_ui.py
Normal file
111
tests/frontend/test_scene_assets_ui.py
Normal file
@@ -0,0 +1,111 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def test_api_exposes_scene_asset_types_and_helpers() -> None:
|
||||
source = Path("apps/web/src/lib/api.ts").read_text(encoding="utf-8")
|
||||
|
||||
assert "export type SceneBackgroundAsset" in source
|
||||
assert "export type SceneComposition" in source
|
||||
assert "uploadSceneBackground" in source
|
||||
assert 'apiPostForm<SceneBackgroundAsset>("/scene-assets/backgrounds", form)' in source
|
||||
assert 'apiPost<SceneComposition>("/scene-assets/compositions", input)' in source
|
||||
|
||||
|
||||
def test_asset_library_has_scene_assets_tab() -> None:
|
||||
source = Path("apps/web/src/components/AssetLibraryWorkspace.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert 'type AssetTab = "exports" | "knowledge" | "memory" | "scenes" | "voices"' in source
|
||||
assert '{ id: "scenes", label: "场景资产" }' in source
|
||||
assert "renderScenesTab" in source
|
||||
assert "背景资产" in source
|
||||
assert "场景组合" in source
|
||||
assert "uploadSceneBackground" in source
|
||||
assert "createSceneComposition" in source
|
||||
|
||||
|
||||
def test_app_passes_avatars_to_asset_library_workspace() -> None:
|
||||
source = Path("apps/web/src/App.tsx").read_text(encoding="utf-8")
|
||||
|
||||
asset_library_mount = source.split('workflow === "assetLibrary" ? (', 1)[1].split(
|
||||
') : workflow === "videoCreation" ? (',
|
||||
1,
|
||||
)[0]
|
||||
assert "<AssetLibraryWorkspace" in asset_library_mount
|
||||
assert "avatars={avatars}" in asset_library_mount
|
||||
|
||||
|
||||
def test_app_wires_scene_selection_and_background_updates_to_asset_library_workspace() -> None:
|
||||
source = Path("apps/web/src/App.tsx").read_text(encoding="utf-8")
|
||||
|
||||
asset_library_mount = source.split('workflow === "assetLibrary" ? (', 1)[1].split(
|
||||
') : workflow === "videoCreation" ? (',
|
||||
1,
|
||||
)[0]
|
||||
assert "selectedSceneIdsByAvatar={selectedSceneIdsByAvatar}" in asset_library_mount
|
||||
assert "onSceneSelect={handleSceneSelect}" in asset_library_mount
|
||||
assert "onSceneBackgroundsChange={setSceneBackgrounds}" in asset_library_mount
|
||||
|
||||
|
||||
def test_realtime_stage_uses_avatar_default_scene_without_cross_avatar_leakage() -> None:
|
||||
source = Path("apps/web/src/App.tsx").read_text(encoding="utf-8")
|
||||
|
||||
selected_scene_block = source.split("const selectedScene = useMemo(", 1)[1].split(
|
||||
"const dismissToast",
|
||||
1,
|
||||
)[0]
|
||||
assert "selectedSceneIdsByAvatar[avatarId]" in selected_scene_block
|
||||
assert "scene.avatar_id === avatarId" in selected_scene_block
|
||||
assert "matchingScenes.length === 1 ? matchingScenes[0] : null" in selected_scene_block
|
||||
assert "[avatarId, sceneCompositions, selectedSceneIdsByAvatar]" in selected_scene_block
|
||||
|
||||
|
||||
def test_asset_library_scene_cards_can_select_created_scene_and_sync_backgrounds() -> None:
|
||||
source = Path("apps/web/src/components/AssetLibraryWorkspace.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert "selectedSceneIdsByAvatar?: Record<string, string>;" in source
|
||||
assert "onSceneSelect?: (scene: SceneComposition) => void;" in source
|
||||
assert "onSceneClear?: (avatarId: string) => void;" in source
|
||||
assert "onSceneBackgroundsChange?: (backgrounds: SceneBackgroundAsset[]) => void;" in source
|
||||
assert "onSceneSelect?.(created)" in source
|
||||
assert "onSceneBackgroundsChange?.(nextBackgrounds)" in source
|
||||
assert "onSceneSelect?.(scene)" in source
|
||||
assert "selectedSceneIdsByAvatar[scene.avatar_id] === scene.id" in source
|
||||
assert "设为默认" in source
|
||||
assert "当前默认" in source
|
||||
assert "取消默认" in source
|
||||
|
||||
|
||||
def test_asset_library_groups_scene_compositions_by_avatar() -> None:
|
||||
source = Path("apps/web/src/components/AssetLibraryWorkspace.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert "sceneGroups" in source
|
||||
assert "avatarById" in source
|
||||
assert "sceneCompositions.filter((scene) => scene.avatar_id === avatar.id)" in source
|
||||
|
||||
|
||||
def test_scene_delete_actions_use_error_handled_handlers() -> None:
|
||||
source = Path("apps/web/src/components/AssetLibraryWorkspace.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert "const handleDeleteSceneBackground = useCallback(async (background: SceneBackgroundAsset)" in source
|
||||
assert "const handleDeleteSceneComposition = useCallback(async (scene: SceneComposition)" in source
|
||||
assert "delete scene background failed" in source
|
||||
assert "delete scene composition failed" in source
|
||||
assert "err instanceof ApiError ? err.detail : null" in source
|
||||
assert "onNotify?.(detail ? `删除失败:${detail}` : \"删除失败,请稍后重试。\", \"error\")" in source
|
||||
assert "await loadScenes()" in source
|
||||
assert ".then(loadScenes)" not in source
|
||||
|
||||
|
||||
def test_scene_ui_omits_matting_readiness_copy() -> None:
|
||||
asset_source = Path("apps/web/src/components/AssetLibraryWorkspace.tsx").read_text(encoding="utf-8")
|
||||
avatar_source = Path("apps/web/src/components/AvatarSelectionStage.tsx").read_text(encoding="utf-8")
|
||||
api_source = Path("apps/web/src/lib/api.ts").read_text(encoding="utf-8")
|
||||
|
||||
assert 'matting_status: "unknown" | "opaque" | "transparent_ready";' in api_source
|
||||
assert "已抠像/透明数字人" not in asset_source
|
||||
assert "抠像状态未知" not in asset_source
|
||||
assert "未抠像" not in asset_source
|
||||
assert 'avatar?.matting_status === "unknown"' not in asset_source
|
||||
assert "matting_status" not in avatar_source
|
||||
@@ -338,12 +338,14 @@ def test_asset_library_workspace_lists_exported_videos():
|
||||
assert "AssetLibraryWorkspace" in app
|
||||
assert "assetLibrary" in topbar
|
||||
assert "导出视频" in asset
|
||||
assert "Avatar资产" in asset
|
||||
assert "场景资产" in asset
|
||||
assert "声音资产" in asset
|
||||
assert 'apiGet<{ items: ExportVideoItem[] }>("/exports/videos")' in asset
|
||||
assert "listSceneCompositions()" in asset
|
||||
assert "download_url" in asset
|
||||
assert "navigator.clipboard.writeText" in asset
|
||||
assert "apiDelete(`/exports/videos/${item.id}`)" in asset
|
||||
assert "deleteSceneComposition(scene.id)" in asset
|
||||
|
||||
|
||||
def test_realtime_recording_uses_browser_media_recorder_and_uploads_export():
|
||||
|
||||
Reference in New Issue
Block a user