mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-13 05:31:55 +08:00
feat(xai): support Grok Imagine Video 1.5 (#103316)
* feat(media): add per-model generation catalog metadata * feat(xai): support Grok Imagine Video 1.5 * test(plugin-sdk): update public surface budget
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
071e30ecd5
commit
75dfd3dc09
@@ -582,6 +582,9 @@ request. Plugin dependencies are expected to be present before runtime load.
|
||||
- Provider-specific Vydra coverage:
|
||||
- `OPENCLAW_LIVE_TEST=1 OPENCLAW_LIVE_VYDRA_VIDEO=1 pnpm test:live -- extensions/vydra/vydra.live.test.ts`
|
||||
- That file runs `veo3` text-to-video plus a `kling` image-to-video lane that uses a remote image URL fixture by default (`OPENCLAW_LIVE_VYDRA_KLING_IMAGE_URL` to override).
|
||||
- Provider-specific xAI Video 1.5 coverage:
|
||||
- `OPENCLAW_LIVE_TEST=1 OPENCLAW_LIVE_XAI_VIDEO_15=1 pnpm test:live -- extensions/xai/xai.live.test.ts -t "Grok Imagine Video 1.5"`
|
||||
- The case generates a local PNG first frame, requests a one-second 1080P image-to-video clip, polls to completion, and verifies the downloaded video buffer.
|
||||
- Current `videoToVideo` live coverage:
|
||||
- `runway` only when the selected model resolves to `gen4_aleph`
|
||||
- Current declared-but-skipped `videoToVideo` providers in the shared sweep:
|
||||
|
||||
@@ -934,6 +934,7 @@ catalog, API-key auth, and dynamic model resolution.
|
||||
id: "acme-ai",
|
||||
label: "Acme Video",
|
||||
defaultTimeoutMs: 600_000,
|
||||
models: ["acme-video", "acme-image-video"],
|
||||
capabilities: {
|
||||
generate: { maxVideos: 1, maxDurationSeconds: 10, supportsResolution: true },
|
||||
imageToVideo: {
|
||||
@@ -945,6 +946,21 @@ catalog, API-key auth, and dynamic model resolution.
|
||||
},
|
||||
videoToVideo: { enabled: false },
|
||||
},
|
||||
catalogByModel: {
|
||||
"acme-image-video": {
|
||||
modes: ["imageToVideo"],
|
||||
capabilities: {
|
||||
imageToVideo: {
|
||||
enabled: true,
|
||||
maxVideos: 1,
|
||||
maxInputImages: 1,
|
||||
resolutions: ["480P", "720P", "1080P"],
|
||||
supportsResolution: true,
|
||||
},
|
||||
videoToVideo: { enabled: false },
|
||||
},
|
||||
},
|
||||
},
|
||||
generateVideo: async (req) => ({ videos: [] }),
|
||||
});
|
||||
```
|
||||
@@ -952,6 +968,13 @@ catalog, API-key auth, and dynamic model resolution.
|
||||
`capabilities` is required on both provider types; `edit` and the
|
||||
video transform blocks (`imageToVideo`, `videoToVideo`) always need an
|
||||
explicit `enabled` flag.
|
||||
|
||||
Use `catalogByModel` when a listed model's static modes or capabilities
|
||||
differ from the provider defaults. This metadata keeps
|
||||
`video_generate action=list` and model catalogs accurate without
|
||||
invoking provider code. Request-time capability lookup and enforcement
|
||||
still belong in `resolveModelCapabilities` and `generateVideo`; reuse
|
||||
the same capability constant for both paths when possible.
|
||||
</Tab>
|
||||
<Tab title="Web fetch and search">
|
||||
```typescript
|
||||
|
||||
@@ -134,7 +134,7 @@ below or under known limits.
|
||||
| Server-side X search | `x_search` tool | Yes |
|
||||
| Server-side code execution | `code_execution` tool | Yes |
|
||||
| Images | `image_generate` | Yes |
|
||||
| Videos | `video_generate` | Classic model; Video 1.5 is not exposed yet |
|
||||
| Videos | `video_generate` | Classic full workflow; Video 1.5 image-to-video |
|
||||
| Batch text-to-speech | `messages.tts.provider: "xai"` / `tts` | Yes |
|
||||
| Streaming TTS | - | Not implemented by the xAI provider yet |
|
||||
| Batch speech-to-text | `tools.media.audio` media understanding | Yes |
|
||||
@@ -210,13 +210,17 @@ stale context metadata on active 4.20 rows. It does not pin active 4.20
|
||||
The bundled `xai` plugin registers video generation through the shared
|
||||
`video_generate` tool.
|
||||
|
||||
- Default video model: `xai/grok-imagine-video`
|
||||
- Modes: text-to-video, image-to-video, reference-image generation, remote
|
||||
video edit, and remote video extension
|
||||
- Aspect ratios: `1:1`, `16:9`, `9:16`, `4:3`, `3:4`, `3:2`, `2:3`
|
||||
- Resolutions: `480P`, `720P`
|
||||
- Default model: `xai/grok-imagine-video`
|
||||
- Additional model: `xai/grok-imagine-video-1.5`
|
||||
- Classic modes: text-to-video, image-to-video, reference-image generation,
|
||||
remote video edit, and remote video extension
|
||||
- Video 1.5 mode: image-to-video only, with exactly one first-frame image
|
||||
- Aspect ratios: `1:1`, `16:9`, `9:16`, `4:3`, `3:4`, `3:2`, `2:3`;
|
||||
Video 1.5 inherits the source image ratio when omitted
|
||||
- Resolutions: classic `480P`/`720P`; Video 1.5 supports `480P`, `720P`,
|
||||
and `1080P`, and defaults to `480P`
|
||||
- Duration: 1-15 seconds for generation/image-to-video, 1-10 seconds when
|
||||
using `reference_image` roles, 2-10 seconds for extension
|
||||
using classic `reference_image` roles, 2-10 seconds for classic extension
|
||||
- Reference-image generation: set `imageRoles` to `reference_image` for
|
||||
every supplied image; xAI accepts up to 7 such images
|
||||
- Default operation timeout: 600 seconds unless `video_generate.timeoutMs`
|
||||
@@ -228,6 +232,10 @@ stale context metadata on active 4.20 rows. It does not pin active 4.20
|
||||
OpenClaw encodes those as data URLs for xAI.
|
||||
</Warning>
|
||||
|
||||
Video 1.5 also recognizes xAI's `grok-imagine-video-1.5-preview` and
|
||||
`grok-imagine-video-1.5-2026-05-30` identifiers. OpenClaw forwards the
|
||||
selected identifier unchanged, but applies the same image-only validation.
|
||||
|
||||
To use xAI as the default video provider:
|
||||
|
||||
```json5
|
||||
@@ -502,9 +510,6 @@ stale context metadata on active 4.20 rows. It does not pin active 4.20
|
||||
- xAI Realtime voice is not registered as an OpenClaw provider yet. It
|
||||
needs a different bidirectional voice session contract than batch STT
|
||||
or streaming transcription.
|
||||
- `grok-imagine-video-1.5` is not exposed yet. Unlike the classic video
|
||||
model, it is image-to-video only and needs model-specific mode and 1080p
|
||||
validation in the shared provider contract.
|
||||
- xAI image `quality`, image `mask`, and extra native-only aspect ratios
|
||||
are not exposed until the shared `image_generate` tool has
|
||||
corresponding cross-provider controls.
|
||||
@@ -546,6 +551,7 @@ The xAI media paths are covered by unit tests and opt-in live suites. Export
|
||||
```bash
|
||||
pnpm test extensions/xai
|
||||
OPENCLAW_LIVE_TEST=1 OPENCLAW_LIVE_TEST_QUIET=1 pnpm test:live -- extensions/xai/xai.live.test.ts
|
||||
OPENCLAW_LIVE_TEST=1 OPENCLAW_LIVE_XAI_VIDEO_15=1 pnpm test:live -- extensions/xai/xai.live.test.ts -t "Grok Imagine Video 1.5"
|
||||
OPENCLAW_LIVE_TEST=1 OPENCLAW_LIVE_TEST_QUIET=1 pnpm test:live -- extensions/xai/x-search.live.test.ts
|
||||
OPENCLAW_LIVE_GATEWAY_MODELS="xai/grok-4.5,xai/grok-build-0.1,xai/grok-4.3,xai/grok-4.20-0309-reasoning,xai/grok-4.20-0309-non-reasoning" OPENCLAW_LIVE_GATEWAY_MAX_MODELS=0 OPENCLAW_LIVE_GATEWAY_SMOKE=0 pnpm test:live -- src/gateway/gateway-models.profiles.live.test.ts
|
||||
OPENCLAW_LIVE_TEST=1 OPENCLAW_LIVE_TEST_QUIET=1 OPENCLAW_LIVE_IMAGE_GENERATION_PROVIDERS=xai pnpm test:live -- test/image-generation.runtime.live.test.ts
|
||||
@@ -555,7 +561,9 @@ The provider-specific live file synthesizes normal TTS, telephony-friendly PCM
|
||||
TTS, transcribes audio through xAI batch STT, streams the same PCM through xAI
|
||||
realtime STT, generates text-to-image output, and edits a reference image.
|
||||
The shared image live file verifies the same xAI provider through OpenClaw's
|
||||
runtime selection, fallback, normalization, and media attachment path.
|
||||
runtime selection, fallback, normalization, and media attachment path. The
|
||||
opt-in Video 1.5 case submits one generated first-frame image at 1080P and
|
||||
verifies the completed video download.
|
||||
|
||||
## Related
|
||||
|
||||
|
||||
@@ -119,7 +119,7 @@ openclaw tasks cancel <lookup>
|
||||
| Runway | `gen4.5` | ✓ | 1 image | 1 video | `RUNWAYML_API_SECRET` |
|
||||
| Together | `Wan-AI/Wan2.2-T2V-A14B` | ✓ | `Wan-AI/Wan2.2-I2V-A14B` only | - | `TOGETHER_API_KEY` |
|
||||
| Vydra | `veo3` | ✓ | 1 image (`kling`) | - | `VYDRA_API_KEY` |
|
||||
| xAI | `grok-imagine-video` | ✓ | 1 first-frame image or up to 7 `reference_image`s | 1 video | `XAI_API_KEY` |
|
||||
| xAI | `grok-imagine-video` | ✓ | Classic: 1 first frame or 7 references; 1.5: 1 frame | Classic: 1 video | `XAI_API_KEY` |
|
||||
|
||||
Some providers accept additional or alternate API key env vars. See
|
||||
individual [provider pages](#related) for details.
|
||||
@@ -147,7 +147,7 @@ the shared live sweep:
|
||||
| Runway | ✓ | ✓ | ✓ | `generate`, `imageToVideo`; `videoToVideo` runs only when the selected model is `runway/gen4_aleph` |
|
||||
| Together | ✓ | ✓ | - | `generate`, `imageToVideo` |
|
||||
| Vydra | ✓ | ✓ | - | `generate`; shared `imageToVideo` skipped because bundled `veo3` is text-only and bundled `kling` requires a remote image URL |
|
||||
| xAI | ✓ | ✓ | ✓ | `generate`, `imageToVideo`; `videoToVideo` skipped because this provider currently needs a remote MP4 URL |
|
||||
| xAI | ✓ | ✓ | ✓ | Classic supports all modes; Video 1.5 is image-to-video only; remote MP4 input keeps `videoToVideo` out of the shared sweep |
|
||||
|
||||
## Tool parameters
|
||||
|
||||
@@ -423,9 +423,16 @@ only the explicit `model`, `primary`, and `fallbacks` entries.
|
||||
a remote image URL.
|
||||
</Accordion>
|
||||
<Accordion title="xAI">
|
||||
Supports text-to-video, single first-frame image-to-video, up to 7
|
||||
`reference_image` inputs through xAI `reference_images`, and remote
|
||||
video edit/extend flows.
|
||||
The default `grok-imagine-video` model supports text-to-video, single
|
||||
first-frame image-to-video, up to 7 `reference_image` inputs through xAI
|
||||
`reference_images`, and remote video edit/extend flows.
|
||||
|
||||
`grok-imagine-video-1.5` is image-to-video only: provide exactly one image.
|
||||
It supports 1-15 seconds and `480P`, `720P`, or `1080P`, defaulting to
|
||||
`480P`; omit `aspectRatio` to inherit the source image ratio. The preview
|
||||
and dated 1.5 identifiers receive the same validation and are forwarded
|
||||
unchanged.
|
||||
|
||||
</Accordion>
|
||||
</AccordionGroup>
|
||||
|
||||
|
||||
@@ -4,10 +4,15 @@ import {
|
||||
installProviderHttpMockCleanup,
|
||||
} from "openclaw/plugin-sdk/provider-http-test-mocks";
|
||||
import { expectExplicitVideoGenerationCapabilities } from "openclaw/plugin-sdk/provider-test-contracts";
|
||||
import type { VideoGenerationRequest } from "openclaw/plugin-sdk/video-generation";
|
||||
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { postJsonRequestMock, fetchWithTimeoutMock, readProviderJsonResponseMock } =
|
||||
getProviderHttpMocks();
|
||||
const {
|
||||
postJsonRequestMock,
|
||||
fetchWithTimeoutMock,
|
||||
readProviderJsonResponseMock,
|
||||
resolveApiKeyForProviderMock,
|
||||
} = getProviderHttpMocks();
|
||||
|
||||
let buildXaiVideoGenerationProvider: typeof import("./video-generation-provider.js").buildXaiVideoGenerationProvider;
|
||||
|
||||
@@ -161,6 +166,142 @@ describe("xai video generation provider", () => {
|
||||
expectExplicitVideoGenerationCapabilities(buildXaiVideoGenerationProvider());
|
||||
});
|
||||
|
||||
it("advertises canonical 1.5 and resolves capabilities for all API aliases", async () => {
|
||||
const provider = buildXaiVideoGenerationProvider();
|
||||
|
||||
expect(provider.defaultModel).toBe("grok-imagine-video");
|
||||
expect(provider.models).toEqual(["grok-imagine-video", "grok-imagine-video-1.5"]);
|
||||
expect(provider.catalogByModel?.["grok-imagine-video-1.5"]).toMatchObject({
|
||||
modes: ["imageToVideo"],
|
||||
capabilities: {
|
||||
imageToVideo: {
|
||||
enabled: true,
|
||||
maxInputImages: 1,
|
||||
resolutions: ["480P", "720P", "1080P"],
|
||||
},
|
||||
videoToVideo: { enabled: false },
|
||||
},
|
||||
});
|
||||
|
||||
for (const model of [
|
||||
"grok-imagine-video-1.5",
|
||||
"grok-imagine-video-1.5-preview",
|
||||
"grok-imagine-video-1.5-2026-05-30",
|
||||
]) {
|
||||
const capabilities = await provider.resolveModelCapabilities?.({
|
||||
provider: "xai",
|
||||
model,
|
||||
cfg: {},
|
||||
});
|
||||
expect(capabilities?.imageToVideo).toMatchObject({
|
||||
enabled: true,
|
||||
maxInputImages: 1,
|
||||
maxDurationSeconds: 15,
|
||||
resolutions: ["480P", "720P", "1080P"],
|
||||
});
|
||||
expect(capabilities?.videoToVideo?.enabled).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it("uses the 1.5 default while preserving aliases, 1080p, and source aspect ratio", async () => {
|
||||
const models = [
|
||||
"grok-imagine-video-1.5",
|
||||
"grok-imagine-video-1.5-preview",
|
||||
"grok-imagine-video-1.5-2026-05-30",
|
||||
];
|
||||
const provider = buildXaiVideoGenerationProvider();
|
||||
|
||||
for (const [index, model] of models.entries()) {
|
||||
const requestId = `req_15_${index}`;
|
||||
postJsonRequestMock.mockResolvedValueOnce({
|
||||
response: { json: async () => ({ request_id: requestId }) },
|
||||
release: vi.fn(async () => {}),
|
||||
});
|
||||
fetchWithTimeoutMock
|
||||
.mockResolvedValueOnce({
|
||||
json: async () => ({
|
||||
request_id: requestId,
|
||||
status: "done",
|
||||
video: { url: `https://cdn.x.ai/${requestId}.mp4` },
|
||||
}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
headers: new Headers({ "content-type": "video/mp4" }),
|
||||
arrayBuffer: async () => Buffer.from("video-bytes"),
|
||||
});
|
||||
|
||||
const result = await provider.generateVideo({
|
||||
provider: "xai",
|
||||
model,
|
||||
prompt: "Animate this still image",
|
||||
cfg: {},
|
||||
durationSeconds: 20,
|
||||
resolution: index === 0 ? undefined : "1080P",
|
||||
inputImages: [
|
||||
{
|
||||
url: "https://example.com/first-frame.png",
|
||||
...(index === 0 ? {} : { role: "first_frame" as const }),
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const body = requirePostJsonCall(index).body ?? {};
|
||||
expect(body.model).toBe(model);
|
||||
expect(body.image).toEqual({ url: "https://example.com/first-frame.png" });
|
||||
expect(body.duration).toBe(15);
|
||||
expect(body.resolution).toBe(index === 0 ? "480p" : "1080p");
|
||||
expect(body).not.toHaveProperty("aspect_ratio");
|
||||
expect(result.model).toBe(model);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects unsupported 1.5 modes before submitting a request", async () => {
|
||||
const provider = buildXaiVideoGenerationProvider();
|
||||
const cases: Array<
|
||||
Pick<VideoGenerationRequest, "model" | "inputImages" | "inputVideos"> & { error: string }
|
||||
> = [
|
||||
{
|
||||
model: "grok-imagine-video-1.5",
|
||||
inputImages: undefined,
|
||||
error: "xAI grok-imagine-video-1.5 requires exactly one first-frame image.",
|
||||
},
|
||||
{
|
||||
model: "grok-imagine-video-1.5-preview",
|
||||
inputImages: [{ url: "https://example.com/reference.png", role: "reference_image" }],
|
||||
error: "xAI grok-imagine-video-1.5 supports only an ordinary or first_frame image.",
|
||||
},
|
||||
{
|
||||
model: "grok-imagine-video-1.5-2026-05-30",
|
||||
inputImages: [
|
||||
{ url: "https://example.com/first.png" },
|
||||
{ url: "https://example.com/second.png", role: "first_frame" },
|
||||
],
|
||||
error: "xAI grok-imagine-video-1.5 requires exactly one first-frame image.",
|
||||
},
|
||||
{
|
||||
model: "grok-imagine-video-1.5",
|
||||
inputImages: undefined,
|
||||
inputVideos: [{ url: "https://example.com/input.mp4" }],
|
||||
error: "xAI grok-imagine-video-1.5 does not support video inputs.",
|
||||
},
|
||||
];
|
||||
|
||||
for (const testCase of cases) {
|
||||
await expect(
|
||||
provider.generateVideo({
|
||||
provider: "xai",
|
||||
model: testCase.model,
|
||||
prompt: "Unsupported 1.5 request",
|
||||
cfg: {},
|
||||
inputImages: testCase.inputImages,
|
||||
inputVideos: testCase.inputVideos,
|
||||
}),
|
||||
).rejects.toThrow(testCase.error);
|
||||
}
|
||||
expect(resolveApiKeyForProviderMock).not.toHaveBeenCalled();
|
||||
expect(postJsonRequestMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("creates, polls, and downloads a generated video", async () => {
|
||||
postJsonRequestMock.mockResolvedValue({
|
||||
response: {
|
||||
|
||||
@@ -21,15 +21,37 @@ import { isRecord, normalizeOptionalString } from "openclaw/plugin-sdk/string-co
|
||||
import type {
|
||||
GeneratedVideoAsset,
|
||||
VideoGenerationProvider,
|
||||
VideoGenerationProviderCapabilities,
|
||||
VideoGenerationRequest,
|
||||
} from "openclaw/plugin-sdk/video-generation";
|
||||
|
||||
const DEFAULT_XAI_VIDEO_BASE_URL = "https://api.x.ai/v1";
|
||||
const DEFAULT_XAI_VIDEO_MODEL = "grok-imagine-video";
|
||||
const XAI_VIDEO_15_MODEL = "grok-imagine-video-1.5";
|
||||
const XAI_VIDEO_15_MODEL_IDS = new Set([
|
||||
XAI_VIDEO_15_MODEL,
|
||||
"grok-imagine-video-1.5-preview",
|
||||
"grok-imagine-video-1.5-2026-05-30",
|
||||
]);
|
||||
const DEFAULT_TIMEOUT_MS = 600_000;
|
||||
const POLL_INTERVAL_MS = 5_000;
|
||||
const MAX_POLL_ATTEMPTS = 120;
|
||||
const XAI_VIDEO_ASPECT_RATIOS = new Set(["1:1", "16:9", "9:16", "4:3", "3:4", "3:2", "2:3"]);
|
||||
const XAI_VIDEO_15_CAPABILITIES = {
|
||||
imageToVideo: {
|
||||
enabled: true,
|
||||
maxVideos: 1,
|
||||
maxInputImages: 1,
|
||||
maxDurationSeconds: 15,
|
||||
aspectRatios: [...XAI_VIDEO_ASPECT_RATIOS],
|
||||
resolutions: ["480P", "720P", "1080P"],
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
},
|
||||
videoToVideo: {
|
||||
enabled: false,
|
||||
},
|
||||
} satisfies VideoGenerationProviderCapabilities;
|
||||
const XAI_VIDEO_MALFORMED_RESPONSE = "xAI video generation response malformed";
|
||||
// xAI documents these as the only meaningful values; everything else (queued,
|
||||
// processing, submitted, pending, in_progress, ...) means "keep polling".
|
||||
@@ -37,6 +59,7 @@ const XAI_VIDEO_TERMINAL_FAILURE_STATUSES = new Set(["failed", "error", "expired
|
||||
const XAI_VIDEO_DEFAULT_DURATION_SECONDS = 8;
|
||||
const XAI_VIDEO_DEFAULT_ASPECT_RATIO = "16:9";
|
||||
const XAI_VIDEO_DEFAULT_RESOLUTION = "720p";
|
||||
const XAI_VIDEO_15_DEFAULT_RESOLUTION = "480p";
|
||||
const DEFAULT_GENERATED_VIDEO_MAX_BYTES = 16 * 1024 * 1024;
|
||||
|
||||
type XaiVideoCreateResponse = {
|
||||
@@ -155,6 +178,32 @@ function isReferenceImage(input: VideoGenerationSourceInput): boolean {
|
||||
return normalizeOptionalString(input.role)?.toLowerCase() === "reference_image";
|
||||
}
|
||||
|
||||
function isXaiVideo15Model(model: string | undefined): boolean {
|
||||
const normalized = normalizeOptionalString(model);
|
||||
return normalized ? XAI_VIDEO_15_MODEL_IDS.has(normalized) : false;
|
||||
}
|
||||
|
||||
function isFirstFrameImage(input: VideoGenerationSourceInput): boolean {
|
||||
const role = normalizeOptionalString(input.role)?.toLowerCase();
|
||||
return role === undefined || role === "first_frame";
|
||||
}
|
||||
|
||||
function validateXaiVideo15Request(req: VideoGenerationRequest): void {
|
||||
if (!isXaiVideo15Model(req.model)) {
|
||||
return;
|
||||
}
|
||||
if ((req.inputVideos?.length ?? 0) > 0) {
|
||||
throw new Error("xAI grok-imagine-video-1.5 does not support video inputs.");
|
||||
}
|
||||
const inputImages = req.inputImages ?? [];
|
||||
if (inputImages.length !== 1) {
|
||||
throw new Error("xAI grok-imagine-video-1.5 requires exactly one first-frame image.");
|
||||
}
|
||||
if (!isFirstFrameImage(inputImages[0])) {
|
||||
throw new Error("xAI grok-imagine-video-1.5 supports only an ordinary or first_frame image.");
|
||||
}
|
||||
}
|
||||
|
||||
function resolveInputVideoUrl(input: VideoGenerationSourceInput | undefined): string | undefined {
|
||||
if (!input) {
|
||||
return undefined;
|
||||
@@ -189,7 +238,10 @@ function resolveAspectRatio(value: string | undefined): string | undefined {
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
function resolveResolution(value: string | undefined): "480p" | "720p" | undefined {
|
||||
function resolveResolution(
|
||||
value: string | undefined,
|
||||
options?: { allow1080p?: boolean },
|
||||
): "480p" | "720p" | "1080p" | undefined {
|
||||
if (typeof value !== "string") {
|
||||
return undefined;
|
||||
}
|
||||
@@ -197,9 +249,12 @@ function resolveResolution(value: string | undefined): "480p" | "720p" | undefin
|
||||
if (normalized === "480p") {
|
||||
return "480p";
|
||||
}
|
||||
if (normalized === "720p" || normalized === "1080p") {
|
||||
if (normalized === "720p") {
|
||||
return "720p";
|
||||
}
|
||||
if (normalized === "1080p") {
|
||||
return options?.allow1080p ? "1080p" : "720p";
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
@@ -223,6 +278,7 @@ function resolveXaiVideoMode(
|
||||
}
|
||||
|
||||
function buildCreateBody(req: VideoGenerationRequest): Record<string, unknown> {
|
||||
validateXaiVideo15Request(req);
|
||||
const inputImages = req.inputImages ?? [];
|
||||
const hasReferenceImages = inputImages.some(isReferenceImage);
|
||||
if (hasReferenceImages && !inputImages.every(isReferenceImage)) {
|
||||
@@ -245,11 +301,14 @@ function buildCreateBody(req: VideoGenerationRequest): Record<string, unknown> {
|
||||
|
||||
const mode = resolveXaiVideoMode(req);
|
||||
const body: Record<string, unknown> = {
|
||||
// Aliases are API-owned routing choices. Preserve the selected identifier
|
||||
// instead of silently pinning it to the canonical 1.5 model.
|
||||
model: normalizeOptionalString(req.model) ?? DEFAULT_XAI_VIDEO_MODEL,
|
||||
prompt: req.prompt,
|
||||
};
|
||||
|
||||
if (mode === "generate") {
|
||||
const isVideo15 = isXaiVideo15Model(req.model);
|
||||
const imageUrl = resolveImageUrl(req.inputImages?.[0]);
|
||||
if (imageUrl) {
|
||||
body.image = { url: imageUrl };
|
||||
@@ -260,8 +319,16 @@ function buildCreateBody(req: VideoGenerationRequest): Record<string, unknown> {
|
||||
min: 1,
|
||||
max: 15,
|
||||
}) ?? XAI_VIDEO_DEFAULT_DURATION_SECONDS;
|
||||
body.aspect_ratio = resolveAspectRatio(req.aspectRatio) ?? XAI_VIDEO_DEFAULT_ASPECT_RATIO;
|
||||
body.resolution = resolveResolution(req.resolution) ?? XAI_VIDEO_DEFAULT_RESOLUTION;
|
||||
const aspectRatio = resolveAspectRatio(req.aspectRatio);
|
||||
// 1.5 inherits the source image ratio when callers do not choose one.
|
||||
if (aspectRatio || !isVideo15) {
|
||||
body.aspect_ratio = aspectRatio ?? XAI_VIDEO_DEFAULT_ASPECT_RATIO;
|
||||
}
|
||||
const defaultResolution = isVideo15
|
||||
? XAI_VIDEO_15_DEFAULT_RESOLUTION
|
||||
: XAI_VIDEO_DEFAULT_RESOLUTION;
|
||||
body.resolution =
|
||||
resolveResolution(req.resolution, { allow1080p: isVideo15 }) ?? defaultResolution;
|
||||
return body;
|
||||
}
|
||||
|
||||
@@ -380,7 +447,13 @@ export function buildXaiVideoGenerationProvider(): VideoGenerationProvider {
|
||||
label: "xAI",
|
||||
defaultModel: DEFAULT_XAI_VIDEO_MODEL,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
models: [DEFAULT_XAI_VIDEO_MODEL],
|
||||
models: [DEFAULT_XAI_VIDEO_MODEL, XAI_VIDEO_15_MODEL],
|
||||
catalogByModel: {
|
||||
[XAI_VIDEO_15_MODEL]: {
|
||||
capabilities: XAI_VIDEO_15_CAPABILITIES,
|
||||
modes: ["imageToVideo"],
|
||||
},
|
||||
},
|
||||
isConfigured: ({ agentDir }) =>
|
||||
isProviderApiKeyConfigured({
|
||||
provider: "xai",
|
||||
@@ -414,7 +487,17 @@ export function buildXaiVideoGenerationProvider(): VideoGenerationProvider {
|
||||
supportsResolution: true,
|
||||
},
|
||||
},
|
||||
resolveModelCapabilities: ({ model }): VideoGenerationProviderCapabilities | undefined => {
|
||||
if (!isXaiVideo15Model(model)) {
|
||||
return undefined;
|
||||
}
|
||||
return XAI_VIDEO_15_CAPABILITIES;
|
||||
},
|
||||
async generateVideo(req) {
|
||||
// Validate provider/model mode constraints before auth or HTTP setup so
|
||||
// unsupported 1.5 requests cannot be submitted and billed accidentally.
|
||||
const createBody = buildCreateBody(req);
|
||||
const createEndpoint = resolveCreateEndpoint(req);
|
||||
const auth = await resolveApiKeyForProvider({
|
||||
provider: "xai",
|
||||
cfg: req.cfg,
|
||||
@@ -448,9 +531,9 @@ export function buildXaiVideoGenerationProvider(): VideoGenerationProvider {
|
||||
const submitHeaders = new Headers(headers);
|
||||
submitHeaders.set("x-idempotency-key", crypto.randomUUID());
|
||||
const { response, release } = await postJsonRequest({
|
||||
url: `${baseUrl}${resolveCreateEndpoint(req)}`,
|
||||
url: `${baseUrl}${createEndpoint}`,
|
||||
headers: submitHeaders,
|
||||
body: buildCreateBody(req),
|
||||
body: createBody,
|
||||
timeoutMs: resolveProviderOperationTimeoutMs({
|
||||
deadline,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
|
||||
@@ -21,6 +21,7 @@ import { XAI_DEFAULT_STT_MODEL } from "./stt.js";
|
||||
|
||||
const XAI_API_KEY = process.env.XAI_API_KEY ?? "";
|
||||
const LIVE_IMAGE_MODEL = process.env.OPENCLAW_LIVE_XAI_IMAGE_MODEL?.trim() || "grok-imagine-image";
|
||||
const ENABLE_VIDEO_15_LIVE = process.env.OPENCLAW_LIVE_XAI_VIDEO_15 === "1";
|
||||
const liveEnabled = XAI_API_KEY.trim().length > 0 && process.env.OPENCLAW_LIVE_TEST === "1";
|
||||
const describeLive = liveEnabled ? describe : describe.skip;
|
||||
const EMPTY_AUTH_STORE = { version: 1, profiles: {} } as const;
|
||||
@@ -63,6 +64,27 @@ function createReferencePng(): Buffer {
|
||||
return encodePngRgba(buf, width, height);
|
||||
}
|
||||
|
||||
function createVideoReferencePng(): Buffer {
|
||||
const width = 384;
|
||||
const height = 384;
|
||||
const buf = Buffer.alloc(width * height * 4, 255);
|
||||
|
||||
for (let y = 0; y < height; y += 1) {
|
||||
for (let x = 0; x < width; x += 1) {
|
||||
const blue = Math.round(160 + (80 * y) / height);
|
||||
fillPixel(buf, x, y, width, 32, 96, blue, 255);
|
||||
}
|
||||
}
|
||||
|
||||
for (let y = 112; y < 272; y += 1) {
|
||||
for (let x = 112; x < 272; x += 1) {
|
||||
fillPixel(buf, x, y, width, 255, 153, 51, 255);
|
||||
}
|
||||
}
|
||||
|
||||
return encodePngRgba(buf, width, height);
|
||||
}
|
||||
|
||||
async function createTempAgentDir(): Promise<string> {
|
||||
return await fs.mkdtemp(path.join(os.tmpdir(), "xai-plugin-live-"));
|
||||
}
|
||||
@@ -357,4 +379,54 @@ describeLive("xai plugin live", () => {
|
||||
}
|
||||
});
|
||||
}, 300_000);
|
||||
|
||||
it.skipIf(!ENABLE_VIDEO_15_LIVE)(
|
||||
"generates a Grok Imagine Video 1.5 clip from one image",
|
||||
async () => {
|
||||
await runXaiLiveCase("video-1.5", async () => {
|
||||
const { videoProviders } = await registerXaiPlugin();
|
||||
const videoProvider = requireRegisteredProvider(videoProviders, "xai");
|
||||
const cfg = createLiveConfig();
|
||||
const agentDir = await createTempAgentDir();
|
||||
|
||||
try {
|
||||
const generated = await videoProvider.generateVideo({
|
||||
provider: "xai",
|
||||
model: "grok-imagine-video-1.5",
|
||||
prompt:
|
||||
"Animate the orange square with a subtle slow rotation. Keep the framing fixed.",
|
||||
cfg,
|
||||
agentDir,
|
||||
authStore: EMPTY_AUTH_STORE,
|
||||
timeoutMs: 10 * 60_000,
|
||||
durationSeconds: 1,
|
||||
resolution: "1080P",
|
||||
inputImages: [
|
||||
{
|
||||
buffer: createVideoReferencePng(),
|
||||
mimeType: "image/png",
|
||||
fileName: "video-reference.png",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(generated.model).toBe("grok-imagine-video-1.5");
|
||||
expect(generated.videos).toHaveLength(1);
|
||||
const video = generated.videos[0];
|
||||
if (!video?.buffer) {
|
||||
throw new Error("xAI Video 1.5 did not return a buffered video");
|
||||
}
|
||||
expect(video.mimeType.startsWith("video/")).toBe(true);
|
||||
expect(video.buffer.byteLength).toBeGreaterThan(1_000);
|
||||
const outputPath = process.env.OPENCLAW_LIVE_XAI_VIDEO_15_OUTPUT?.trim();
|
||||
if (outputPath) {
|
||||
await fs.writeFile(outputPath, video.buffer);
|
||||
}
|
||||
} finally {
|
||||
await fs.rm(agentDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
},
|
||||
12 * 60_000,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -56,6 +56,49 @@ describe("media-generation catalog", () => {
|
||||
).toEqual(["video-default", "video-pro"]);
|
||||
});
|
||||
|
||||
it("uses per-model capabilities and modes when provided", () => {
|
||||
type VideoCapabilities = {
|
||||
generate?: { maxVideos: number };
|
||||
imageToVideo?: { enabled: boolean; maxInputImages: number };
|
||||
};
|
||||
const providerCapabilities: VideoCapabilities = {
|
||||
generate: { maxVideos: 1 },
|
||||
};
|
||||
const alternateCapabilities: VideoCapabilities = {
|
||||
imageToVideo: { enabled: true, maxInputImages: 1 },
|
||||
};
|
||||
|
||||
const rows = synthesizeMediaGenerationCatalogEntries({
|
||||
kind: "video_generation",
|
||||
provider: {
|
||||
id: "example",
|
||||
defaultModel: "default-video",
|
||||
models: ["default-video", "image-video"],
|
||||
capabilities: providerCapabilities,
|
||||
catalogByModel: {
|
||||
"image-video": {
|
||||
capabilities: alternateCapabilities,
|
||||
modes: ["imageToVideo"],
|
||||
},
|
||||
},
|
||||
},
|
||||
modes: ["generate"],
|
||||
});
|
||||
|
||||
expect(rows).toEqual([
|
||||
expect.objectContaining({
|
||||
model: "default-video",
|
||||
capabilities: providerCapabilities,
|
||||
modes: ["generate"],
|
||||
}),
|
||||
expect.objectContaining({
|
||||
model: "image-video",
|
||||
capabilities: alternateCapabilities,
|
||||
modes: ["imageToVideo"],
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("marks a trimmed default model as the catalog default", () => {
|
||||
expect(
|
||||
synthesizeMediaGenerationCatalogEntries({
|
||||
|
||||
@@ -30,6 +30,12 @@ export type MediaGenerationCatalogEntry<TCapabilities = unknown> = {
|
||||
warnings?: readonly string[];
|
||||
};
|
||||
|
||||
/** Static catalog metadata that overrides provider defaults for one model. */
|
||||
export type MediaGenerationCatalogModelEntry<TCapabilities = unknown> = {
|
||||
capabilities?: TCapabilities;
|
||||
modes?: readonly string[];
|
||||
};
|
||||
|
||||
/** Provider metadata used to synthesize static media generation catalog entries. */
|
||||
export type MediaGenerationCatalogProvider<TCapabilities = unknown> = {
|
||||
id: string;
|
||||
@@ -38,6 +44,7 @@ export type MediaGenerationCatalogProvider<TCapabilities = unknown> = {
|
||||
defaultModel?: string;
|
||||
models?: readonly string[];
|
||||
capabilities: TCapabilities;
|
||||
catalogByModel?: Readonly<Record<string, MediaGenerationCatalogModelEntry<TCapabilities>>>;
|
||||
};
|
||||
|
||||
/** Return unique configured models with default model first when present. */
|
||||
@@ -53,12 +60,13 @@ export function synthesizeMediaGenerationCatalogEntries<TCapabilities>(params: {
|
||||
}): Array<MediaGenerationCatalogEntry<TCapabilities>> {
|
||||
const defaultModel = uniqueTrimmedStrings([params.provider.defaultModel])[0];
|
||||
return uniqueModels(params.provider).map((model) => {
|
||||
const modelCatalogEntry = params.provider.catalogByModel?.[model];
|
||||
const entry: MediaGenerationCatalogEntry<TCapabilities> = {
|
||||
kind: params.kind,
|
||||
provider: params.provider.id,
|
||||
model,
|
||||
source: "static",
|
||||
capabilities: params.provider.capabilities,
|
||||
capabilities: modelCatalogEntry?.capabilities ?? params.provider.capabilities,
|
||||
};
|
||||
if (params.provider.label) {
|
||||
entry.label = params.provider.label;
|
||||
@@ -66,8 +74,9 @@ export function synthesizeMediaGenerationCatalogEntries<TCapabilities>(params: {
|
||||
if (model === defaultModel) {
|
||||
entry.default = true;
|
||||
}
|
||||
if (params.modes) {
|
||||
entry.modes = params.modes;
|
||||
const modes = modelCatalogEntry?.modes ?? params.modes;
|
||||
if (modes) {
|
||||
entry.modes = modes;
|
||||
}
|
||||
return entry;
|
||||
});
|
||||
|
||||
@@ -195,7 +195,7 @@ export function readPluginSdkSurfaceBudgets(env = process.env) {
|
||||
),
|
||||
publicExports: readPluginSdkSurfaceBudgetEnv(
|
||||
"OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_EXPORTS",
|
||||
10478,
|
||||
10480,
|
||||
env,
|
||||
),
|
||||
publicFunctionExports: readPluginSdkSurfaceBudgetEnv(
|
||||
|
||||
@@ -26,6 +26,7 @@ type MediaGenerateProvider = {
|
||||
defaultModel?: string;
|
||||
models?: readonly string[];
|
||||
capabilities: unknown;
|
||||
catalogByModel?: Readonly<Record<string, { capabilities?: unknown; modes?: readonly string[] }>>;
|
||||
isConfigured?: (ctx: { cfg?: OpenClawConfig; agentDir?: string }) => boolean;
|
||||
};
|
||||
|
||||
@@ -41,6 +42,11 @@ type MediaGenerateListProviderDetails<TProvider extends MediaGenerateProvider> =
|
||||
catalog: ReturnType<typeof synthesizeMediaGenerationCatalogEntries<TProvider["capabilities"]>>;
|
||||
};
|
||||
|
||||
type MediaGenerateCapabilitySummaryOptions = {
|
||||
modes?: readonly string[];
|
||||
includeModes?: boolean;
|
||||
};
|
||||
|
||||
/** Common tool result shape for media generation list/status actions. */
|
||||
export type { MediaGenerateActionResult };
|
||||
|
||||
@@ -56,7 +62,10 @@ export function createMediaGenerateProviderListActionResult<
|
||||
agentDir?: string;
|
||||
authStore?: AuthProfileStore;
|
||||
listModes: (provider: TProvider) => string[];
|
||||
summarizeCapabilities: (provider: TProvider) => string;
|
||||
summarizeCapabilities: (
|
||||
provider: TProvider,
|
||||
options?: MediaGenerateCapabilitySummaryOptions,
|
||||
) => string;
|
||||
formatAuthHint?: (provider: { id: string; authEnvVars: readonly string[] }) => string | undefined;
|
||||
}): MediaGenerateActionResult {
|
||||
if (params.providers.length === 0) {
|
||||
@@ -104,6 +113,22 @@ export function createMediaGenerateProviderListActionResult<
|
||||
const authHint =
|
||||
params.formatAuthHint?.({ id: details.id, authEnvVars: authHints }) ??
|
||||
(authHints.length > 0 ? `set ${authHints.join(" / ")} to use ${details.id}/*` : undefined);
|
||||
const modelCapabilityLines = details.catalog.flatMap((entry) => {
|
||||
if (!provider.catalogByModel?.[entry.model]) {
|
||||
return [];
|
||||
}
|
||||
const modelProvider = {
|
||||
...provider,
|
||||
capabilities: entry.capabilities ?? provider.capabilities,
|
||||
} as TProvider;
|
||||
const modelCapabilities = params.summarizeCapabilities(modelProvider, {
|
||||
modes: entry.modes,
|
||||
includeModes: false,
|
||||
});
|
||||
const modelModes = entry.modes?.length ? `modes=${entry.modes.join("/")}` : undefined;
|
||||
const modelSummary = [modelModes, modelCapabilities || undefined].filter(Boolean).join(", ");
|
||||
return [` model ${entry.model}: ${modelSummary || "no capabilities declared"}`];
|
||||
});
|
||||
return [
|
||||
`${details.id}${details.defaultModel ? ` (default ${details.defaultModel})` : ""}`,
|
||||
` models: ${modelLine}`,
|
||||
@@ -111,6 +136,7 @@ export function createMediaGenerateProviderListActionResult<
|
||||
...(authHint ? [` auth: ${authHint}`] : []),
|
||||
" source: static",
|
||||
...(capabilities ? [` capabilities: ${capabilities}`] : []),
|
||||
...modelCapabilityLines,
|
||||
];
|
||||
});
|
||||
|
||||
|
||||
@@ -24,11 +24,26 @@ type VideoGenerateActionResult = MediaGenerateActionResult;
|
||||
|
||||
function summarizeVideoGenerationCapabilities(
|
||||
provider: ReturnType<typeof listRuntimeVideoGenerationProviders>[number],
|
||||
options?: { modes?: readonly string[]; includeModes?: boolean },
|
||||
): string {
|
||||
const supportedModes = listSupportedVideoGenerationModes(provider);
|
||||
const supportedModes = options?.modes ?? listSupportedVideoGenerationModes(provider);
|
||||
const generate = provider.capabilities.generate;
|
||||
const imageToVideo = provider.capabilities.imageToVideo;
|
||||
const videoToVideo = provider.capabilities.videoToVideo;
|
||||
const activeModeCapabilities = [
|
||||
supportedModes.includes("generate") ? generate : undefined,
|
||||
supportedModes.includes("imageToVideo") && imageToVideo?.enabled ? imageToVideo : undefined,
|
||||
supportedModes.includes("videoToVideo") && videoToVideo?.enabled ? videoToVideo : undefined,
|
||||
].filter((capabilities) => capabilities !== undefined);
|
||||
const maxDurationSeconds = activeModeCapabilities
|
||||
.map((capabilities) => capabilities.maxDurationSeconds)
|
||||
.find((value) => typeof value === "number");
|
||||
const supportedDurationSeconds = activeModeCapabilities
|
||||
.map((capabilities) => capabilities.supportedDurationSeconds)
|
||||
.find((value) => value && value.length > 0);
|
||||
const supportedDurationSecondsByModel = activeModeCapabilities
|
||||
.map((capabilities) => capabilities.supportedDurationSecondsByModel)
|
||||
.find((value) => value && Object.keys(value).length > 0);
|
||||
// providerOptions may be declared at the mode level (generate) or at the flat
|
||||
// provider-capabilities level. The runtime checks both; surface the union so
|
||||
// the agent sees a single merged view of which opaque keys each provider
|
||||
@@ -52,28 +67,39 @@ function summarizeVideoGenerationCapabilities(
|
||||
videoToVideo?.maxInputAudios ??
|
||||
provider.capabilities.maxInputAudios;
|
||||
const capabilities = [
|
||||
supportedModes.length > 0 ? `modes=${supportedModes.join("/")}` : null,
|
||||
options?.includeModes !== false && supportedModes.length > 0
|
||||
? `modes=${supportedModes.join("/")}`
|
||||
: null,
|
||||
generate?.maxVideos ? `maxVideos=${generate.maxVideos}` : null,
|
||||
imageToVideo?.maxInputImages ? `maxInputImages=${imageToVideo.maxInputImages}` : null,
|
||||
videoToVideo?.maxInputVideos ? `maxInputVideos=${videoToVideo.maxInputVideos}` : null,
|
||||
typeof maxInputAudios === "number" && maxInputAudios > 0
|
||||
? `maxInputAudios=${maxInputAudios}`
|
||||
: null,
|
||||
generate?.maxDurationSeconds ? `maxDurationSeconds=${generate.maxDurationSeconds}` : null,
|
||||
generate?.supportedDurationSeconds?.length
|
||||
? `supportedDurationSeconds=${generate.supportedDurationSeconds.join("/")}`
|
||||
maxDurationSeconds ? `maxDurationSeconds=${maxDurationSeconds}` : null,
|
||||
supportedDurationSeconds
|
||||
? `supportedDurationSeconds=${supportedDurationSeconds.join("/")}`
|
||||
: null,
|
||||
generate?.supportedDurationSecondsByModel &&
|
||||
Object.keys(generate.supportedDurationSecondsByModel).length > 0
|
||||
? `supportedDurationSecondsByModel=${Object.entries(generate.supportedDurationSecondsByModel)
|
||||
supportedDurationSecondsByModel
|
||||
? `supportedDurationSecondsByModel=${Object.entries(supportedDurationSecondsByModel)
|
||||
.map(([modelId, durations]) => `${modelId}:${durations.join("/")}`)
|
||||
.join("; ")}`
|
||||
: null,
|
||||
generate?.supportsResolution ? "resolution" : null,
|
||||
generate?.supportsAspectRatio ? "aspectRatio" : null,
|
||||
generate?.supportsSize ? "size" : null,
|
||||
generate?.supportsAudio ? "audio" : null,
|
||||
generate?.supportsWatermark ? "watermark" : null,
|
||||
activeModeCapabilities.some((modeCapabilities) => modeCapabilities.supportsResolution)
|
||||
? "resolution"
|
||||
: null,
|
||||
activeModeCapabilities.some((modeCapabilities) => modeCapabilities.supportsAspectRatio)
|
||||
? "aspectRatio"
|
||||
: null,
|
||||
activeModeCapabilities.some((modeCapabilities) => modeCapabilities.supportsSize)
|
||||
? "size"
|
||||
: null,
|
||||
activeModeCapabilities.some((modeCapabilities) => modeCapabilities.supportsAudio)
|
||||
? "audio"
|
||||
: null,
|
||||
activeModeCapabilities.some((modeCapabilities) => modeCapabilities.supportsWatermark)
|
||||
? "watermark"
|
||||
: null,
|
||||
Object.keys(declaredProviderOptions).length > 0
|
||||
? `providerOptions={${Object.entries(declaredProviderOptions)
|
||||
.map(([key, type]) => `${key}:${type}`)
|
||||
|
||||
@@ -1295,6 +1295,73 @@ describe("createVideoGenerateTool", () => {
|
||||
expect(providers[0]?.modes).toEqual(["generate", "imageToVideo"]);
|
||||
});
|
||||
|
||||
it("lists model-specific catalog capabilities and modes", async () => {
|
||||
const imageToVideoCapabilities = {
|
||||
imageToVideo: {
|
||||
enabled: true,
|
||||
maxInputImages: 1,
|
||||
maxDurationSeconds: 15,
|
||||
resolutions: ["480P", "720P", "1080P"] as const,
|
||||
aspectRatios: ["16:9", "9:16"] as const,
|
||||
supportsResolution: true,
|
||||
supportsAspectRatio: true,
|
||||
},
|
||||
};
|
||||
vi.spyOn(videoGenerationRuntime, "listRuntimeVideoGenerationProviders").mockReturnValue([
|
||||
{
|
||||
id: "video-plugin",
|
||||
defaultModel: "text-video",
|
||||
models: ["text-video", "image-video"],
|
||||
capabilities: {
|
||||
generate: {
|
||||
maxDurationSeconds: 10,
|
||||
},
|
||||
},
|
||||
catalogByModel: {
|
||||
"image-video": {
|
||||
capabilities: imageToVideoCapabilities,
|
||||
modes: ["imageToVideo"],
|
||||
},
|
||||
},
|
||||
generateVideo: vi.fn(async () => {
|
||||
throw new Error("not used");
|
||||
}),
|
||||
},
|
||||
]);
|
||||
|
||||
const tool = createVideoGenerateTool({
|
||||
config: asConfig({
|
||||
agents: {
|
||||
defaults: {
|
||||
videoGenerationModel: { primary: "video-plugin/text-video" },
|
||||
},
|
||||
},
|
||||
}),
|
||||
});
|
||||
if (!tool) {
|
||||
throw new Error("expected video_generate tool");
|
||||
}
|
||||
|
||||
const result = await tool.execute("call-1", { action: "list" });
|
||||
const text = (result.content?.[0] as { text: string } | undefined)?.text ?? "";
|
||||
expect(text).toContain(
|
||||
"model image-video: modes=imageToVideo, maxInputImages=1, maxDurationSeconds=15, resolution, aspectRatio",
|
||||
);
|
||||
const providers = resultDetails(result).providers as Array<{
|
||||
catalog?: Array<{
|
||||
model?: string;
|
||||
capabilities?: unknown;
|
||||
modes?: string[];
|
||||
}>;
|
||||
}>;
|
||||
const catalogEntry = providers[0]?.catalog?.find((entry) => entry.model === "image-video");
|
||||
expect(catalogEntry).toMatchObject({
|
||||
model: "image-video",
|
||||
capabilities: imageToVideoCapabilities,
|
||||
modes: ["imageToVideo"],
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects image-to-video when the provider disables that mode", async () => {
|
||||
vi.spyOn(videoGenerationRuntime, "listRuntimeVideoGenerationProviders").mockReturnValue([
|
||||
{
|
||||
|
||||
@@ -5,6 +5,7 @@ export type { FallbackAttempt } from "../agents/model-fallback.types.js";
|
||||
export type { VideoGenerationProviderPlugin } from "../plugins/types.js";
|
||||
export type {
|
||||
GeneratedVideoAsset,
|
||||
VideoGenerationCatalogModelEntry,
|
||||
VideoGenerationIgnoredOverride,
|
||||
VideoGenerationMode,
|
||||
VideoGenerationModeCapabilities,
|
||||
|
||||
@@ -8,6 +8,7 @@ import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import type {
|
||||
GeneratedVideoAsset as CoreGeneratedVideoAsset,
|
||||
VideoGenerationAssetRole as CoreVideoGenerationAssetRole,
|
||||
VideoGenerationCatalogModelEntry as CoreVideoGenerationCatalogModelEntry,
|
||||
VideoGenerationMode as CoreVideoGenerationMode,
|
||||
VideoGenerationModeCapabilities as CoreVideoGenerationModeCapabilities,
|
||||
VideoGenerationModelCapabilitiesContext as CoreVideoGenerationModelCapabilitiesContext,
|
||||
@@ -171,6 +172,12 @@ export type VideoGenerationProviderCapabilities = VideoGenerationModeCapabilitie
|
||||
videoToVideo?: VideoGenerationTransformCapabilities;
|
||||
};
|
||||
|
||||
/** Static catalog metadata that overrides provider defaults for one video model. */
|
||||
export type VideoGenerationCatalogModelEntry = {
|
||||
capabilities?: VideoGenerationProviderCapabilities;
|
||||
modes?: readonly VideoGenerationMode[];
|
||||
};
|
||||
|
||||
/** Video generation provider contract implemented by provider plugins. */
|
||||
export type VideoGenerationProvider = {
|
||||
id: string;
|
||||
@@ -181,6 +188,7 @@ export type VideoGenerationProvider = {
|
||||
defaultTimeoutMs?: number;
|
||||
models?: string[];
|
||||
capabilities: VideoGenerationProviderCapabilities;
|
||||
catalogByModel?: Readonly<Record<string, VideoGenerationCatalogModelEntry>>;
|
||||
isConfigured?: (ctx: VideoGenerationProviderConfiguredContext) => boolean;
|
||||
resolveModelCapabilities?: (
|
||||
ctx: VideoGenerationModelCapabilitiesContext,
|
||||
@@ -201,6 +209,8 @@ const videoGenerationSdkCompat: [
|
||||
AssertAssignable<CoreVideoGenerationProviderOptionType, VideoGenerationProviderOptionType>,
|
||||
AssertAssignable<VideoGenerationMode, CoreVideoGenerationMode>,
|
||||
AssertAssignable<CoreVideoGenerationMode, VideoGenerationMode>,
|
||||
AssertAssignable<VideoGenerationCatalogModelEntry, CoreVideoGenerationCatalogModelEntry>,
|
||||
AssertAssignable<CoreVideoGenerationCatalogModelEntry, VideoGenerationCatalogModelEntry>,
|
||||
AssertAssignable<VideoGenerationModeCapabilities, CoreVideoGenerationModeCapabilities>,
|
||||
AssertAssignable<CoreVideoGenerationModeCapabilities, VideoGenerationModeCapabilities>,
|
||||
AssertAssignable<VideoGenerationProvider, CoreVideoGenerationProvider>,
|
||||
|
||||
@@ -156,6 +156,12 @@ export type VideoGenerationProviderCapabilities = VideoGenerationModeCapabilitie
|
||||
videoToVideo?: VideoGenerationTransformCapabilities;
|
||||
};
|
||||
|
||||
/** Static catalog metadata that overrides provider defaults for one video model. */
|
||||
export type VideoGenerationCatalogModelEntry = {
|
||||
capabilities?: VideoGenerationProviderCapabilities;
|
||||
modes?: readonly VideoGenerationMode[];
|
||||
};
|
||||
|
||||
export type VideoGenerationNormalization = {
|
||||
size?: MediaNormalizationEntry<string>;
|
||||
aspectRatio?: MediaNormalizationEntry<string>;
|
||||
@@ -172,6 +178,7 @@ export type VideoGenerationProvider = {
|
||||
defaultTimeoutMs?: number;
|
||||
models?: string[];
|
||||
capabilities: VideoGenerationProviderCapabilities;
|
||||
catalogByModel?: Readonly<Record<string, VideoGenerationCatalogModelEntry>>;
|
||||
isConfigured?: (ctx: VideoGenerationProviderConfiguredContext) => boolean;
|
||||
resolveModelCapabilities?: (
|
||||
ctx: VideoGenerationModelCapabilitiesContext,
|
||||
|
||||
Reference in New Issue
Block a user