fix: read sqlite transcripts in qa mention scans

extensions/qa-lab/src/fixture-utils.ts

@@ -1,6 +1,7 @@
 // Qa Lab plugin module provides reusable fixture utilities.
 import fs from "node:fs/promises";
 import path from "node:path";
+import { DatabaseSync } from "node:sqlite";
 import { clearTimeout as clearNodeTimeout, setTimeout as setNodeTimeout } from "node:timers";
 
 export type QaFixtureFetchJsonOptions = {
@@ -264,15 +265,40 @@ function recordRole(record: unknown): string | undefined {
   return typeof message.role === "string" ? message.role : undefined;
 }
 
-function shouldScanSessionLogLine(line: string): boolean {
+function collectStringLeaves(value: unknown, output: string[]) {
+  if (typeof value === "string") {
+    output.push(value);
+    return;
+  }
+  if (Array.isArray(value)) {
+    for (const item of value) {
+      collectStringLeaves(item, output);
+    }
+    return;
+  }
+  if (!value || typeof value !== "object") {
+    return;
+  }
+  for (const item of Object.values(value)) {
+    collectStringLeaves(item, output);
+  }
+}
+
+function sessionLogScanText(line: string): string | null {
   const trimmed = line.trim();
   if (!trimmed) {
-    return false;
+    return null;
   }
   try {
-    return recordRole(JSON.parse(trimmed)) !== "user";
+    const record = JSON.parse(trimmed) as unknown;
+    if (recordRole(record) === "user") {
+      return null;
+    }
+    const strings: string[] = [];
+    collectStringLeaves(record, strings);
+    return strings.join("\n");
   } catch {
-    return true;
+    return line;
   }
 }
 
@@ -280,16 +306,78 @@ async function countNeedlesInFile(filePath: string, needles: Record<string, stri
   const text = await fs.readFile(filePath, "utf8").catch(() => "");
   const counts = createCounts(needles);
   for (const line of text.split(/\r?\n/u)) {
-    if (!shouldScanSessionLogLine(line)) {
+    const scanText = sessionLogScanText(line);
+    if (scanText === null) {
       continue;
     }
     for (const [key, needle] of Object.entries(needles)) {
-      counts[key] += countOccurrences(line, needle);
+      counts[key] += countOccurrences(scanText, needle);
     }
   }
   return counts;
 }
 
+function resolveAgentSqlitePathFromSessionsDir(sessionsDir: string): string | null {
+  if (path.basename(sessionsDir) !== "sessions") {
+    return null;
+  }
+  return path.join(path.dirname(sessionsDir), "agent", "openclaw-agent.sqlite");
+}
+
+function countNeedlesInSqliteTranscriptEvents(
+  sqlitePath: string,
+  needles: Record<string, string>,
+): Record<string, number> {
+  const counts = createCounts(needles);
+  let db: DatabaseSync | null = null;
+  try {
+    db = new DatabaseSync(sqlitePath, { readOnly: true });
+    const hasTranscriptEvents = db
+      .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'transcript_events'")
+      .get();
+    if (!hasTranscriptEvents) {
+      return counts;
+    }
+    const rows = db.prepare("SELECT event_json FROM transcript_events ORDER BY session_id, seq");
+    for (const row of rows.iterate() as Iterable<{ event_json?: unknown }>) {
+      if (typeof row.event_json !== "string") {
+        continue;
+      }
+      const scanText = sessionLogScanText(row.event_json);
+      if (scanText === null) {
+        continue;
+      }
+      for (const [key, needle] of Object.entries(needles)) {
+        counts[key] += countOccurrences(scanText, needle);
+      }
+    }
+    return counts;
+  } catch {
+    return counts;
+  } finally {
+    db?.close();
+  }
+}
+
+async function countNeedlesInSqliteTranscriptStore(
+  sessionsDir: string,
+  needles: Record<string, string>,
+): Promise<Record<string, number>> {
+  const sqlitePath = resolveAgentSqlitePathFromSessionsDir(sessionsDir);
+  if (!sqlitePath) {
+    return createCounts(needles);
+  }
+  try {
+    const stat = await fs.stat(sqlitePath);
+    if (!stat.isFile()) {
+      return createCounts(needles);
+    }
+  } catch {
+    return createCounts(needles);
+  }
+  return countNeedlesInSqliteTranscriptEvents(sqlitePath, needles);
+}
+
 export async function countSessionLogMentions(params: {
   sessionsDir: string;
   needles: Record<string, string>;
@@ -308,6 +396,10 @@ export async function countSessionLogMentions(params: {
       counts[key] = (counts[key] ?? 0) + count;
     }
   }
+  const sqliteCounts = await countNeedlesInSqliteTranscriptStore(params.sessionsDir, params.needles);
+  for (const [key, count] of Object.entries(sqliteCounts)) {
+    counts[key] = (counts[key] ?? 0) + count;
+  }
   return counts;
 }
 

extensions/qa-lab/src/tool-search-gateway.fixture.test.ts

@@ -2,6 +2,7 @@
 import fs from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
+import { DatabaseSync } from "node:sqlite";
 import { describe, expect, it } from "vitest";
 import {
   countSessionLogMentions,
@@ -138,6 +139,68 @@ describe("tool search gateway e2e session log scanner", () => {
       await fs.rm(stateDir, { recursive: true, force: true });
     }
   });
+
+  it("counts target mentions from SQLite transcript rows", async () => {
+    const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-tool-search-sqlite-"));
+    const sqlitePath = path.join(stateDir, "agents", "qa", "agent", "openclaw-agent.sqlite");
+    await fs.mkdir(path.dirname(sqlitePath), { recursive: true });
+    const db = new DatabaseSync(sqlitePath);
+    try {
+      const sessionsDir = path.join(stateDir, "agents", "qa", "sessions");
+      db.exec(`
+        CREATE TABLE transcript_events (
+          session_id TEXT NOT NULL,
+          seq INTEGER NOT NULL,
+          event_json TEXT NOT NULL,
+          created_at INTEGER NOT NULL,
+          PRIMARY KEY (session_id, seq)
+        );
+      `);
+      const insert = db.prepare(
+        "INSERT INTO transcript_events (session_id, seq, event_json, created_at) VALUES (?, ?, ?, ?)",
+      );
+      insert.run(
+        "sqlite-session",
+        1,
+        JSON.stringify({
+          message: {
+            role: "user",
+            content: "tool search qa check target=fake_plugin_tool_17",
+          },
+        }),
+        1,
+      );
+      insert.run(
+        "sqlite-session",
+        2,
+        JSON.stringify({
+          message: {
+            role: "assistant",
+            content: 'FAKE_PLUGIN_OK fake_plugin_tool_17 via tool_search_code quoted_call("alpha")',
+          },
+        }),
+        2,
+      );
+
+      await expect(
+        countSessionLogMentions({
+          sessionsDir,
+          needles: {
+            fake_plugin_tool_17: "fake_plugin_tool_17",
+            quoted_call: 'quoted_call("alpha")',
+            tool_search_code: "tool_search_code",
+          },
+        }),
+      ).resolves.toEqual({
+        fake_plugin_tool_17: 1,
+        quoted_call: 1,
+        tool_search_code: 1,
+      });
+    } finally {
+      db.close();
+      await fs.rm(stateDir, { recursive: true, force: true });
+    }
+  });
 });
 
 describe("qa fixture response helpers", () => {

scripts/e2e/lib/session-log-mentions.ts

@@ -1,6 +1,7 @@
 // Session Log Mentions script supports OpenClaw repository automation.
 import fs from "node:fs/promises";
 import path from "node:path";
+import { DatabaseSync } from "node:sqlite";
 import { readPositiveIntEnv } from "./env-limits.mjs";
 
 export type SessionLogMentionLimits = {
@@ -69,15 +70,40 @@ function recordRole(record: unknown): string | undefined {
   return typeof message.role === "string" ? message.role : undefined;
 }
 
-function shouldScanSessionLogLine(line: string): boolean {
+function collectStringLeaves(value: unknown, output: string[]) {
+  if (typeof value === "string") {
+    output.push(value);
+    return;
+  }
+  if (Array.isArray(value)) {
+    for (const item of value) {
+      collectStringLeaves(item, output);
+    }
+    return;
+  }
+  if (!value || typeof value !== "object") {
+    return;
+  }
+  for (const item of Object.values(value)) {
+    collectStringLeaves(item, output);
+  }
+}
+
+function sessionLogScanText(line: string): string | null {
   const trimmed = line.trim();
   if (!trimmed) {
-    return false;
+    return null;
   }
   try {
-    return recordRole(JSON.parse(trimmed)) !== "user";
+    const record = JSON.parse(trimmed) as unknown;
+    if (recordRole(record) === "user") {
+      return null;
+    }
+    const strings: string[] = [];
+    collectStringLeaves(record, strings);
+    return strings.join("\n");
   } catch {
-    return true;
+    return line;
   }
 }
 
@@ -104,11 +130,16 @@ export async function countSessionLogMentions(params: {
 }): Promise<Record<string, number>> {
   const limits = params.limits ?? readSessionLogMentionLimits();
   const counts = createCounts(params.needles);
+  const addCounts = (nextCounts: Record<string, number>) => {
+    for (const [key, count] of Object.entries(nextCounts)) {
+      counts[key] = (counts[key] ?? 0) + count;
+    }
+  };
   let files: string[];
   try {
     files = await fs.readdir(params.sessionsDir);
   } catch {
-    return counts;
+    files = [];
   }
 
   let totalBytes = 0;
@@ -140,13 +171,91 @@ export async function countSessionLogMentions(params: {
       limit: limits.fileMaxBytes,
     });
     for (const line of raw.split(/\r?\n/u)) {
-      if (!shouldScanSessionLogLine(line)) {
+      const scanText = sessionLogScanText(line);
+      if (scanText === null) {
         continue;
       }
       for (const [key, needle] of Object.entries(params.needles)) {
-        counts[key] += countOccurrences(line, needle);
+        counts[key] += countOccurrences(scanText, needle);
       }
     }
   }
+  addCounts(
+    await countSqliteTranscriptMentions({
+      limits,
+      needles: params.needles,
+      sessionsDir: params.sessionsDir,
+      startingBytes: totalBytes,
+    }),
+  );
   return counts;
 }
+
+function resolveAgentSqlitePathFromSessionsDir(sessionsDir: string): string | null {
+  if (path.basename(sessionsDir) !== "sessions") {
+    return null;
+  }
+  return path.join(path.dirname(sessionsDir), "agent", "openclaw-agent.sqlite");
+}
+
+async function countSqliteTranscriptMentions(params: {
+  limits: SessionLogMentionLimits;
+  needles: SessionLogNeedles;
+  sessionsDir: string;
+  startingBytes: number;
+}): Promise<Record<string, number>> {
+  const counts = createCounts(params.needles);
+  const sqlitePath = resolveAgentSqlitePathFromSessionsDir(params.sessionsDir);
+  if (!sqlitePath) {
+    return counts;
+  }
+  const stat = await fs.stat(sqlitePath).catch(() => null);
+  if (!stat?.isFile()) {
+    return counts;
+  }
+  let totalBytes = params.startingBytes;
+  let db: DatabaseSync | null = null;
+  try {
+    db = new DatabaseSync(sqlitePath, { readOnly: true });
+    const hasTranscriptEvents = db
+      .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'transcript_events'")
+      .get();
+    if (!hasTranscriptEvents) {
+      return counts;
+    }
+    const rows = db.prepare("SELECT event_json FROM transcript_events ORDER BY session_id, seq");
+    for (const row of rows.iterate() as Iterable<{ event_json?: unknown }>) {
+      if (typeof row.event_json !== "string") {
+        continue;
+      }
+      const byteCount = Buffer.byteLength(row.event_json, "utf8");
+      assertWithinLimit({
+        byteCount,
+        filePath: sqlitePath,
+        label: "per-file",
+        limit: params.limits.fileMaxBytes,
+      });
+      totalBytes += byteCount;
+      assertWithinLimit({
+        byteCount: totalBytes,
+        label: "total",
+        limit: params.limits.totalMaxBytes,
+      });
+      const scanText = sessionLogScanText(row.event_json);
+      if (scanText === null) {
+        continue;
+      }
+      for (const [key, needle] of Object.entries(params.needles)) {
+        counts[key] += countOccurrences(scanText, needle);
+      }
+    }
+    return counts;
+  } catch (error) {
+    if (error && typeof error === "object" && "code" in error) {
+      throw error;
+    }
+    return counts;
+  } finally {
+    db?.close();
+  }
+}

test/scripts/session-log-mentions.test.ts

@@ -3,6 +3,7 @@ import { mkdtempSync, rmSync } from "node:fs";
 import fs from "node:fs/promises";
 import { tmpdir } from "node:os";
 import path from "node:path";
+import { DatabaseSync } from "node:sqlite";
 import { afterEach, describe, expect, it } from "vitest";
 import {
   countSessionLogMentions,
@@ -61,7 +62,7 @@ describe("session log mention scanner", () => {
         }),
         JSON.stringify({
           role: "assistant",
-          content: "API.read MCP.fixture fixture__lookup_note",
+          content: 'API.read MCP.fixture fixture__lookup_note tools.search("lookup note")',
         }),
         "raw transcript fallback API.read",
         "",
@@ -75,12 +76,114 @@ describe("session log mention scanner", () => {
           apiFileRead: "API.read",
           mcpNamespace: "MCP.fixture",
           mcpTool: "fixture__lookup_note",
+          toolSearchPollution: 'tools.search("lookup note"',
         },
       }),
     ).resolves.toEqual({
       apiFileRead: 2,
       mcpNamespace: 1,
       mcpTool: 1,
+      toolSearchPollution: 1,
+    });
+  });
+
+  it("counts mentions from SQLite transcript rows", async () => {
+    const root = makeTempRoot();
+    const sessionsDir = path.join(root, "agents", "main", "sessions");
+    const sqlitePath = path.join(root, "agents", "main", "agent", "openclaw-agent.sqlite");
+    await fs.mkdir(path.dirname(sqlitePath), { recursive: true });
+    const db = new DatabaseSync(sqlitePath);
+    try {
+      db.exec(`
+        CREATE TABLE transcript_events (
+          session_id TEXT NOT NULL,
+          seq INTEGER NOT NULL,
+          event_json TEXT NOT NULL,
+          created_at INTEGER NOT NULL,
+          PRIMARY KEY (session_id, seq)
+        );
+      `);
+      const insert = db.prepare(
+        "INSERT INTO transcript_events (session_id, seq, event_json, created_at) VALUES (?, ?, ?, ?)",
+      );
+      insert.run(
+        "sqlite-session",
+        1,
+        JSON.stringify({
+          message: {
+            role: "user",
+            content: "Use API.read and MCP.fixture from the prompt.",
+          },
+        }),
+        1,
+      );
+      insert.run(
+        "sqlite-session",
+        2,
+        JSON.stringify({
+          message: {
+            role: "assistant",
+            content: 'API.read MCP.fixture fixture__lookup_note tools.search("lookup note")',
+          },
+        }),
+        2,
+      );
+    } finally {
+      db.close();
+    }
+
+    await expect(
+      countSessionLogMentions({
+        sessionsDir,
+        needles: {
+          apiFileRead: "API.read",
+          mcpNamespace: "MCP.fixture",
+          mcpTool: "fixture__lookup_note",
+          toolSearchPollution: 'tools.search("lookup note"',
+        },
+      }),
+    ).resolves.toEqual({
+      apiFileRead: 1,
+      mcpNamespace: 1,
+      mcpTool: 1,
+      toolSearchPollution: 1,
+    });
+  });
+
+  it("rejects oversized SQLite transcript rows before counting them", async () => {
+    const root = makeTempRoot();
+    const sessionsDir = path.join(root, "agents", "main", "sessions");
+    const sqlitePath = path.join(root, "agents", "main", "agent", "openclaw-agent.sqlite");
+    await fs.mkdir(path.dirname(sqlitePath), { recursive: true });
+    const db = new DatabaseSync(sqlitePath);
+    try {
+      db.exec(`
+        CREATE TABLE transcript_events (
+          session_id TEXT NOT NULL,
+          seq INTEGER NOT NULL,
+          event_json TEXT NOT NULL,
+          created_at INTEGER NOT NULL,
+          PRIMARY KEY (session_id, seq)
+        );
+      `);
+      db.prepare(
+        "INSERT INTO transcript_events (session_id, seq, event_json, created_at) VALUES (?, ?, ?, ?)",
+      ).run("sqlite-session", 1, "API.read ".repeat(16), 1);
+    } finally {
+      db.close();
+    }
+
+    await expect(
+      countSessionLogMentions({
+        limits: { fileMaxBytes: 32, totalMaxBytes: 1024 },
+        sessionsDir,
+        needles: {
+          apiFileRead: "API.read",
+        },
+      }),
+    ).rejects.toMatchObject({
+      code: "ETOOBIG",
+      message: expect.stringContaining("per-file limit"),
     });
   });