mirror of
https://github.com/larksuite/cli.git
synced 2026-07-07 00:55:53 +08:00
Compare commits
12 Commits
feat/app_r
...
feat/opt-i
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d48f84c096 | ||
|
|
cbd6e56ac0 | ||
|
|
572eb8da41 | ||
|
|
82a099feaf | ||
|
|
51f2a70e6d | ||
|
|
237a77feb3 | ||
|
|
040ef17eae | ||
|
|
736b131cdf | ||
|
|
5efaf65aec | ||
|
|
0991da7446 | ||
|
|
80bea45c6a | ||
|
|
c775cb4360 |
19
.github/workflows/release.yml
vendored
19
.github/workflows/release.yml
vendored
@@ -9,11 +9,7 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
# All platforms (incl. darwin keychain_signer) are CGO-free and cross-compiled
|
||||
# on a single ubuntu runner in one goreleaser run (one checksums.txt). The
|
||||
# darwin signer's runtime FFI is validated separately by the signer-test job.
|
||||
goreleaser:
|
||||
needs: signer-test-macos
|
||||
runs-on: ubuntu-22.04
|
||||
permissions:
|
||||
contents: write
|
||||
@@ -38,21 +34,6 @@ jobs:
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
# Validate the macOS keychain signer on real hardware. The release binaries are
|
||||
# cross-compiled on ubuntu (CGO-free purego FFI), so this is the only step that
|
||||
# needs a Mac — and it gates the release rather than producing it.
|
||||
signer-test-macos:
|
||||
runs-on: macos-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: '1.23'
|
||||
- name: Keychain signer round-trip (CGO-free purego FFI)
|
||||
run: LARK_KEYCHAIN_IT=1 CGO_ENABLED=0 go test -tags keychain_signer -run Keychain -v ./internal/keysigner/
|
||||
|
||||
publish-npm:
|
||||
needs: goreleaser
|
||||
runs-on: ubuntu-22.04
|
||||
|
||||
@@ -5,53 +5,15 @@ before:
|
||||
- python3 scripts/fetch_meta.py
|
||||
|
||||
builds:
|
||||
# Linux & Windows: pure-Go TPM 2.0 signer is compiled in by default (no build
|
||||
# tag), cross-compiled with CGO disabled — the binaries ship the platform key
|
||||
# signer for private_key_jwt. windows/arm64 is the one exception: the sks
|
||||
# Windows dependency stack (go-ole) has no arm64 support, so the signer file is
|
||||
# arch-excluded there and that binary falls back to client_secret only.
|
||||
- id: linux
|
||||
binary: lark-cli
|
||||
main: .
|
||||
- binary: lark-cli
|
||||
env:
|
||||
- CGO_ENABLED=0
|
||||
flags:
|
||||
- -trimpath
|
||||
ldflags:
|
||||
- -s -w -X github.com/larksuite/cli/internal/build.Version={{ .Version }} -X github.com/larksuite/cli/internal/build.Date={{ .Date }}
|
||||
goos:
|
||||
- linux
|
||||
goarch:
|
||||
- amd64
|
||||
- arm64
|
||||
- id: windows
|
||||
binary: lark-cli
|
||||
main: .
|
||||
env:
|
||||
- CGO_ENABLED=0
|
||||
flags:
|
||||
- -trimpath
|
||||
ldflags:
|
||||
- -s -w -X github.com/larksuite/cli/internal/build.Version={{ .Version }} -X github.com/larksuite/cli/internal/build.Date={{ .Date }}
|
||||
goos:
|
||||
- windows
|
||||
goarch:
|
||||
- amd64
|
||||
- arm64
|
||||
# macOS: the keychain signer calls Security.framework via runtime FFI (purego),
|
||||
# so it is CGO-free, compiled into every darwin build (no build tag), and
|
||||
# cross-compiles from the same ubuntu runner as linux/windows.
|
||||
- id: darwin
|
||||
binary: lark-cli
|
||||
main: .
|
||||
env:
|
||||
- CGO_ENABLED=0
|
||||
flags:
|
||||
- -trimpath
|
||||
ldflags:
|
||||
- -s -w -X github.com/larksuite/cli/internal/build.Version={{ .Version }} -X github.com/larksuite/cli/internal/build.Date={{ .Date }}
|
||||
goos:
|
||||
- darwin
|
||||
- linux
|
||||
- windows
|
||||
goarch:
|
||||
- amd64
|
||||
- arm64
|
||||
@@ -61,7 +23,7 @@ archives:
|
||||
- name_template: "lark-cli-{{ .Version }}-{{ .Os }}-{{ .Arch }}"
|
||||
format_overrides:
|
||||
- goos: windows
|
||||
formats: [zip]
|
||||
format: zip
|
||||
files:
|
||||
- README.md
|
||||
- LICENSE
|
||||
|
||||
6
Makefile
6
Makefile
@@ -33,11 +33,7 @@ build: fetch_meta
|
||||
go build -trimpath -ldflags "$(LDFLAGS)" -o $(BINARY) .
|
||||
|
||||
vet: fetch_meta
|
||||
# -unsafeptr=false: the macOS keychain signer dereferences dylib data-symbol
|
||||
# addresses from purego.Dlsym (uintptr->unsafe.Pointer over stable C memory) —
|
||||
# safe FFI, but go vet's unsafeptr can't prove it and has no inline suppress.
|
||||
# golangci-lint still runs full govet (honoring the //nolint:govet) in CI.
|
||||
go vet -unsafeptr=false ./...
|
||||
go vet ./...
|
||||
|
||||
# fmt-check fails when any file would be reformatted by gofmt. Keep this
|
||||
# in sync with the fast-gate "Check formatting" step in CI.
|
||||
|
||||
@@ -265,7 +265,7 @@ func authLoginRun(opts *LoginOptions) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
authResp, err := larkauth.RequestDeviceAuthorization(opts.Ctx, httpClient, larkauth.ClientAuthFromConfig(config), config.Brand, finalScope, f.IOStreams.ErrOut)
|
||||
authResp, err := larkauth.RequestDeviceAuthorization(httpClient, config.AppID, config.AppSecret, config.Brand, finalScope, f.IOStreams.ErrOut)
|
||||
if err != nil {
|
||||
return errs.NewAuthenticationError(errs.SubtypeUnknown, "device authorization failed: %v", err).WithCause(err)
|
||||
}
|
||||
@@ -325,7 +325,7 @@ func authLoginRun(opts *LoginOptions) error {
|
||||
|
||||
// Step 3: Poll for token
|
||||
log(msg.WaitingAuth)
|
||||
result := pollDeviceToken(opts.Ctx, httpClient, larkauth.ClientAuthFromConfig(config), config.Brand,
|
||||
result := pollDeviceToken(opts.Ctx, httpClient, config.AppID, config.AppSecret, config.Brand,
|
||||
authResp.DeviceCode, authResp.Interval, authResp.ExpiresIn, f.IOStreams.ErrOut)
|
||||
|
||||
if !result.OK {
|
||||
@@ -415,7 +415,7 @@ func authLoginPollDeviceCode(opts *LoginOptions, config *core.CliConfig, msg *lo
|
||||
fmt.Fprintln(f.IOStreams.ErrOut, msg.AgentTimeoutHint)
|
||||
}
|
||||
log(msg.WaitingAuth)
|
||||
result := pollDeviceToken(opts.Ctx, httpClient, larkauth.ClientAuthFromConfig(config), config.Brand,
|
||||
result := pollDeviceToken(opts.Ctx, httpClient, config.AppID, config.AppSecret, config.Brand,
|
||||
opts.DeviceCode, 5, 600, f.IOStreams.ErrOut)
|
||||
|
||||
if !result.OK {
|
||||
|
||||
@@ -847,7 +847,7 @@ func TestAuthLoginRun_DeviceCodeTokenNilCleansScopeCache(t *testing.T) {
|
||||
|
||||
original := pollDeviceToken
|
||||
t.Cleanup(func() { pollDeviceToken = original })
|
||||
pollDeviceToken = func(ctx context.Context, httpClient *http.Client, ca larkauth.ClientAuth, brand core.LarkBrand, deviceCode string, interval, expiresIn int, errOut io.Writer) *larkauth.DeviceFlowResult {
|
||||
pollDeviceToken = func(ctx context.Context, httpClient *http.Client, appId, appSecret string, brand core.LarkBrand, deviceCode string, interval, expiresIn int, errOut io.Writer) *larkauth.DeviceFlowResult {
|
||||
return &larkauth.DeviceFlowResult{OK: true, Token: nil}
|
||||
}
|
||||
|
||||
@@ -886,7 +886,7 @@ func TestAuthLoginRun_JSONAbort_StdoutEventOnly_StderrEmpty(t *testing.T) {
|
||||
|
||||
original := pollDeviceToken
|
||||
t.Cleanup(func() { pollDeviceToken = original })
|
||||
pollDeviceToken = func(ctx context.Context, httpClient *http.Client, ca larkauth.ClientAuth, brand core.LarkBrand, deviceCode string, interval, expiresIn int, errOut io.Writer) *larkauth.DeviceFlowResult {
|
||||
pollDeviceToken = func(ctx context.Context, httpClient *http.Client, appId, appSecret string, brand core.LarkBrand, deviceCode string, interval, expiresIn int, errOut io.Writer) *larkauth.DeviceFlowResult {
|
||||
return &larkauth.DeviceFlowResult{OK: false, Message: "user denied"}
|
||||
}
|
||||
|
||||
|
||||
@@ -193,7 +193,7 @@ func TestSaveInitConfig_OmitLangPreservesPrior(t *testing.T) {
|
||||
t.Fatalf("seed config: %v", err)
|
||||
}
|
||||
|
||||
if err := saveInitConfig("", existing, f, "cli_x", core.PlainSecret("s2"), core.BrandFeishu, "", "", nil); err != nil {
|
||||
if err := saveInitConfig("", existing, f, "cli_x", core.PlainSecret("s2"), core.BrandFeishu, ""); err != nil {
|
||||
t.Fatalf("saveInitConfig (no --lang): %v", err)
|
||||
}
|
||||
|
||||
@@ -206,88 +206,6 @@ func TestSaveInitConfig_OmitLangPreservesPrior(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeyRefFromResult_PrivateKeyJWT(t *testing.T) {
|
||||
ref := keyRefFromResult(&configInitResult{
|
||||
AuthMethod: core.AuthMethodPrivateKeyJWT,
|
||||
KeyLabel: "lark-cli-default",
|
||||
})
|
||||
if ref == nil {
|
||||
t.Fatal("keyRefFromResult returned nil")
|
||||
}
|
||||
if ref.Source != "tee" || ref.ID != "lark-cli-default" {
|
||||
t.Fatalf("key ref = %#v, want tee/lark-cli-default", ref)
|
||||
}
|
||||
|
||||
if ref := keyRefFromResult(&configInitResult{AuthMethod: core.AuthMethodPrivateKeyJWT}); ref != nil {
|
||||
t.Fatalf("missing key label should not persist key ref, got %#v", ref)
|
||||
}
|
||||
if ref := keyRefFromResult(&configInitResult{AuthMethod: core.AuthMethodClientSecret, KeyLabel: "ignored"}); ref != nil {
|
||||
t.Fatalf("client_secret should not persist key ref, got %#v", ref)
|
||||
}
|
||||
if ref := keyRefFromResult(nil); ref != nil {
|
||||
t.Fatalf("nil result should not persist key ref, got %#v", ref)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSaveInitConfig_PrivateKeyJWTSingleAppPersistsSecretlessAuth(t *testing.T) {
|
||||
t.Setenv("LARKSUITE_CLI_CONFIG_DIR", t.TempDir())
|
||||
f, _, _, _ := cmdutil.TestFactory(t, nil)
|
||||
|
||||
keyRef := &core.SecretRef{Source: "tee", ID: "lark-cli-default"}
|
||||
if err := saveInitConfig("", nil, f, "cli_pkjwt", core.SecretInput{}, core.BrandFeishu, "en_us", core.AuthMethodPrivateKeyJWT, keyRef); err != nil {
|
||||
t.Fatalf("saveInitConfig private_key_jwt single app: %v", err)
|
||||
}
|
||||
|
||||
got, err := core.LoadMultiAppConfig()
|
||||
if err != nil {
|
||||
t.Fatalf("LoadMultiAppConfig: %v", err)
|
||||
}
|
||||
if len(got.Apps) != 1 {
|
||||
t.Fatalf("apps len = %d, want 1", len(got.Apps))
|
||||
}
|
||||
app := got.Apps[0]
|
||||
if app.AppId != "cli_pkjwt" {
|
||||
t.Fatalf("AppId = %q, want cli_pkjwt", app.AppId)
|
||||
}
|
||||
if app.AuthMethod != core.AuthMethodPrivateKeyJWT {
|
||||
t.Fatalf("AuthMethod = %q, want private_key_jwt", app.AuthMethod)
|
||||
}
|
||||
if app.KeyRef == nil || app.KeyRef.Source != "tee" || app.KeyRef.ID != "lark-cli-default" {
|
||||
t.Fatalf("KeyRef = %#v, want tee/lark-cli-default", app.KeyRef)
|
||||
}
|
||||
if app.AppSecret.Ref != nil || app.AppSecret.Plain != "" {
|
||||
t.Fatalf("private_key_jwt config must stay secretless, AppSecret=%#v", app.AppSecret)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSaveInitConfig_PrivateKeyJWTProfilePersistsSecretlessAuth(t *testing.T) {
|
||||
t.Setenv("LARKSUITE_CLI_CONFIG_DIR", t.TempDir())
|
||||
f, _, _, _ := cmdutil.TestFactory(t, nil)
|
||||
|
||||
keyRef := &core.SecretRef{Source: "tee", ID: "lark-cli-default"}
|
||||
if err := saveInitConfig("prod", &core.MultiAppConfig{}, f, "cli_pkjwt", core.SecretInput{}, core.BrandLark, "en_us", core.AuthMethodPrivateKeyJWT, keyRef); err != nil {
|
||||
t.Fatalf("saveInitConfig private_key_jwt profile: %v", err)
|
||||
}
|
||||
|
||||
got, err := core.LoadMultiAppConfig()
|
||||
if err != nil {
|
||||
t.Fatalf("LoadMultiAppConfig: %v", err)
|
||||
}
|
||||
app := got.FindApp("prod")
|
||||
if app == nil {
|
||||
t.Fatalf("profile prod not saved: %#v", got.Apps)
|
||||
}
|
||||
if app.AuthMethod != core.AuthMethodPrivateKeyJWT {
|
||||
t.Fatalf("AuthMethod = %q, want private_key_jwt", app.AuthMethod)
|
||||
}
|
||||
if app.KeyRef == nil || app.KeyRef.Source != "tee" || app.KeyRef.ID != "lark-cli-default" {
|
||||
t.Fatalf("KeyRef = %#v, want tee/lark-cli-default", app.KeyRef)
|
||||
}
|
||||
if app.AppSecret.Ref != nil || app.AppSecret.Plain != "" {
|
||||
t.Fatalf("private_key_jwt profile must stay secretless, AppSecret=%#v", app.AppSecret)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigInitCmd_InvalidLang verifies a non-empty --lang on config init is
|
||||
// strictly validated the same way bind validates: wrong-case / typo / removed
|
||||
// codes / hyphen form all exit with ExitValidation. (Empty is a no-op.)
|
||||
@@ -470,7 +388,7 @@ func TestSaveAsProfile_RejectsProfileNameCollisionWithExistingAppID(t *testing.T
|
||||
},
|
||||
}
|
||||
|
||||
err := saveAsProfile(existing, keychain.KeychainAccess(&noopConfigKeychain{}), "cli_prod", "app-new", core.PlainSecret("new-secret"), core.BrandLark, "en", "", nil)
|
||||
err := saveAsProfile(existing, keychain.KeychainAccess(&noopConfigKeychain{}), "cli_prod", "app-new", core.PlainSecret("new-secret"), core.BrandLark, "en")
|
||||
if err == nil {
|
||||
t.Fatal("expected conflict error")
|
||||
}
|
||||
@@ -509,46 +427,6 @@ func TestWrapSaveConfigError_PassesTypedValidationThrough(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSaveAsProfile_UpdatePersistsPrivateKeyJWT(t *testing.T) {
|
||||
t.Setenv("LARKSUITE_CLI_CONFIG_DIR", t.TempDir())
|
||||
|
||||
existing := &core.MultiAppConfig{
|
||||
Apps: []core.AppConfig{{
|
||||
Name: "prod",
|
||||
AppId: "cli_prod",
|
||||
AppSecret: core.PlainSecret("old-secret"),
|
||||
Brand: core.BrandFeishu,
|
||||
Users: []core.AppUser{{UserOpenId: "ou_1", UserName: "User"}},
|
||||
}},
|
||||
}
|
||||
keyRef := &core.SecretRef{Source: "tee", ID: "lark-cli-default"}
|
||||
|
||||
if err := saveAsProfile(existing, keychain.KeychainAccess(&noopConfigKeychain{}), "prod", "cli_prod", core.SecretInput{}, core.BrandLark, "en_us", core.AuthMethodPrivateKeyJWT, keyRef); err != nil {
|
||||
t.Fatalf("saveAsProfile update private_key_jwt: %v", err)
|
||||
}
|
||||
|
||||
got, err := core.LoadMultiAppConfig()
|
||||
if err != nil {
|
||||
t.Fatalf("LoadMultiAppConfig: %v", err)
|
||||
}
|
||||
app := got.FindApp("prod")
|
||||
if app == nil {
|
||||
t.Fatalf("profile prod not saved: %#v", got.Apps)
|
||||
}
|
||||
if app.AuthMethod != core.AuthMethodPrivateKeyJWT {
|
||||
t.Fatalf("AuthMethod = %q, want private_key_jwt", app.AuthMethod)
|
||||
}
|
||||
if app.KeyRef == nil || app.KeyRef.Source != "tee" || app.KeyRef.ID != "lark-cli-default" {
|
||||
t.Fatalf("KeyRef = %#v, want tee/lark-cli-default", app.KeyRef)
|
||||
}
|
||||
if app.AppSecret.Ref != nil || app.AppSecret.Plain != "" {
|
||||
t.Fatalf("private_key_jwt update must stay secretless, AppSecret=%#v", app.AppSecret)
|
||||
}
|
||||
if len(app.Users) != 1 || app.Users[0].UserOpenId != "ou_1" {
|
||||
t.Fatalf("same-app update should preserve users, Users=%#v", app.Users)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateExistingProfileWithoutSecret_RejectsAppIDChange(t *testing.T) {
|
||||
multi := &core.MultiAppConfig{
|
||||
CurrentApp: "prod",
|
||||
|
||||
@@ -19,7 +19,6 @@ import (
|
||||
"github.com/larksuite/cli/internal/core"
|
||||
"github.com/larksuite/cli/internal/i18n"
|
||||
"github.com/larksuite/cli/internal/keychain"
|
||||
"github.com/larksuite/cli/internal/keysigner"
|
||||
"github.com/larksuite/cli/internal/output"
|
||||
)
|
||||
|
||||
@@ -32,7 +31,6 @@ type ConfigInitOptions struct {
|
||||
AppSecretStdin bool // read app-secret from stdin (avoids process list exposure)
|
||||
Brand string
|
||||
New bool
|
||||
AuthMethod string // --auth-method for --new: "" (default client_secret) | private_key_jwt
|
||||
|
||||
Lang string // raw --lang (string for cobra); normalized to canonical/"" in validateInitLang
|
||||
langExplicit bool // true when --lang was explicitly passed
|
||||
@@ -41,8 +39,6 @@ type ConfigInitOptions struct {
|
||||
|
||||
ProfileName string // when set, create/update a named profile instead of replacing Apps[0]
|
||||
|
||||
Restore bool // Restore re-registers the app already in config to recover a lost credential
|
||||
|
||||
// ForceInit overrides the agent-workspace guard. Without it, running
|
||||
// init under OPENCLAW_HOME / HERMES_HOME refuses and points the caller
|
||||
// at config bind — which is what AI agents almost always want. Manual
|
||||
@@ -85,13 +81,11 @@ if the user explicitly wants a separate app inside the Agent workspace.`,
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&opts.New, "new", false, "create a new app directly (skip mode selection)")
|
||||
cmd.Flags().StringVar(&opts.AuthMethod, "auth-method", "", "auth method for --new: client_secret (default) or private_key_jwt (signed by a platform key, no app secret)")
|
||||
cmd.Flags().StringVar(&opts.AppID, "app-id", "", "App ID (non-interactive)")
|
||||
cmd.Flags().BoolVar(&opts.AppSecretStdin, "app-secret-stdin", false, "Read App Secret from stdin to avoid process list exposure")
|
||||
cmd.Flags().StringVar(&opts.Brand, "brand", "feishu", "feishu or lark (non-interactive, default feishu)")
|
||||
cmd.Flags().StringVar(&opts.Lang, "lang", "", "language preference (e.g. zh or zh_cn)")
|
||||
cmd.Flags().StringVar(&opts.ProfileName, "name", "", "create or update a named profile (append instead of replace)")
|
||||
cmd.Flags().BoolVar(&opts.Restore, "restore", false, "re-register the app already in config to recover a lost credential (keychain key / app secret); reuses the stored app ID and auth method")
|
||||
cmd.Flags().BoolVar(&opts.ForceInit, "force-init", false, "allow init inside an Agent workspace (OPENCLAW_HOME / HERMES_HOME); use config bind instead unless you really want a separate app")
|
||||
cmdutil.SetRisk(cmd, "write")
|
||||
|
||||
@@ -138,7 +132,7 @@ func guardAgentWorkspace(opts *ConfigInitOptions) error {
|
||||
|
||||
// hasAnyNonInteractiveFlag returns true if any non-interactive flag is set.
|
||||
func (o *ConfigInitOptions) hasAnyNonInteractiveFlag() bool {
|
||||
return o.New || o.Restore || o.AppID != "" || o.AppSecretStdin
|
||||
return o.New || o.AppID != "" || o.AppSecretStdin
|
||||
}
|
||||
|
||||
// cleanupOldConfig clears keychain entries (AppSecret + UAT) for all apps in existing config except the app whose AppId equals skipAppID.
|
||||
@@ -157,44 +151,11 @@ func cleanupOldConfig(existing *core.MultiAppConfig, f *cmdutil.Factory, skipApp
|
||||
}
|
||||
}
|
||||
|
||||
// removeStaleSecretForPKJWT clears a secret left in the keychain when the SAME
|
||||
// appId is migrated from client_secret to private_key_jwt. cleanupOldConfig
|
||||
// explicitly skips a matching appId, and saveAsProfile only cleans up on an
|
||||
// appId change, so a same-appId migration would orphan the old secret. This
|
||||
// fills that gap. RemoveSecretStore only deletes Source=="keychain" entries, so
|
||||
// the new pkjwt tee key handle is never touched.
|
||||
func removeStaleSecretForPKJWT(existing *core.MultiAppConfig, profileName, appID string, kc keychain.KeychainAccess) {
|
||||
if existing == nil {
|
||||
return
|
||||
}
|
||||
var prior *core.AppConfig
|
||||
if profileName != "" {
|
||||
if idx := findProfileIndexByName(existing, profileName); idx >= 0 {
|
||||
prior = &existing.Apps[idx]
|
||||
}
|
||||
} else {
|
||||
prior = existing.CurrentAppConfig("")
|
||||
}
|
||||
if prior != nil && prior.AppId == appID && !prior.AppSecret.IsZero() {
|
||||
core.RemoveSecretStore(prior.AppSecret, kc)
|
||||
}
|
||||
}
|
||||
|
||||
// keyRefFromResult builds the TEE key reference to persist for a private_key_jwt
|
||||
// registration result, or nil for client_secret.
|
||||
func keyRefFromResult(r *configInitResult) *core.SecretRef {
|
||||
if r != nil && r.AuthMethod == core.AuthMethodPrivateKeyJWT && r.KeyLabel != "" {
|
||||
return &core.SecretRef{Source: "tee", ID: r.KeyLabel}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// saveAsOnlyApp overwrites config.json with a single-app config.
|
||||
func saveAsOnlyApp(appId string, secret core.SecretInput, brand core.LarkBrand, lang, authMethod string, keyRef *core.SecretRef) error {
|
||||
func saveAsOnlyApp(appId string, secret core.SecretInput, brand core.LarkBrand, lang string) error {
|
||||
config := &core.MultiAppConfig{
|
||||
Apps: []core.AppConfig{{
|
||||
AppId: appId, AppSecret: secret, Brand: brand, Lang: i18n.Lang(lang), Users: []core.AppUser{},
|
||||
AuthMethod: authMethod, KeyRef: keyRef,
|
||||
}},
|
||||
}
|
||||
return core.SaveMultiAppConfig(config)
|
||||
@@ -203,11 +164,9 @@ func saveAsOnlyApp(appId string, secret core.SecretInput, brand core.LarkBrand,
|
||||
// saveInitConfig saves a new/updated app config, respecting --profile mode.
|
||||
// With profileName: appends or updates the named profile (preserves other profiles).
|
||||
// Without profileName: cleans up old config and saves as the only app.
|
||||
// authMethod/keyRef carry the credential type: ("", nil) for client_secret,
|
||||
// (private_key_jwt, &{tee,label}) for the secretless TEE flow.
|
||||
func saveInitConfig(profileName string, existing *core.MultiAppConfig, f *cmdutil.Factory, appId string, secret core.SecretInput, brand core.LarkBrand, lang, authMethod string, keyRef *core.SecretRef) error {
|
||||
func saveInitConfig(profileName string, existing *core.MultiAppConfig, f *cmdutil.Factory, appId string, secret core.SecretInput, brand core.LarkBrand, lang string) error {
|
||||
if profileName != "" {
|
||||
return saveAsProfile(existing, f.Keychain, profileName, appId, secret, brand, lang, authMethod, keyRef)
|
||||
return saveAsProfile(existing, f.Keychain, profileName, appId, secret, brand, lang)
|
||||
}
|
||||
cleanupOldConfig(existing, f, appId)
|
||||
var prior i18n.Lang
|
||||
@@ -216,7 +175,7 @@ func saveInitConfig(profileName string, existing *core.MultiAppConfig, f *cmduti
|
||||
prior = app.Lang
|
||||
}
|
||||
}
|
||||
return saveAsOnlyApp(appId, secret, brand, string(preferredLang(i18n.Lang(lang), prior)), authMethod, keyRef)
|
||||
return saveAsOnlyApp(appId, secret, brand, string(preferredLang(i18n.Lang(lang), prior)))
|
||||
}
|
||||
|
||||
// wrapSaveConfigError passes an already-typed error (e.g. the --name conflict
|
||||
@@ -236,7 +195,7 @@ func wrapSaveConfigError(err error) error {
|
||||
// saveAsProfile appends or updates a named profile in the config.
|
||||
// If a profile with the same name exists, it updates it; otherwise appends.
|
||||
// When updating, cleans up old keychain secrets if AppId changed.
|
||||
func saveAsProfile(existing *core.MultiAppConfig, kc keychain.KeychainAccess, profileName, appId string, secret core.SecretInput, brand core.LarkBrand, lang, authMethod string, keyRef *core.SecretRef) error {
|
||||
func saveAsProfile(existing *core.MultiAppConfig, kc keychain.KeychainAccess, profileName, appId string, secret core.SecretInput, brand core.LarkBrand, lang string) error {
|
||||
multi := existing
|
||||
if multi == nil {
|
||||
multi = &core.MultiAppConfig{}
|
||||
@@ -255,8 +214,6 @@ func saveAsProfile(existing *core.MultiAppConfig, kc keychain.KeychainAccess, pr
|
||||
multi.Apps[idx].AppSecret = secret
|
||||
multi.Apps[idx].Brand = brand
|
||||
multi.Apps[idx].Lang = preferredLang(i18n.Lang(lang), multi.Apps[idx].Lang)
|
||||
multi.Apps[idx].AuthMethod = authMethod
|
||||
multi.Apps[idx].KeyRef = keyRef
|
||||
} else {
|
||||
if findAppIndexByAppID(multi, profileName) >= 0 {
|
||||
return errs.NewValidationError(errs.SubtypeInvalidArgument,
|
||||
@@ -265,14 +222,12 @@ func saveAsProfile(existing *core.MultiAppConfig, kc keychain.KeychainAccess, pr
|
||||
}
|
||||
// Append new profile
|
||||
multi.Apps = append(multi.Apps, core.AppConfig{
|
||||
Name: profileName,
|
||||
AppId: appId,
|
||||
AppSecret: secret,
|
||||
Brand: brand,
|
||||
Lang: i18n.Lang(lang),
|
||||
Users: []core.AppUser{},
|
||||
AuthMethod: authMethod,
|
||||
KeyRef: keyRef,
|
||||
Name: profileName,
|
||||
AppId: appId,
|
||||
AppSecret: secret,
|
||||
Brand: brand,
|
||||
Lang: i18n.Lang(lang),
|
||||
Users: []core.AppUser{},
|
||||
})
|
||||
}
|
||||
return core.SaveMultiAppConfig(multi)
|
||||
@@ -350,94 +305,6 @@ func updateExistingProfileWithoutSecret(existing *core.MultiAppConfig, profileNa
|
||||
return core.SaveMultiAppConfig(existing)
|
||||
}
|
||||
|
||||
// persistAndProbeResult saves a registration/restore result into profileName and
|
||||
// runs the post-registration probe. profileName == "" replaces the single app
|
||||
// (legacy); a named profile is updated in place. Shared by --new and --restore.
|
||||
func persistAndProbeResult(opts *ConfigInitOptions, f *cmdutil.Factory, profileName string, result *configInitResult) error {
|
||||
existing, _ := core.LoadMultiAppConfig()
|
||||
|
||||
// private_key_jwt apps have no secret: persist auth method + TEE key ref.
|
||||
// Registration success already validated the key (server bound the public
|
||||
// key), so the app_secret probe is skipped.
|
||||
if result.AuthMethod == core.AuthMethodPrivateKeyJWT {
|
||||
if err := saveInitConfig(profileName, existing, f, result.AppID, core.SecretInput{}, result.Brand, opts.Lang, result.AuthMethod, keyRefFromResult(result)); err != nil {
|
||||
return wrapSaveConfigError(err)
|
||||
}
|
||||
removeStaleSecretForPKJWT(existing, profileName, result.AppID, f.Keychain)
|
||||
printLangPreferenceConfirmation(opts)
|
||||
output.PrintJson(f.IOStreams.Out, map[string]interface{}{"appId": result.AppID, "authMethod": result.AuthMethod, "brand": result.Brand})
|
||||
return runProbePKJWT(opts.Ctx, f, result.Brand, result.AppID, keysigner.Active(), result.KeyLabel)
|
||||
}
|
||||
|
||||
secret, err := core.ForStorage(result.AppID, core.PlainSecret(result.AppSecret), f.Keychain)
|
||||
if err != nil {
|
||||
return errs.NewInternalError(errs.SubtypeSDKError, "%v", err).WithCause(err)
|
||||
}
|
||||
if err := saveInitConfig(profileName, existing, f, result.AppID, secret, result.Brand, opts.Lang, "", nil); err != nil {
|
||||
return wrapSaveConfigError(err)
|
||||
}
|
||||
printLangPreferenceConfirmation(opts)
|
||||
output.PrintJson(f.IOStreams.Out, map[string]interface{}{"appId": result.AppID, "appSecret": "****", "brand": result.Brand})
|
||||
return runProbe(opts.Ctx, f, result.AppID, result.AppSecret, result.Brand)
|
||||
}
|
||||
|
||||
// runRestoreFlow re-registers the app already in config to recover a lost
|
||||
// credential (deleted keychain key / lost app secret). It reads the existing
|
||||
// app id + auth method + brand from config (no secret needed — that's the lost
|
||||
// part) and re-runs the device-flow registration with the app id sent on begin,
|
||||
// so the server re-registers that app instead of creating a new one. The
|
||||
// re-issued credential is written back to the same profile.
|
||||
func runRestoreFlow(opts *ConfigInitOptions, existing *core.MultiAppConfig, f *cmdutil.Factory, msg *initMsg) error {
|
||||
if existing == nil {
|
||||
return errs.NewConfigError(errs.SubtypeNotConfigured, "nothing to restore: no config found").
|
||||
WithHint("run: lark-cli config init")
|
||||
}
|
||||
app := existing.CurrentAppConfig(opts.ProfileName)
|
||||
if app == nil || app.AppId == "" {
|
||||
return errs.NewConfigError(errs.SubtypeNotConfigured, "nothing to restore: no app id in config%s", profileSuffix(opts.ProfileName)).
|
||||
WithHint("run: lark-cli config init")
|
||||
}
|
||||
|
||||
restoreAppID := app.AppId
|
||||
// Reuse the stored auth method authoritatively — never prompt. Empty on disk
|
||||
// means client_secret (omitempty back-compat); pass it explicitly so
|
||||
// resolveRegisterAuthMethod doesn't fall through to the interactive picker.
|
||||
authMethod := app.AuthMethod
|
||||
if authMethod == "" {
|
||||
authMethod = core.AuthMethodClientSecret
|
||||
}
|
||||
result, err := runCreateAppFlow(opts.Ctx, f, app.Brand, authMethod, msg, restoreAppID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if result == nil {
|
||||
return errs.NewInternalError(errs.SubtypeSDKError, "app restore returned no result")
|
||||
}
|
||||
|
||||
// Safety: if the server did not honor app_id (e.g. not yet supported), it may
|
||||
// have created a NEW app instead of restoring. Warn so the user is not silently
|
||||
// switched to a different app id.
|
||||
if result.AppID != restoreAppID {
|
||||
fmt.Fprintf(f.IOStreams.ErrOut, "[lark-cli] [WARN] restore: server returned app %s, expected %s — it may have created a new app instead of restoring\n", result.AppID, restoreAppID)
|
||||
}
|
||||
|
||||
// Write back to the profile we restored: an explicit --name, else the resolved
|
||||
// app's own name. Empty name => legacy single-app replace.
|
||||
saveProfile := opts.ProfileName
|
||||
if saveProfile == "" {
|
||||
saveProfile = app.Name
|
||||
}
|
||||
return persistAndProbeResult(opts, f, saveProfile, result)
|
||||
}
|
||||
|
||||
// profileSuffix renders " (profile %q)" for error messages, or "" when unnamed.
|
||||
func profileSuffix(profileName string) string {
|
||||
if profileName == "" {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf(" (profile %q)", profileName)
|
||||
}
|
||||
|
||||
func configInitRun(opts *ConfigInitOptions) error {
|
||||
f := opts.Factory
|
||||
|
||||
@@ -468,17 +335,6 @@ func configInitRun(opts *ConfigInitOptions) error {
|
||||
}
|
||||
}
|
||||
|
||||
// --restore recovers an existing app; it is incompatible with creating a new
|
||||
// app (--new) or importing one non-interactively (--app-id / stdin secret).
|
||||
if opts.Restore {
|
||||
if opts.New {
|
||||
return errs.NewValidationError(errs.SubtypeInvalidArgument, "--restore cannot be combined with --new").WithParam("--restore")
|
||||
}
|
||||
if opts.AppID != "" || opts.AppSecretStdin {
|
||||
return errs.NewValidationError(errs.SubtypeInvalidArgument, "--restore cannot be combined with --app-id / --app-secret-stdin").WithParam("--restore")
|
||||
}
|
||||
}
|
||||
|
||||
// Mode 1: Non-interactive
|
||||
if opts.AppID != "" && opts.appSecret != "" {
|
||||
brand := parseBrand(opts.Brand)
|
||||
@@ -486,7 +342,7 @@ func configInitRun(opts *ConfigInitOptions) error {
|
||||
if err != nil {
|
||||
return errs.NewInternalError(errs.SubtypeSDKError, "%v", err).WithCause(err)
|
||||
}
|
||||
if err := saveInitConfig(opts.ProfileName, existing, f, opts.AppID, secret, brand, opts.Lang, "", nil); err != nil {
|
||||
if err := saveInitConfig(opts.ProfileName, existing, f, opts.AppID, secret, brand, opts.Lang); err != nil {
|
||||
return wrapSaveConfigError(err)
|
||||
}
|
||||
output.PrintSuccess(f.IOStreams.ErrOut, fmt.Sprintf("Configuration saved to %s", core.GetConfigPath()))
|
||||
@@ -512,26 +368,34 @@ func configInitRun(opts *ConfigInitOptions) error {
|
||||
|
||||
msg := getInitMsg(opts.UILang)
|
||||
|
||||
// Mode: Restore (--restore) — re-register the app already in config.
|
||||
if opts.Restore {
|
||||
return runRestoreFlow(opts, existing, f, msg)
|
||||
}
|
||||
|
||||
// Mode 3: Create new app directly (--new)
|
||||
if opts.New {
|
||||
result, err := runCreateAppFlow(opts.Ctx, f, parseBrand(opts.Brand), opts.AuthMethod, msg, "")
|
||||
result, err := runCreateAppFlow(opts.Ctx, f, parseBrand(opts.Brand), msg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if result == nil {
|
||||
return errs.NewInternalError(errs.SubtypeSDKError, "app creation returned no result")
|
||||
}
|
||||
return persistAndProbeResult(opts, f, opts.ProfileName, result)
|
||||
existing, _ := core.LoadMultiAppConfig()
|
||||
secret, err := core.ForStorage(result.AppID, core.PlainSecret(result.AppSecret), f.Keychain)
|
||||
if err != nil {
|
||||
return errs.NewInternalError(errs.SubtypeSDKError, "%v", err).WithCause(err)
|
||||
}
|
||||
if err := saveInitConfig(opts.ProfileName, existing, f, result.AppID, secret, result.Brand, opts.Lang); err != nil {
|
||||
return wrapSaveConfigError(err)
|
||||
}
|
||||
printLangPreferenceConfirmation(opts)
|
||||
output.PrintJson(f.IOStreams.Out, map[string]interface{}{"appId": result.AppID, "appSecret": "****", "brand": result.Brand})
|
||||
if err := runProbe(opts.Ctx, f, result.AppID, result.AppSecret, result.Brand); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Mode 4: Interactive TUI (terminal)
|
||||
if !opts.hasAnyNonInteractiveFlag() && f.IOStreams.IsTerminal {
|
||||
result, err := runInteractiveConfigInit(opts.Ctx, f, opts.AuthMethod, msg)
|
||||
result, err := runInteractiveConfigInit(opts.Ctx, f, msg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -542,22 +406,13 @@ func configInitRun(opts *ConfigInitOptions) error {
|
||||
|
||||
existing, _ := core.LoadMultiAppConfig()
|
||||
|
||||
if result.AuthMethod == core.AuthMethodPrivateKeyJWT {
|
||||
// Secretless create: persist auth method + TEE key ref, no secret.
|
||||
if err := saveInitConfig(opts.ProfileName, existing, f, result.AppID, core.SecretInput{}, result.Brand, opts.Lang, result.AuthMethod, keyRefFromResult(result)); err != nil {
|
||||
return wrapSaveConfigError(err)
|
||||
}
|
||||
removeStaleSecretForPKJWT(existing, opts.ProfileName, result.AppID, f.Keychain)
|
||||
if err := runProbePKJWT(opts.Ctx, f, result.Brand, result.AppID, keysigner.Active(), result.KeyLabel); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if result.AppSecret != "" {
|
||||
if result.AppSecret != "" {
|
||||
// New secret provided (either from "create" or "existing" with input)
|
||||
secret, err := core.ForStorage(result.AppID, core.PlainSecret(result.AppSecret), f.Keychain)
|
||||
if err != nil {
|
||||
return errs.NewInternalError(errs.SubtypeSDKError, "%v", err).WithCause(err)
|
||||
}
|
||||
if err := saveInitConfig(opts.ProfileName, existing, f, result.AppID, secret, result.Brand, opts.Lang, "", nil); err != nil {
|
||||
if err := saveInitConfig(opts.ProfileName, existing, f, result.AppID, secret, result.Brand, opts.Lang); err != nil {
|
||||
return wrapSaveConfigError(err)
|
||||
}
|
||||
} else if result.Mode == "existing" && result.AppID != "" {
|
||||
@@ -662,7 +517,7 @@ func configInitRun(opts *ConfigInitOptions) error {
|
||||
if err != nil {
|
||||
return errs.NewInternalError(errs.SubtypeSDKError, "%v", err).WithCause(err)
|
||||
}
|
||||
if err := saveInitConfig(opts.ProfileName, existing, f, resolvedAppId, storedSecret, parseBrand(resolvedBrand), opts.Lang, "", nil); err != nil {
|
||||
if err := saveInitConfig(opts.ProfileName, existing, f, resolvedAppId, storedSecret, parseBrand(resolvedBrand), opts.Lang); err != nil {
|
||||
return wrapSaveConfigError(err)
|
||||
}
|
||||
output.PrintSuccess(f.IOStreams.ErrOut, fmt.Sprintf("Configuration saved to %s", core.GetConfigPath()))
|
||||
|
||||
@@ -1,102 +0,0 @@
|
||||
// Copyright (c) 2026 Lark Technologies Pte. Ltd.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto"
|
||||
"testing"
|
||||
|
||||
"github.com/larksuite/cli/internal/cmdutil"
|
||||
"github.com/larksuite/cli/internal/core"
|
||||
"github.com/larksuite/cli/internal/keysigner"
|
||||
)
|
||||
|
||||
type authMethodTestSigner struct{}
|
||||
|
||||
func (authMethodTestSigner) EnsureKey(context.Context, keysigner.KeyRef) (crypto.PublicKey, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (authMethodTestSigner) PublicKey(context.Context, keysigner.KeyRef) (crypto.PublicKey, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (authMethodTestSigner) Sign(context.Context, keysigner.KeyRef, []byte) ([]byte, string, error) {
|
||||
return nil, "", nil
|
||||
}
|
||||
|
||||
// TestResolveRegisterAuthMethod covers the non-interactive gating paths. The
|
||||
// darwin keychain signer is compiled into every build, so the test cannot rely
|
||||
// on the binary lacking a signer — it forces a known no-signer state for the
|
||||
// rejection cases, then registers a stub for the success case.
|
||||
func TestResolveRegisterAuthMethod(t *testing.T) {
|
||||
f := &cmdutil.Factory{}
|
||||
|
||||
prevSigner := keysigner.Active()
|
||||
t.Cleanup(func() { keysigner.Register(prevSigner) })
|
||||
keysigner.Register(nil)
|
||||
|
||||
if m, err := resolveRegisterAuthMethod(f, core.AuthMethodClientSecret); err != nil || m != core.AuthMethodClientSecret {
|
||||
t.Errorf("client_secret: got (%q, %v), want (client_secret, nil)", m, err)
|
||||
}
|
||||
|
||||
if m, err := resolveRegisterAuthMethod(f, ""); err != nil || m != core.AuthMethodClientSecret {
|
||||
t.Errorf("default: got (%q, %v), want (client_secret, nil)", m, err)
|
||||
}
|
||||
|
||||
if _, err := resolveRegisterAuthMethod(f, "bogus"); err == nil {
|
||||
t.Error("bogus auth-method: expected error")
|
||||
}
|
||||
|
||||
if _, err := resolveRegisterAuthMethod(f, core.AuthMethodPrivateKeyJWT); err == nil {
|
||||
t.Error("private_key_jwt without a signer: expected error")
|
||||
}
|
||||
|
||||
keysigner.Register(authMethodTestSigner{})
|
||||
|
||||
if m, err := resolveRegisterAuthMethod(f, core.AuthMethodPrivateKeyJWT); err != nil || m != core.AuthMethodPrivateKeyJWT {
|
||||
t.Errorf("private_key_jwt with signer: got (%q, %v), want (private_key_jwt, nil)", m, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestValidatePKJWTKeyBinding covers the guard that rejects a registration
|
||||
// resolving to private_key_jwt with no signing key bound (e.g. an existing
|
||||
// secret-based app was selected on the confirm page).
|
||||
func TestValidatePKJWTKeyBinding(t *testing.T) {
|
||||
if err := validatePKJWTKeyBinding(core.AuthMethodPrivateKeyJWT, ""); err == nil {
|
||||
t.Error("pkjwt with empty keyLabel: expected error")
|
||||
}
|
||||
if err := validatePKJWTKeyBinding(core.AuthMethodPrivateKeyJWT, "agent-key"); err != nil {
|
||||
t.Errorf("pkjwt with keyLabel: expected nil, got %v", err)
|
||||
}
|
||||
if err := validatePKJWTKeyBinding(core.AuthMethodClientSecret, ""); err != nil {
|
||||
t.Errorf("client_secret: expected nil, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveFinalAuthMethod locks the authoritative-method logic. The 2nd case
|
||||
// is the real bug: we requested private_key_jwt but the server resolved to an
|
||||
// existing client_secret app — we must persist client_secret, not pkjwt.
|
||||
func TestResolveFinalAuthMethod(t *testing.T) {
|
||||
if m := resolveFinalAuthMethod([]string{"client_secret", "private_key_jwt"}, core.AuthMethodClientSecret); m != core.AuthMethodPrivateKeyJWT {
|
||||
t.Errorf("prefers private_key_jwt: got %q", m)
|
||||
}
|
||||
if m := resolveFinalAuthMethod([]string{"client_secret"}, core.AuthMethodPrivateKeyJWT); m != core.AuthMethodClientSecret {
|
||||
t.Errorf("server client_secret must override requested pkjwt: got %q", m)
|
||||
}
|
||||
if m := resolveFinalAuthMethod(nil, core.AuthMethodPrivateKeyJWT); m != core.AuthMethodPrivateKeyJWT {
|
||||
t.Errorf("fallback to requested when server is silent: got %q", m)
|
||||
}
|
||||
// Explicit empty slice (not just nil) also falls back to requested — the same
|
||||
// len()==0 back-compat allowance the init guard relies on to let private_key_jwt
|
||||
// proceed against an older server (see internal/auth
|
||||
// TestRequestAppRegistrationInit_EmptySupportedAuthMethods).
|
||||
if m := resolveFinalAuthMethod([]string{}, core.AuthMethodPrivateKeyJWT); m != core.AuthMethodPrivateKeyJWT {
|
||||
t.Errorf("empty []string should fall back to requested private_key_jwt: got %q", m)
|
||||
}
|
||||
if m := resolveFinalAuthMethod(nil, ""); m != core.AuthMethodClientSecret {
|
||||
t.Errorf("default to client_secret: got %q", m)
|
||||
}
|
||||
}
|
||||
@@ -5,11 +5,7 @@ package config
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/charmbracelet/huh"
|
||||
"github.com/larksuite/cli/internal/build"
|
||||
@@ -17,26 +13,22 @@ import (
|
||||
|
||||
"github.com/larksuite/cli/errs"
|
||||
larkauth "github.com/larksuite/cli/internal/auth"
|
||||
"github.com/larksuite/cli/internal/auth/jwt"
|
||||
"github.com/larksuite/cli/internal/cmdutil"
|
||||
"github.com/larksuite/cli/internal/core"
|
||||
"github.com/larksuite/cli/internal/keysigner"
|
||||
"github.com/larksuite/cli/internal/output"
|
||||
"github.com/larksuite/cli/internal/transport"
|
||||
)
|
||||
|
||||
// configInitResult holds the result of the interactive config init flow.
|
||||
type configInitResult struct {
|
||||
Mode string // "create" or "existing"
|
||||
Brand core.LarkBrand
|
||||
AppID string
|
||||
AppSecret string
|
||||
AuthMethod string // "" == client_secret; core.AuthMethodPrivateKeyJWT
|
||||
KeyLabel string // TEE key handle when AuthMethod == private_key_jwt
|
||||
Mode string // "create" or "existing"
|
||||
Brand core.LarkBrand
|
||||
AppID string
|
||||
AppSecret string
|
||||
}
|
||||
|
||||
// runInteractiveConfigInit shows an interactive TUI for config init.
|
||||
func runInteractiveConfigInit(ctx context.Context, f *cmdutil.Factory, authMethodFlag string, msg *initMsg) (*configInitResult, error) {
|
||||
func runInteractiveConfigInit(ctx context.Context, f *cmdutil.Factory, msg *initMsg) (*configInitResult, error) {
|
||||
// Phase 1: Choose mode
|
||||
var mode string
|
||||
form1 := huh.NewForm(
|
||||
@@ -62,7 +54,7 @@ func runInteractiveConfigInit(ctx context.Context, f *cmdutil.Factory, authMetho
|
||||
return runExistingAppForm(f, msg)
|
||||
}
|
||||
|
||||
return runCreateAppFlow(ctx, f, "", authMethodFlag, msg, "")
|
||||
return runCreateAppFlow(ctx, f, "", msg)
|
||||
}
|
||||
|
||||
// runExistingAppForm shows a huh form for manually entering App ID / App Secret / Brand.
|
||||
@@ -154,59 +146,9 @@ func runExistingAppForm(f *cmdutil.Factory, msg *initMsg) (*configInitResult, er
|
||||
}, nil
|
||||
}
|
||||
|
||||
// resolveRegisterAuthMethod decides the auth method for a new-app registration.
|
||||
// An explicit --auth-method flag wins; otherwise, on an interactive terminal with
|
||||
// a TEE signer available, the user is prompted; the default is client_secret.
|
||||
func resolveRegisterAuthMethod(f *cmdutil.Factory, flag string) (string, error) {
|
||||
signerAvailable := keysigner.Active() != nil
|
||||
switch flag {
|
||||
case core.AuthMethodPrivateKeyJWT:
|
||||
if !signerAvailable {
|
||||
return "", errs.NewConfigError(errs.SubtypeInvalidClient,
|
||||
"--auth-method private_key_jwt requires a platform key signer, which is unavailable on this device/build").
|
||||
WithHint("omit --auth-method (or pass --auth-method client_secret) to register with an app secret")
|
||||
}
|
||||
return core.AuthMethodPrivateKeyJWT, nil
|
||||
case core.AuthMethodClientSecret:
|
||||
return core.AuthMethodClientSecret, nil
|
||||
case "":
|
||||
// fall through to interactive / default
|
||||
default:
|
||||
return "", errs.NewValidationError(errs.SubtypeInvalidArgument,
|
||||
"unknown --auth-method %q (use client_secret or private_key_jwt)", flag)
|
||||
}
|
||||
|
||||
if signerAvailable && f.IOStreams.IsTerminal {
|
||||
var choice string
|
||||
form := huh.NewForm(
|
||||
huh.NewGroup(
|
||||
huh.NewSelect[string]().
|
||||
Title("Authentication method").
|
||||
Options(
|
||||
huh.NewOption("App Secret (client_secret)", core.AuthMethodClientSecret),
|
||||
huh.NewOption("Secure key signer, no secret (private_key_jwt)", core.AuthMethodPrivateKeyJWT),
|
||||
).
|
||||
Value(&choice),
|
||||
),
|
||||
).WithTheme(cmdutil.ThemeFeishu())
|
||||
if err := form.Run(); err != nil {
|
||||
if errors.Is(err, huh.ErrUserAborted) {
|
||||
return "", output.ErrBare(1)
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
return choice, nil
|
||||
}
|
||||
return core.AuthMethodClientSecret, nil
|
||||
}
|
||||
|
||||
// runCreateAppFlow runs the "create new app" flow via OpenClaw device flow.
|
||||
// If brandOverride is non-empty, skip the interactive brand selection.
|
||||
// authMethodFlag is the raw --auth-method value ("" when unset).
|
||||
// restoreAppID, when non-empty, is sent on the registration begin request so the
|
||||
// server re-registers that existing app (credential recovery) instead of creating
|
||||
// a new one. Empty preserves the normal new-app flow.
|
||||
func runCreateAppFlow(ctx context.Context, f *cmdutil.Factory, brandOverride core.LarkBrand, authMethodFlag string, msg *initMsg, restoreAppID string) (*configInitResult, error) {
|
||||
func runCreateAppFlow(ctx context.Context, f *cmdutil.Factory, brandOverride core.LarkBrand, msg *initMsg) (*configInitResult, error) {
|
||||
var larkBrand core.LarkBrand
|
||||
if brandOverride != "" {
|
||||
larkBrand = brandOverride
|
||||
@@ -234,51 +176,11 @@ func runCreateAppFlow(ctx context.Context, f *cmdutil.Factory, brandOverride cor
|
||||
larkBrand = parseBrand(brand)
|
||||
}
|
||||
|
||||
authMethod, err := resolveRegisterAuthMethod(f, authMethodFlag)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Step 1: Request app registration (begin).
|
||||
// Step 1: Request app registration (begin)
|
||||
// Use the shared proxy-plugin-aware transport so registration traffic is not
|
||||
// a bypass of proxy plugin mode.
|
||||
httpClient := transport.NewHTTPClient(0)
|
||||
|
||||
// For private_key_jwt: init to obtain a nonce, then sign a TEE attestation
|
||||
// (carrying the public key in its jwk header) to send with begin.
|
||||
beginOpts := larkauth.AppRegistrationBeginOptions{}
|
||||
keyLabel := ""
|
||||
if authMethod == core.AuthMethodPrivateKeyJWT {
|
||||
signer := keysigner.Active() // non-nil, guaranteed by resolveRegisterAuthMethod
|
||||
initResp, initErr := larkauth.RequestAppRegistrationInit(httpClient)
|
||||
if initErr != nil {
|
||||
return nil, errs.NewConfigError(errs.SubtypeInvalidClient, "app registration init failed: %v", initErr).WithCause(initErr)
|
||||
}
|
||||
// An empty SupportedAuthMethods is intentionally treated as "older server /
|
||||
// unknown": len()==0 makes this guard false, so the requested
|
||||
// private_key_jwt proceeds. This mirrors resolveFinalAuthMethod's
|
||||
// back-compat fallback to the requested method. Only an explicit list that
|
||||
// omits private_key_jwt rejects here.
|
||||
if len(initResp.SupportedAuthMethods) > 0 && !slices.Contains(initResp.SupportedAuthMethods, core.AuthMethodPrivateKeyJWT) {
|
||||
return nil, errs.NewConfigError(errs.SubtypeInvalidClient,
|
||||
"server does not support private_key_jwt for this app type (supported: %s)", strings.Join(initResp.SupportedAuthMethods, ", ")).
|
||||
WithHint("register with --auth-method client_secret instead")
|
||||
}
|
||||
keyLabel = keysigner.DefaultKeyLabel
|
||||
attestation, signErr := jwt.SignAttestation(ctx, signer, keysigner.KeyRef{Label: keyLabel}, initResp.Nonce, time.Now())
|
||||
if signErr != nil {
|
||||
return nil, errs.NewConfigError(errs.SubtypeInvalidClient, "failed to sign registration attestation: %v", signErr).WithCause(signErr)
|
||||
}
|
||||
beginOpts = larkauth.AppRegistrationBeginOptions{
|
||||
AuthMethod: core.AuthMethodPrivateKeyJWT,
|
||||
AuthAttestation: attestation,
|
||||
}
|
||||
}
|
||||
|
||||
// Restore flow: re-register the existing app instead of creating a new one.
|
||||
beginOpts.RestoreAppID = restoreAppID
|
||||
|
||||
authResp, err := larkauth.RequestAppRegistration(httpClient, larkBrand, beginOpts, f.IOStreams.ErrOut)
|
||||
authResp, err := larkauth.RequestAppRegistration(httpClient, larkBrand, f.IOStreams.ErrOut)
|
||||
if err != nil {
|
||||
return nil, errs.NewConfigError(errs.SubtypeInvalidClient, "app registration failed: %v", err).WithCause(err)
|
||||
}
|
||||
@@ -311,28 +213,18 @@ func runCreateAppFlow(ctx context.Context, f *cmdutil.Factory, brandOverride cor
|
||||
return nil, errs.NewAuthenticationError(errs.SubtypeUnknown, "%v", err).WithCause(err)
|
||||
}
|
||||
|
||||
// The final auth method is decided by the user/admin at confirmation and
|
||||
// returned by poll — NOT necessarily what we requested. Selecting an existing
|
||||
// client_secret app, for example, yields client_secret even though we sent
|
||||
// private_key_jwt. Trust the result so we persist the truth.
|
||||
finalMethod := resolveFinalAuthMethod(result.AuthMethods, authMethod)
|
||||
|
||||
// Lark brand special case (client_secret only): a lark-tenant app returns its
|
||||
// secret only from the lark endpoint. private_key_jwt returns no secret, so
|
||||
// this retry does not apply.
|
||||
if finalMethod != core.AuthMethodPrivateKeyJWT && result.ClientSecret == "" && result.UserInfo != nil && result.UserInfo.TenantBrand == "lark" {
|
||||
// Step 4: Handle Lark brand special case
|
||||
// If tenant_brand=lark and no client_secret, retry with lark brand endpoint
|
||||
if result.ClientSecret == "" && result.UserInfo != nil && result.UserInfo.TenantBrand == "lark" {
|
||||
// fmt.Fprintf(f.IOStreams.ErrOut, "%s\n", msg.DetectedLarkTenant)
|
||||
result, err = larkauth.PollAppRegistration(ctx, httpClient, core.BrandLark, authResp.DeviceCode, authResp.Interval, authResp.ExpiresIn, f.IOStreams.ErrOut)
|
||||
if err != nil {
|
||||
return nil, errs.NewNetworkError(errs.SubtypeNetworkTransport, "lark endpoint retry failed: %v", err).WithCause(err)
|
||||
}
|
||||
finalMethod = resolveFinalAuthMethod(result.AuthMethods, authMethod)
|
||||
}
|
||||
|
||||
if result.ClientID == "" {
|
||||
return nil, errs.NewConfigError(errs.SubtypeInvalidClient, "app registration succeeded but missing client_id")
|
||||
}
|
||||
if finalMethod != core.AuthMethodPrivateKeyJWT && result.ClientSecret == "" {
|
||||
return nil, errs.NewConfigError(errs.SubtypeInvalidClient, "app registration succeeded but missing client_secret")
|
||||
if result.ClientID == "" || result.ClientSecret == "" {
|
||||
return nil, errs.NewConfigError(errs.SubtypeInvalidClient, "app registration succeeded but missing client_id or client_secret")
|
||||
}
|
||||
|
||||
// Determine final brand from response
|
||||
@@ -343,67 +235,13 @@ func runCreateAppFlow(ctx context.Context, f *cmdutil.Factory, brandOverride cor
|
||||
finalBrand = core.BrandFeishu
|
||||
}
|
||||
|
||||
// Surface a downgrade: requested private_key_jwt but the app resolved to a
|
||||
// secret-based method (e.g. an existing app was selected). The key was NOT
|
||||
// bound, so we must store the secret method, not private_key_jwt.
|
||||
if authMethod == core.AuthMethodPrivateKeyJWT && finalMethod != core.AuthMethodPrivateKeyJWT {
|
||||
fmt.Fprintf(f.IOStreams.ErrOut, "[lark-cli] note: requested private_key_jwt, but the app uses %q (e.g. an existing app was selected); storing %q.\n", finalMethod, finalMethod)
|
||||
}
|
||||
|
||||
fmt.Fprintln(f.IOStreams.ErrOut)
|
||||
output.PrintSuccess(f.IOStreams.ErrOut, fmt.Sprintf(msg.AppCreated, result.ClientID))
|
||||
|
||||
keyToStore := ""
|
||||
if finalMethod == core.AuthMethodPrivateKeyJWT {
|
||||
keyToStore = keyLabel
|
||||
}
|
||||
if err := validatePKJWTKeyBinding(finalMethod, keyToStore); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &configInitResult{
|
||||
Mode: "create",
|
||||
Brand: finalBrand,
|
||||
AppID: result.ClientID,
|
||||
AppSecret: result.ClientSecret, // empty for private_key_jwt; real secret otherwise
|
||||
AuthMethod: finalMethod,
|
||||
KeyLabel: keyToStore,
|
||||
Mode: "create",
|
||||
Brand: finalBrand,
|
||||
AppID: result.ClientID,
|
||||
AppSecret: result.ClientSecret,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// validatePKJWTKeyBinding rejects a registration that resolved to
|
||||
// private_key_jwt without a signing key bound to it. keyLabel is non-empty only
|
||||
// when the local flow chose private_key_jwt and signed a TEE attestation; a
|
||||
// resolved method of private_key_jwt with no key handle would save an unusable
|
||||
// config (rejected later at config load, surfacing as "saved OK, fails on first
|
||||
// use"), so it is caught here at registration time instead.
|
||||
func validatePKJWTKeyBinding(finalMethod, keyLabel string) error {
|
||||
if finalMethod == core.AuthMethodPrivateKeyJWT && keyLabel == "" {
|
||||
return errs.NewConfigError(errs.SubtypeInvalidClient,
|
||||
"registration resolved to private_key_jwt but no signing key was bound to this app (an existing secret-based app may have been selected)").
|
||||
WithHint("re-register with: lark-cli config init --new --auth-method private_key_jwt")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// resolveFinalAuthMethod picks the authoritative method from the poll result,
|
||||
// preferring private_key_jwt, then client_secret. It falls back to the requested
|
||||
// method when the server returns nothing (older servers).
|
||||
func resolveFinalAuthMethod(serverMethods []string, requested string) string {
|
||||
if len(serverMethods) == 0 {
|
||||
if requested == "" {
|
||||
return core.AuthMethodClientSecret
|
||||
}
|
||||
return requested
|
||||
}
|
||||
for _, m := range serverMethods {
|
||||
if m == core.AuthMethodPrivateKeyJWT {
|
||||
return core.AuthMethodPrivateKeyJWT
|
||||
}
|
||||
}
|
||||
for _, m := range serverMethods {
|
||||
if m == core.AuthMethodClientSecret {
|
||||
return core.AuthMethodClientSecret
|
||||
}
|
||||
}
|
||||
return serverMethods[0]
|
||||
}
|
||||
|
||||
@@ -16,7 +16,6 @@ import (
|
||||
"github.com/larksuite/cli/internal/cmdutil"
|
||||
"github.com/larksuite/cli/internal/core"
|
||||
"github.com/larksuite/cli/internal/credential"
|
||||
"github.com/larksuite/cli/internal/keysigner"
|
||||
)
|
||||
|
||||
// probeTimeout is the total wall-clock budget for the credential probe step
|
||||
@@ -91,32 +90,3 @@ func runProbe(parent context.Context, factory *cmdutil.Factory, appID, appSecret
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
return nil
|
||||
}
|
||||
|
||||
// runProbePKJWT does a best-effort key-binding validation after a private_key_jwt
|
||||
// config is saved: it signs a client_assertion with the local platform key and
|
||||
// mints a token. A typed error (a deterministic server rejection — e.g. the key
|
||||
// is not bound to this app) is propagated so `config init` exits non-zero with
|
||||
// the canonical envelope; untyped errors (transport / HTTP / parse / timeout)
|
||||
// are swallowed (return nil). The mint itself is the probe — no second call.
|
||||
func runProbePKJWT(parent context.Context, factory *cmdutil.Factory, brand core.LarkBrand, clientID string, signer keysigner.Signer, keyLabel string) error {
|
||||
if factory == nil || signer == nil {
|
||||
return nil
|
||||
}
|
||||
httpClient, err := factory.HttpClient()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(parent, probeTimeout)
|
||||
defer cancel()
|
||||
|
||||
if _, err := credential.FetchTATWithAssertion(ctx, httpClient, brand, clientID, signer, keyLabel); err != nil {
|
||||
// Typed = deterministic credential rejection → propagate. Untyped
|
||||
// (transport / HTTP / parse / timeout) is ambiguous → stay silent.
|
||||
if errs.IsTyped(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -6,11 +6,6 @@ package config
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
crand "crypto/rand"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -22,17 +17,14 @@ import (
|
||||
"github.com/larksuite/cli/internal/build"
|
||||
"github.com/larksuite/cli/internal/cmdutil"
|
||||
"github.com/larksuite/cli/internal/core"
|
||||
"github.com/larksuite/cli/internal/keysigner"
|
||||
)
|
||||
|
||||
// fakeRT routes requests to per-path handlers and records what it saw.
|
||||
type fakeRT struct {
|
||||
tatHandler func(req *http.Request) (*http.Response, error)
|
||||
probeHandler func(req *http.Request) (*http.Response, error)
|
||||
oauthHandler func(req *http.Request) (*http.Response, error)
|
||||
tatCalls int
|
||||
probeCalls int
|
||||
oauthCalls int
|
||||
probeReq *http.Request
|
||||
probeBody string
|
||||
}
|
||||
@@ -56,50 +48,10 @@ func (f *fakeRT) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
return jsonResp(200, `{"code":0,"data":{},"msg":"success"}`), nil
|
||||
}
|
||||
return f.probeHandler(req)
|
||||
case strings.HasSuffix(req.URL.Path, "/authen/v2/oauth/token"):
|
||||
f.oauthCalls++
|
||||
if f.oauthHandler == nil {
|
||||
return jsonResp(200, `{"access_token":"t-jwt"}`), nil
|
||||
}
|
||||
return f.oauthHandler(req)
|
||||
}
|
||||
return nil, errors.New("unexpected URL: " + req.URL.String())
|
||||
}
|
||||
|
||||
// probeTestSigner is an in-memory real ECDSA P-256 signer used to sign the
|
||||
// client_assertion in runProbePKJWT tests (authMethodTestSigner returns a nil
|
||||
// key and cannot sign).
|
||||
type probeTestSigner struct{ key *ecdsa.PrivateKey }
|
||||
|
||||
func newProbeTestSigner(t *testing.T) *probeTestSigner {
|
||||
t.Helper()
|
||||
k, err := ecdsa.GenerateKey(elliptic.P256(), crand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &probeTestSigner{key: k}
|
||||
}
|
||||
|
||||
func (p *probeTestSigner) EnsureKey(context.Context, keysigner.KeyRef) (crypto.PublicKey, error) {
|
||||
return p.key.Public(), nil
|
||||
}
|
||||
|
||||
func (p *probeTestSigner) PublicKey(context.Context, keysigner.KeyRef) (crypto.PublicKey, error) {
|
||||
return p.key.Public(), nil
|
||||
}
|
||||
|
||||
func (p *probeTestSigner) Sign(_ context.Context, _ keysigner.KeyRef, in []byte) ([]byte, string, error) {
|
||||
h := sha256.Sum256(in)
|
||||
r, s, err := ecdsa.Sign(crand.Reader, p.key, h[:])
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
sig := make([]byte, 64)
|
||||
r.FillBytes(sig[:32])
|
||||
s.FillBytes(sig[32:])
|
||||
return sig, keysigner.AlgES256, nil
|
||||
}
|
||||
|
||||
func jsonResp(code int, body string) *http.Response {
|
||||
return &http.Response{
|
||||
StatusCode: code,
|
||||
@@ -333,42 +285,3 @@ func TestRunProbe_TimeoutHonored(t *testing.T) {
|
||||
// must stay silent and not block.
|
||||
assertSilent(t, err, errBuf)
|
||||
}
|
||||
|
||||
// runProbePKJWT: a deterministic server rejection (invalid_client) is propagated
|
||||
// as a typed ConfigError so config init exits non-zero.
|
||||
func TestRunProbePKJWT_DeterministicReject_Propagates(t *testing.T) {
|
||||
rt := &fakeRT{oauthHandler: func(*http.Request) (*http.Response, error) {
|
||||
return jsonResp(401, `{"error":"invalid_client","error_description":"unknown key"}`), nil
|
||||
}}
|
||||
f, errBuf := fakeFactory(t, rt)
|
||||
err := runProbePKJWT(context.Background(), f, core.BrandFeishu, "cli_x", newProbeTestSigner(t), "agent-key")
|
||||
if err == nil || !errs.IsTyped(err) {
|
||||
t.Fatalf("expected propagated typed error, got %T %v", err, err)
|
||||
}
|
||||
if errBuf.Len() != 0 {
|
||||
t.Errorf("runProbePKJWT must not write stderr, got %q", errBuf.String())
|
||||
}
|
||||
}
|
||||
|
||||
// runProbePKJWT: ambiguous upstream noise (HTTP 503) is swallowed — silent, exit 0.
|
||||
func TestRunProbePKJWT_Ambiguous_Silent(t *testing.T) {
|
||||
rt := &fakeRT{oauthHandler: func(*http.Request) (*http.Response, error) {
|
||||
return jsonResp(503, `unavailable`), nil
|
||||
}}
|
||||
f, errBuf := fakeFactory(t, rt)
|
||||
assertSilent(t, runProbePKJWT(context.Background(), f, core.BrandFeishu, "cli_x", newProbeTestSigner(t), "agent-key"), errBuf)
|
||||
}
|
||||
|
||||
// runProbePKJWT: a successful mint returns nil.
|
||||
func TestRunProbePKJWT_Success_Silent(t *testing.T) {
|
||||
rt := &fakeRT{} // default oauth handler returns 200 + access_token
|
||||
f, errBuf := fakeFactory(t, rt)
|
||||
assertSilent(t, runProbePKJWT(context.Background(), f, core.BrandFeishu, "cli_x", newProbeTestSigner(t), "agent-key"), errBuf)
|
||||
}
|
||||
|
||||
// runProbePKJWT: a nil signer is a defensive no-op (should not be reached, must
|
||||
// not panic).
|
||||
func TestRunProbePKJWT_NilSigner_Silent(t *testing.T) {
|
||||
f, errBuf := fakeFactory(t, &fakeRT{})
|
||||
assertSilent(t, runProbePKJWT(context.Background(), f, core.BrandFeishu, "cli_x", nil, "k"), errBuf)
|
||||
}
|
||||
|
||||
@@ -10,25 +10,9 @@ import (
|
||||
|
||||
"github.com/larksuite/cli/errs"
|
||||
"github.com/larksuite/cli/internal/core"
|
||||
"github.com/larksuite/cli/internal/keychain"
|
||||
"github.com/larksuite/cli/internal/output"
|
||||
)
|
||||
|
||||
// TestRunRestoreFlow_NothingToRestore covers the early guards that return before
|
||||
// any network/registration call: no config at all, and a config whose resolved
|
||||
// app has no app id (nothing to send on begin).
|
||||
func TestRunRestoreFlow_NothingToRestore(t *testing.T) {
|
||||
// No config on disk.
|
||||
if err := runRestoreFlow(&ConfigInitOptions{}, nil, nil, nil); err == nil {
|
||||
t.Fatal("expected error when there is no config to restore")
|
||||
}
|
||||
// Config present but the resolved app has no app id.
|
||||
existing := &core.MultiAppConfig{Apps: []core.AppConfig{{AppId: ""}}}
|
||||
if err := runRestoreFlow(&ConfigInitOptions{}, existing, nil, nil); err == nil {
|
||||
t.Fatal("expected error when the resolved app has no app id")
|
||||
}
|
||||
}
|
||||
|
||||
// updateExistingProfileWithoutSecret guards four blank-input scenarios. Each
|
||||
// must surface as *ValidationError(SubtypeInvalidArgument) per RFC 6749 §5.2:
|
||||
// SubtypeInvalidClient is reserved for IAM rejection of malformed credentials,
|
||||
@@ -135,62 +119,3 @@ func assertValidationParam(t *testing.T, err error, wantParam string) {
|
||||
t.Errorf("Param = %q, want %q", valErr.Param, wantParam)
|
||||
}
|
||||
}
|
||||
|
||||
// countingKeychain is an in-memory KeychainAccess that records whether Remove
|
||||
// was invoked, so the stale-secret cleanup can be asserted without a real OS
|
||||
// keychain.
|
||||
type countingKeychain struct {
|
||||
store map[string]string
|
||||
removeCalled bool
|
||||
}
|
||||
|
||||
func newCountingKeychain() *countingKeychain {
|
||||
return &countingKeychain{store: map[string]string{}}
|
||||
}
|
||||
|
||||
func (k *countingKeychain) Get(service, account string) (string, error) {
|
||||
v, ok := k.store[service+"/"+account]
|
||||
if !ok {
|
||||
return "", keychain.ErrNotFound
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func (k *countingKeychain) Set(service, account, value string) error {
|
||||
k.store[service+"/"+account] = value
|
||||
return nil
|
||||
}
|
||||
|
||||
func (k *countingKeychain) Remove(service, account string) error {
|
||||
k.removeCalled = true
|
||||
delete(k.store, service+"/"+account)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestRemoveStaleSecretForPKJWT_SameAppID(t *testing.T) {
|
||||
kc := newCountingKeychain()
|
||||
ref, err := core.ForStorage("cli_same", core.PlainSecret("old-secret"), kc) // → Source:"keychain"
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
existing := &core.MultiAppConfig{Apps: []core.AppConfig{{AppId: "cli_same", AppSecret: ref}}}
|
||||
removeStaleSecretForPKJWT(existing, "", "cli_same", kc)
|
||||
if !kc.removeCalled {
|
||||
t.Error("same appId with keychain secret: expected kc.Remove to be invoked")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveStaleSecretForPKJWT_DifferentAppID(t *testing.T) {
|
||||
kc := newCountingKeychain()
|
||||
ref, _ := core.ForStorage("cli_old", core.PlainSecret("old-secret"), kc)
|
||||
kc.removeCalled = false // ForStorage does not call Remove, but reset to be safe
|
||||
existing := &core.MultiAppConfig{Apps: []core.AppConfig{{AppId: "cli_old", AppSecret: ref}}}
|
||||
removeStaleSecretForPKJWT(existing, "", "cli_new", kc)
|
||||
if kc.removeCalled {
|
||||
t.Error("different appId: must NOT remove")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveStaleSecretForPKJWT_NilExisting(t *testing.T) {
|
||||
removeStaleSecretForPKJWT(nil, "", "cli_x", newCountingKeychain()) // must not panic
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"sync"
|
||||
@@ -20,7 +19,6 @@ import (
|
||||
"github.com/larksuite/cli/internal/cmdutil"
|
||||
"github.com/larksuite/cli/internal/core"
|
||||
"github.com/larksuite/cli/internal/identitydiag"
|
||||
"github.com/larksuite/cli/internal/keysigner"
|
||||
"github.com/larksuite/cli/internal/output"
|
||||
"github.com/larksuite/cli/internal/transport"
|
||||
"github.com/larksuite/cli/internal/update"
|
||||
@@ -134,9 +132,6 @@ func doctorRun(opts *DoctorOptions) error {
|
||||
checks = append(checks, fail("identity_ready", "no usable bot or user identity is available", "run: lark-cli auth status --verify"))
|
||||
}
|
||||
|
||||
// ── 3b. private_key_jwt / TEE signer (local; runs even with --offline) ──
|
||||
checks = append(checks, teeSignerCheck(opts.Ctx, cfg))
|
||||
|
||||
// ── 4 & 5. Endpoint reachability ──
|
||||
checks = append(checks, networkChecks(opts.Ctx, opts, ep)...)
|
||||
|
||||
@@ -150,54 +145,6 @@ func identityCheck(name string, id identitydiag.Identity) checkResult {
|
||||
return warn(name, id.Message, id.Hint)
|
||||
}
|
||||
|
||||
const teeUnavailableHint = "ensure the device secure hardware is accessible (Linux TPM: add your user to the 'tss' group or run with sufficient privileges)"
|
||||
|
||||
// teeSignerCheck reports the private_key_jwt signing backend (TEE/TPM) status.
|
||||
// The probe is local hardware only (no network), so it runs even with --offline;
|
||||
// in a build without a TEE signer it short-circuits without touching any
|
||||
// hardware. It is a hard requirement for private_key_jwt apps and purely
|
||||
// informational for client_secret apps.
|
||||
func teeSignerCheck(ctx context.Context, cfg *core.CliConfig) checkResult {
|
||||
usesPKJWT := cfg != nil && cfg.AuthMethod == core.AuthMethodPrivateKeyJWT
|
||||
info, ok, err := keysigner.ProbeActiveHardware(ctx)
|
||||
return teeCheckResult(info, ok, err, usesPKJWT)
|
||||
}
|
||||
|
||||
// teeCheckResult maps a hardware probe to a doctor check. Split out from
|
||||
// teeSignerCheck so the full matrix is unit-testable without a TPM.
|
||||
func teeCheckResult(info keysigner.HardwareInfo, ok bool, probeErr error, usesPKJWT bool) checkResult {
|
||||
const name = "tee_signer"
|
||||
|
||||
// No signer registered → private_key_jwt is unsupported on this build.
|
||||
if !ok {
|
||||
if usesPKJWT {
|
||||
return fail(name,
|
||||
"app uses private_key_jwt but this build has no TEE key signer",
|
||||
"the platform key signer ships by default on macOS, Linux, and Windows/amd64; this platform (e.g. Windows/arm64) has none — use a supported platform or re-register with --auth-method client_secret")
|
||||
}
|
||||
return skip(name, "no TEE signer in this build (only private_key_jwt is affected; client_secret is unaffected)")
|
||||
}
|
||||
|
||||
backend := info.Backend
|
||||
if backend == "" {
|
||||
backend = "tee"
|
||||
}
|
||||
|
||||
switch {
|
||||
case probeErr != nil:
|
||||
return warn(name, fmt.Sprintf("%s signer present but probe errored: %s", backend, probeErr), "")
|
||||
case info.Available:
|
||||
if info.VendorName != "" {
|
||||
return pass(name, fmt.Sprintf("%s TEE available (%s)", backend, info.VendorName))
|
||||
}
|
||||
return pass(name, fmt.Sprintf("%s TEE available", backend))
|
||||
case usesPKJWT:
|
||||
return fail(name, fmt.Sprintf("%s signer present but TEE unavailable: %s", backend, info.Reason), teeUnavailableHint)
|
||||
default:
|
||||
return warn(name, fmt.Sprintf("%s signer present but TEE unavailable: %s", backend, info.Reason), teeUnavailableHint)
|
||||
}
|
||||
}
|
||||
|
||||
// networkChecks probes Open API and MCP endpoints concurrently.
|
||||
func networkChecks(ctx context.Context, opts *DoctorOptions, ep core.Endpoints) []checkResult {
|
||||
if opts.Offline {
|
||||
@@ -287,90 +234,14 @@ func finishDoctor(f *cmdutil.Factory, checks []checkResult) error {
|
||||
}
|
||||
}
|
||||
|
||||
workspace := core.CurrentWorkspace().Display()
|
||||
// A terminal on STDOUT gets a readable report; pipes, redirects, scripts and
|
||||
// tests keep the stable JSON contract (NO_COLOR disables ANSI styling).
|
||||
// StdoutIsTerminal checks stdout specifically — IOStreams.IsTerminal reflects
|
||||
// stdin, which would wrongly send the human report into `doctor | jq`.
|
||||
if f.IOStreams.StdoutIsTerminal() {
|
||||
renderDoctorHuman(f.IOStreams.Out, workspace, checks, allOK, os.Getenv("NO_COLOR") == "")
|
||||
} else {
|
||||
output.PrintJson(f.IOStreams.Out, map[string]interface{}{
|
||||
"ok": allOK,
|
||||
"workspace": workspace,
|
||||
"checks": checks,
|
||||
})
|
||||
result := map[string]interface{}{
|
||||
"ok": allOK,
|
||||
"workspace": core.CurrentWorkspace().Display(),
|
||||
"checks": checks,
|
||||
}
|
||||
output.PrintJson(f.IOStreams.Out, result)
|
||||
if !allOK {
|
||||
return output.ErrBare(1)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// renderDoctorHuman writes a readable health report: one aligned line per check
|
||||
// with a colored status tag, an indented hint when present, and a summary line.
|
||||
func renderDoctorHuman(w io.Writer, workspace string, checks []checkResult, allOK, color bool) {
|
||||
const (
|
||||
green = "\033[32m"
|
||||
yellow = "\033[33m"
|
||||
red = "\033[31m"
|
||||
gray = "\033[90m"
|
||||
bold = "\033[1m"
|
||||
reset = "\033[0m"
|
||||
)
|
||||
colorOf := map[string]string{"pass": green, "warn": yellow, "fail": red, "skip": gray}
|
||||
tagOf := map[string]string{"pass": "PASS", "warn": "WARN", "fail": "FAIL", "skip": "SKIP"}
|
||||
paint := func(code, s string) string {
|
||||
if !color || code == "" {
|
||||
return s
|
||||
}
|
||||
return code + s + reset
|
||||
}
|
||||
|
||||
nameW := 0
|
||||
for _, c := range checks {
|
||||
if len(c.Name) > nameW {
|
||||
nameW = len(c.Name)
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Fprintf(w, "\n%s (workspace: %s)\n\n", paint(bold, "lark-cli doctor"), workspace)
|
||||
|
||||
var passN, warnN, failN, skipN int
|
||||
for _, c := range checks {
|
||||
tag := tagOf[c.Status]
|
||||
if tag == "" {
|
||||
tag = "????"
|
||||
}
|
||||
fmt.Fprintf(w, " %s %-*s %s\n", paint(colorOf[c.Status], "["+tag+"]"), nameW, c.Name, c.Message)
|
||||
if c.Hint != "" {
|
||||
fmt.Fprintf(w, " %-*s %s\n", nameW, "", paint(gray, "↳ "+c.Hint))
|
||||
}
|
||||
switch c.Status {
|
||||
case "pass":
|
||||
passN++
|
||||
case "warn":
|
||||
warnN++
|
||||
case "fail":
|
||||
failN++
|
||||
case "skip":
|
||||
skipN++
|
||||
}
|
||||
}
|
||||
|
||||
headline := paint(green, "healthy")
|
||||
if !allOK {
|
||||
headline = paint(red, "problems found")
|
||||
}
|
||||
fmt.Fprintf(w, "\n %s — %d passed", headline, passN)
|
||||
if warnN > 0 {
|
||||
fmt.Fprintf(w, ", %d warning(s)", warnN)
|
||||
}
|
||||
if failN > 0 {
|
||||
fmt.Fprintf(w, ", %d failed", failN)
|
||||
}
|
||||
if skipN > 0 {
|
||||
fmt.Fprintf(w, ", %d skipped", skipN)
|
||||
}
|
||||
fmt.Fprintln(w)
|
||||
}
|
||||
|
||||
@@ -4,18 +4,14 @@
|
||||
package doctor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/larksuite/cli/internal/cmdutil"
|
||||
"github.com/larksuite/cli/internal/core"
|
||||
"github.com/larksuite/cli/internal/keysigner"
|
||||
)
|
||||
|
||||
func TestNewCmdDoctor_FlagParsing(t *testing.T) {
|
||||
@@ -143,107 +139,6 @@ func TestDoctorRun_SplitsBotAndMissingUserIdentity(t *testing.T) {
|
||||
assertCheck(t, got.Checks, "identity_ready", "pass")
|
||||
}
|
||||
|
||||
func TestTeeCheckResult(t *testing.T) {
|
||||
avail := keysigner.HardwareInfo{Backend: "tpm2", Available: true, VendorName: "ACME"}
|
||||
unavail := keysigner.HardwareInfo{Backend: "tpm2", Reason: "open /dev/tpmrm0: permission denied"}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
info keysigner.HardwareInfo
|
||||
ok bool
|
||||
probeErr error
|
||||
pkjwt bool
|
||||
want string
|
||||
}{
|
||||
{"no signer + private_key_jwt → fail", keysigner.HardwareInfo{}, false, nil, true, "fail"},
|
||||
{"no signer + client_secret → skip", keysigner.HardwareInfo{}, false, nil, false, "skip"},
|
||||
{"available + private_key_jwt → pass", avail, true, nil, true, "pass"},
|
||||
{"available + client_secret → pass", avail, true, nil, false, "pass"},
|
||||
{"unavailable + private_key_jwt → fail", unavail, true, nil, true, "fail"},
|
||||
{"unavailable + client_secret → warn", unavail, true, nil, false, "warn"},
|
||||
{"probe error → warn", keysigner.HardwareInfo{Backend: "tpm2"}, true, errors.New("boom"), true, "warn"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := teeCheckResult(tc.info, tc.ok, tc.probeErr, tc.pkjwt)
|
||||
if got.Name != "tee_signer" {
|
||||
t.Errorf("name = %q, want tee_signer", got.Name)
|
||||
}
|
||||
if got.Status != tc.want {
|
||||
t.Errorf("status = %q, want %q (msg=%q)", got.Status, tc.want, got.Message)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDoctorRun_TeeSignerWired proves the tee_signer check is part of doctorRun.
|
||||
// It asserts the build-independent invariant (a client_secret app must never
|
||||
// FAIL on TEE) so the test passes whether or not a signer is compiled in.
|
||||
func TestDoctorRun_TeeSignerWired(t *testing.T) {
|
||||
t.Setenv("LARKSUITE_CLI_CONFIG_DIR", t.TempDir())
|
||||
if err := core.SaveMultiAppConfig(&core.MultiAppConfig{
|
||||
CurrentApp: "default",
|
||||
Apps: []core.AppConfig{{
|
||||
Name: "default", AppId: "test-app",
|
||||
AppSecret: core.PlainSecret("secret"), Brand: core.BrandFeishu,
|
||||
}},
|
||||
}); err != nil {
|
||||
t.Fatalf("SaveMultiAppConfig() error = %v", err)
|
||||
}
|
||||
f, stdout, _, _ := cmdutil.TestFactory(t, &core.CliConfig{
|
||||
AppID: "test-app", AppSecret: "secret", Brand: core.BrandFeishu,
|
||||
})
|
||||
if err := doctorRun(&DoctorOptions{Factory: f, Ctx: context.Background(), Offline: true}); err != nil {
|
||||
t.Fatalf("doctorRun() error = %v", err)
|
||||
}
|
||||
var got struct {
|
||||
Checks []checkResult `json:"checks"`
|
||||
}
|
||||
if err := json.Unmarshal(stdout.Bytes(), &got); err != nil {
|
||||
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||
}
|
||||
var c *checkResult
|
||||
for i := range got.Checks {
|
||||
if got.Checks[i].Name == "tee_signer" {
|
||||
c = &got.Checks[i]
|
||||
}
|
||||
}
|
||||
if c == nil {
|
||||
t.Fatalf("tee_signer check not present in doctor output: %#v", got.Checks)
|
||||
}
|
||||
if c.Status == "fail" {
|
||||
t.Errorf("tee_signer = fail for a client_secret app; want skip/warn/pass (msg=%q)", c.Message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderDoctorHuman(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
checks := []checkResult{
|
||||
pass("cli_version", "1.0.50"),
|
||||
warn("tee_signer", "tpm2 signer present but TEE unavailable", "add your user to the 'tss' group"),
|
||||
fail("identity_ready", "no usable identity", "run: lark-cli auth status --verify"),
|
||||
skip("endpoint_open", "skipped (--offline)"),
|
||||
}
|
||||
renderDoctorHuman(&buf, "local", checks, false, false)
|
||||
out := buf.String()
|
||||
|
||||
for _, want := range []string{
|
||||
"lark-cli doctor", "workspace: local",
|
||||
"[PASS]", "cli_version", "1.0.50",
|
||||
"[WARN]", "tee_signer", "↳ add your user to the 'tss' group",
|
||||
"[FAIL]", "identity_ready", "↳ run: lark-cli auth status --verify",
|
||||
"[SKIP]", "endpoint_open",
|
||||
"problems found", "1 passed", "1 warning(s)", "1 failed", "1 skipped",
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("output missing %q\n---\n%s", want, out)
|
||||
}
|
||||
}
|
||||
if strings.Contains(out, "\033[") {
|
||||
t.Errorf("color=false but ANSI escapes present:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func assertCheck(t *testing.T, checks []checkResult, name, status string) {
|
||||
t.Helper()
|
||||
for _, check := range checks {
|
||||
|
||||
18
go.mod
18
go.mod
@@ -7,8 +7,6 @@ require (
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.0
|
||||
github.com/charmbracelet/huh v1.0.0
|
||||
github.com/charmbracelet/lipgloss v1.1.0
|
||||
github.com/facebookincubator/flog v0.0.0-20190930132826-d2511d0ce33c
|
||||
github.com/facebookincubator/sks v0.0.0-20251112220143-6823f23937b4
|
||||
github.com/gofrs/flock v0.8.1
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/itchyny/gojq v0.12.17
|
||||
@@ -29,10 +27,7 @@ require (
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require github.com/ebitengine/purego v0.10.1
|
||||
|
||||
require (
|
||||
github.com/StackExchange/wmi v1.2.1 // indirect
|
||||
github.com/atotto/clipboard v0.1.4 // indirect
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/catppuccin/go v0.3.0 // indirect
|
||||
@@ -47,23 +42,12 @@ require (
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
|
||||
github.com/go-ole/go-ole v1.2.5 // indirect
|
||||
github.com/godbus/dbus/v5 v5.2.2 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/google/btree v1.0.1 // indirect
|
||||
github.com/google/certificate-transparency-go v1.1.2 // indirect
|
||||
github.com/google/certtostore v1.0.3-0.20230404221207-8d01647071cc // indirect
|
||||
github.com/google/deck v0.0.0-20230104221208-105ad94aa8ae // indirect
|
||||
github.com/google/go-attestation v0.5.1 // indirect
|
||||
github.com/google/go-tpm v0.9.0 // indirect
|
||||
github.com/google/go-tspi v0.3.0 // indirect
|
||||
github.com/gopherjs/gopherjs v1.17.2 // indirect
|
||||
github.com/gorilla/websocket v1.5.0 // indirect
|
||||
github.com/hashicorp/errwrap v1.0.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/itchyny/timefmt-go v0.1.6 // indirect
|
||||
github.com/jgoguen/go-utils v0.0.0-20200211015258-b42ad41486fd // indirect
|
||||
github.com/jtolds/gls v4.20.0+incompatible // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
@@ -73,12 +57,10 @@ require (
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
|
||||
github.com/muesli/cancelreader v0.2.2 // indirect
|
||||
github.com/muesli/termenv v0.16.0 // indirect
|
||||
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/smarty/assertions v1.15.0 // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/pretty v1.2.0 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
golang.org/x/crypto v0.31.0 // indirect
|
||||
)
|
||||
|
||||
5
harness-opt/.gitignore
vendored
Normal file
5
harness-opt/.gitignore
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
# harness-opt 只入库轻量决策记录;重的原始评测 run 不进版本库(dashboard 仍读磁盘)。
|
||||
baseline/runs/
|
||||
**/child-runs/
|
||||
verify_results/sealed-runs/
|
||||
verify_results/*-runs/
|
||||
5
harness-opt/baseline/baseline-tokens.json
Normal file
5
harness-opt/baseline/baseline-tokens.json
Normal file
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"1": 30086,
|
||||
"2": 34616,
|
||||
"3": 31289
|
||||
}
|
||||
50
harness-opt/baseline/noise-floor.json
Normal file
50
harness-opt/baseline/noise-floor.json
Normal file
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"k": 5,
|
||||
"metrics": {
|
||||
"success_rate": {
|
||||
"mean": 0.4666666666666666,
|
||||
"std": 0.1632993161855452,
|
||||
"k": 5,
|
||||
"band": [
|
||||
0.14006803429557624,
|
||||
0.793265299037757
|
||||
]
|
||||
},
|
||||
"mean_score": {
|
||||
"mean": 0.5111111111111111,
|
||||
"std": 0.1507184440694504,
|
||||
"k": 5,
|
||||
"band": [
|
||||
0.20967422297221028,
|
||||
0.8125479992500119
|
||||
]
|
||||
},
|
||||
"mean_context_window": {
|
||||
"mean": 31997.0,
|
||||
"std": 7166.8411203573105,
|
||||
"k": 5,
|
||||
"band": [
|
||||
17663.31775928538,
|
||||
46330.682240714625
|
||||
]
|
||||
},
|
||||
"mean_duration_ms": {
|
||||
"mean": 50188.86666666667,
|
||||
"std": 7746.3168641619595,
|
||||
"k": 5,
|
||||
"band": [
|
||||
34696.23293834275,
|
||||
65681.50039499058
|
||||
]
|
||||
},
|
||||
"mean_token": {
|
||||
"mean": 263981.06666666665,
|
||||
"std": 27890.193480385413,
|
||||
"k": 5,
|
||||
"band": [
|
||||
208200.67970589583,
|
||||
319761.45362743747
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
33
harness-opt/baseline/summary.json
Normal file
33
harness-opt/baseline/summary.json
Normal file
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"k": 5,
|
||||
"n_cases": 3,
|
||||
"effect": {
|
||||
"mean": 0.5111111111111111,
|
||||
"sigma": 0.1507184440694504
|
||||
},
|
||||
"token": {
|
||||
"mean": 31997.0,
|
||||
"sigma": 7166.8411203573105
|
||||
},
|
||||
"duration": {
|
||||
"mean": 50188.86666666667,
|
||||
"sigma": 7746.3168641619595
|
||||
},
|
||||
"phi0_per_case": {
|
||||
"1": {
|
||||
"effect": 0.6,
|
||||
"token": 30086,
|
||||
"duration": 51004
|
||||
},
|
||||
"2": {
|
||||
"effect": 0.4,
|
||||
"token": 34616,
|
||||
"duration": 52787
|
||||
},
|
||||
"3": {
|
||||
"effect": 0.5333,
|
||||
"token": 31289,
|
||||
"duration": 46776
|
||||
}
|
||||
}
|
||||
}
|
||||
869
harness-opt/coverage.json
Normal file
869
harness-opt/coverage.json
Normal file
@@ -0,0 +1,869 @@
|
||||
{
|
||||
"summary": {
|
||||
"total_cases": 3,
|
||||
"files": 25,
|
||||
"expected_declared": 0,
|
||||
"blind_spots": 22,
|
||||
"overfit_high": 5,
|
||||
"suggest_add_cases": [
|
||||
"skills/lark-im/references/lark-im-chat-identity.md",
|
||||
"skills/lark-im/references/lark-im-flag-cancel.md",
|
||||
"skills/lark-im/references/lark-im-flag-create.md",
|
||||
"skills/lark-im/references/lark-im-message-enrichment.md",
|
||||
"skills/lark-im/references/lark-im-messages-search.md"
|
||||
],
|
||||
"suggest_fix_routing": []
|
||||
},
|
||||
"files": [
|
||||
{
|
||||
"path": "skills/lark-im/references/lark-im-chat-identity.md",
|
||||
"is_domain_skill": false,
|
||||
"actual": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"expected": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"union": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"discoverability_miss": 0,
|
||||
"density_count": 0,
|
||||
"density_pct": 0.0,
|
||||
"density_tier": "盲区",
|
||||
"risk_tier": "高",
|
||||
"risk_lines": {
|
||||
"R0": 5,
|
||||
"R1": 0,
|
||||
"R2": 0,
|
||||
"R3": 50
|
||||
},
|
||||
"total_lines": 55,
|
||||
"overfit_risk": "高",
|
||||
"suggest_add_cases": true,
|
||||
"suggest_fix_routing": false
|
||||
},
|
||||
{
|
||||
"path": "skills/lark-im/references/lark-im-messages-search.md",
|
||||
"is_domain_skill": false,
|
||||
"actual": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"expected": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"union": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"discoverability_miss": 0,
|
||||
"density_count": 0,
|
||||
"density_pct": 0.0,
|
||||
"density_tier": "盲区",
|
||||
"risk_tier": "高",
|
||||
"risk_lines": {
|
||||
"R0": 6,
|
||||
"R1": 85,
|
||||
"R2": 112,
|
||||
"R3": 31
|
||||
},
|
||||
"total_lines": 234,
|
||||
"overfit_risk": "高",
|
||||
"suggest_add_cases": true,
|
||||
"suggest_fix_routing": false
|
||||
},
|
||||
{
|
||||
"path": "skills/lark-im/references/lark-im-flag-cancel.md",
|
||||
"is_domain_skill": false,
|
||||
"actual": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"expected": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"union": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"discoverability_miss": 0,
|
||||
"density_count": 0,
|
||||
"density_pct": 0.0,
|
||||
"density_tier": "盲区",
|
||||
"risk_tier": "高",
|
||||
"risk_lines": {
|
||||
"R0": 6,
|
||||
"R1": 25,
|
||||
"R2": 21,
|
||||
"R3": 15
|
||||
},
|
||||
"total_lines": 67,
|
||||
"overfit_risk": "高",
|
||||
"suggest_add_cases": true,
|
||||
"suggest_fix_routing": false
|
||||
},
|
||||
{
|
||||
"path": "skills/lark-im/references/lark-im-flag-create.md",
|
||||
"is_domain_skill": false,
|
||||
"actual": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"expected": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"union": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"discoverability_miss": 0,
|
||||
"density_count": 0,
|
||||
"density_pct": 0.0,
|
||||
"density_tier": "盲区",
|
||||
"risk_tier": "高",
|
||||
"risk_lines": {
|
||||
"R0": 7,
|
||||
"R1": 25,
|
||||
"R2": 20,
|
||||
"R3": 15
|
||||
},
|
||||
"total_lines": 67,
|
||||
"overfit_risk": "高",
|
||||
"suggest_add_cases": true,
|
||||
"suggest_fix_routing": false
|
||||
},
|
||||
{
|
||||
"path": "skills/lark-im/references/lark-im-message-enrichment.md",
|
||||
"is_domain_skill": false,
|
||||
"actual": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"expected": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"union": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"discoverability_miss": 0,
|
||||
"density_count": 0,
|
||||
"density_pct": 0.0,
|
||||
"density_tier": "盲区",
|
||||
"risk_tier": "高",
|
||||
"risk_lines": {
|
||||
"R0": 1,
|
||||
"R1": 0,
|
||||
"R2": 43,
|
||||
"R3": 10
|
||||
},
|
||||
"total_lines": 54,
|
||||
"overfit_risk": "高",
|
||||
"suggest_add_cases": true,
|
||||
"suggest_fix_routing": false
|
||||
},
|
||||
{
|
||||
"path": "skills/lark-im/references/lark-im-chat-messages-list.md",
|
||||
"is_domain_skill": false,
|
||||
"actual": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"expected": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"union": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"discoverability_miss": 0,
|
||||
"density_count": 0,
|
||||
"density_pct": 0.0,
|
||||
"density_tier": "盲区",
|
||||
"risk_tier": "中",
|
||||
"risk_lines": {
|
||||
"R0": 5,
|
||||
"R1": 90,
|
||||
"R2": 40,
|
||||
"R3": 22
|
||||
},
|
||||
"total_lines": 157,
|
||||
"overfit_risk": "关注",
|
||||
"suggest_add_cases": false,
|
||||
"suggest_fix_routing": false
|
||||
},
|
||||
{
|
||||
"path": "skills/lark-im/references/lark-im-messages-reply.md",
|
||||
"is_domain_skill": false,
|
||||
"actual": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"expected": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"union": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"discoverability_miss": 0,
|
||||
"density_count": 0,
|
||||
"density_pct": 0.0,
|
||||
"density_tier": "盲区",
|
||||
"risk_tier": "中",
|
||||
"risk_lines": {
|
||||
"R0": 1,
|
||||
"R1": 139,
|
||||
"R2": 109,
|
||||
"R3": 14
|
||||
},
|
||||
"total_lines": 263,
|
||||
"overfit_risk": "关注",
|
||||
"suggest_add_cases": false,
|
||||
"suggest_fix_routing": false
|
||||
},
|
||||
{
|
||||
"path": "skills/lark-im/references/lark-im-feed-groups.md",
|
||||
"is_domain_skill": false,
|
||||
"actual": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"expected": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"union": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"discoverability_miss": 0,
|
||||
"density_count": 0,
|
||||
"density_pct": 0.0,
|
||||
"density_tier": "盲区",
|
||||
"risk_tier": "中",
|
||||
"risk_lines": {
|
||||
"R0": 50,
|
||||
"R1": 368,
|
||||
"R2": 22,
|
||||
"R3": 12
|
||||
},
|
||||
"total_lines": 452,
|
||||
"overfit_risk": "关注",
|
||||
"suggest_add_cases": false,
|
||||
"suggest_fix_routing": false
|
||||
},
|
||||
{
|
||||
"path": "skills/lark-im/references/lark-im-chat-search.md",
|
||||
"is_domain_skill": false,
|
||||
"actual": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"expected": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"union": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"discoverability_miss": 0,
|
||||
"density_count": 0,
|
||||
"density_pct": 0.0,
|
||||
"density_tier": "盲区",
|
||||
"risk_tier": "中",
|
||||
"risk_lines": {
|
||||
"R0": 5,
|
||||
"R1": 102,
|
||||
"R2": 24,
|
||||
"R3": 11
|
||||
},
|
||||
"total_lines": 142,
|
||||
"overfit_risk": "关注",
|
||||
"suggest_add_cases": false,
|
||||
"suggest_fix_routing": false
|
||||
},
|
||||
{
|
||||
"path": "skills/lark-im/references/lark-im-chat-update.md",
|
||||
"is_domain_skill": false,
|
||||
"actual": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"expected": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"union": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"discoverability_miss": 0,
|
||||
"density_count": 0,
|
||||
"density_pct": 0.0,
|
||||
"density_tier": "盲区",
|
||||
"risk_tier": "中",
|
||||
"risk_lines": {
|
||||
"R0": 5,
|
||||
"R1": 67,
|
||||
"R2": 2,
|
||||
"R3": 10
|
||||
},
|
||||
"total_lines": 84,
|
||||
"overfit_risk": "关注",
|
||||
"suggest_add_cases": false,
|
||||
"suggest_fix_routing": false
|
||||
},
|
||||
{
|
||||
"path": "skills/lark-im/references/lark-im-messages-resources-download.md",
|
||||
"is_domain_skill": false,
|
||||
"actual": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"expected": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"union": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"discoverability_miss": 0,
|
||||
"density_count": 0,
|
||||
"density_pct": 0.0,
|
||||
"density_tier": "盲区",
|
||||
"risk_tier": "中",
|
||||
"risk_lines": {
|
||||
"R0": 5,
|
||||
"R1": 55,
|
||||
"R2": 24,
|
||||
"R3": 10
|
||||
},
|
||||
"total_lines": 94,
|
||||
"overfit_risk": "关注",
|
||||
"suggest_add_cases": false,
|
||||
"suggest_fix_routing": false
|
||||
},
|
||||
{
|
||||
"path": "skills/lark-im/references/lark-im-threads-messages-list.md",
|
||||
"is_domain_skill": false,
|
||||
"actual": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"expected": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"union": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"discoverability_miss": 0,
|
||||
"density_count": 0,
|
||||
"density_pct": 0.0,
|
||||
"density_tier": "盲区",
|
||||
"risk_tier": "中",
|
||||
"risk_lines": {
|
||||
"R0": 6,
|
||||
"R1": 72,
|
||||
"R2": 28,
|
||||
"R3": 9
|
||||
},
|
||||
"total_lines": 115,
|
||||
"overfit_risk": "关注",
|
||||
"suggest_add_cases": false,
|
||||
"suggest_fix_routing": false
|
||||
},
|
||||
{
|
||||
"path": "skills/lark-im/references/lark-im-chat-list.md",
|
||||
"is_domain_skill": false,
|
||||
"actual": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"expected": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"union": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"discoverability_miss": 0,
|
||||
"density_count": 0,
|
||||
"density_pct": 0.0,
|
||||
"density_tier": "盲区",
|
||||
"risk_tier": "中",
|
||||
"risk_lines": {
|
||||
"R0": 1,
|
||||
"R1": 103,
|
||||
"R2": 56,
|
||||
"R3": 6
|
||||
},
|
||||
"total_lines": 166,
|
||||
"overfit_risk": "关注",
|
||||
"suggest_add_cases": false,
|
||||
"suggest_fix_routing": false
|
||||
},
|
||||
{
|
||||
"path": "skills/lark-im/references/lark-im-flag-list.md",
|
||||
"is_domain_skill": false,
|
||||
"actual": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"expected": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"union": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"discoverability_miss": 0,
|
||||
"density_count": 0,
|
||||
"density_pct": 0.0,
|
||||
"density_tier": "盲区",
|
||||
"risk_tier": "中",
|
||||
"risk_lines": {
|
||||
"R0": 5,
|
||||
"R1": 80,
|
||||
"R2": 9,
|
||||
"R3": 6
|
||||
},
|
||||
"total_lines": 100,
|
||||
"overfit_risk": "关注",
|
||||
"suggest_add_cases": false,
|
||||
"suggest_fix_routing": false
|
||||
},
|
||||
{
|
||||
"path": "skills/lark-im/references/lark-im-reactions.md",
|
||||
"is_domain_skill": false,
|
||||
"actual": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"expected": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"union": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"discoverability_miss": 0,
|
||||
"density_count": 0,
|
||||
"density_pct": 0.0,
|
||||
"density_tier": "盲区",
|
||||
"risk_tier": "中",
|
||||
"risk_lines": {
|
||||
"R0": 73,
|
||||
"R1": 206,
|
||||
"R2": 18,
|
||||
"R3": 2
|
||||
},
|
||||
"total_lines": 299,
|
||||
"overfit_risk": "关注",
|
||||
"suggest_add_cases": false,
|
||||
"suggest_fix_routing": false
|
||||
},
|
||||
{
|
||||
"path": "skills/lark-im/references/lark-im-feed-group-list-item.md",
|
||||
"is_domain_skill": false,
|
||||
"actual": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"expected": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"union": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"discoverability_miss": 0,
|
||||
"density_count": 0,
|
||||
"density_pct": 0.0,
|
||||
"density_tier": "盲区",
|
||||
"risk_tier": "中",
|
||||
"risk_lines": {
|
||||
"R0": 7,
|
||||
"R1": 44,
|
||||
"R2": 17,
|
||||
"R3": 0
|
||||
},
|
||||
"total_lines": 68,
|
||||
"overfit_risk": "关注",
|
||||
"suggest_add_cases": false,
|
||||
"suggest_fix_routing": false
|
||||
},
|
||||
{
|
||||
"path": "skills/lark-im/references/lark-im-feed-group-list.md",
|
||||
"is_domain_skill": false,
|
||||
"actual": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"expected": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"union": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"discoverability_miss": 0,
|
||||
"density_count": 0,
|
||||
"density_pct": 0.0,
|
||||
"density_tier": "盲区",
|
||||
"risk_tier": "中",
|
||||
"risk_lines": {
|
||||
"R0": 6,
|
||||
"R1": 44,
|
||||
"R2": 15,
|
||||
"R3": 0
|
||||
},
|
||||
"total_lines": 65,
|
||||
"overfit_risk": "关注",
|
||||
"suggest_add_cases": false,
|
||||
"suggest_fix_routing": false
|
||||
},
|
||||
{
|
||||
"path": "skills/lark-im/references/lark-im-feed-group-query-item.md",
|
||||
"is_domain_skill": false,
|
||||
"actual": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"expected": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"union": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"discoverability_miss": 0,
|
||||
"density_count": 0,
|
||||
"density_pct": 0.0,
|
||||
"density_tier": "盲区",
|
||||
"risk_tier": "中",
|
||||
"risk_lines": {
|
||||
"R0": 6,
|
||||
"R1": 21,
|
||||
"R2": 17,
|
||||
"R3": 0
|
||||
},
|
||||
"total_lines": 44,
|
||||
"overfit_risk": "关注",
|
||||
"suggest_add_cases": false,
|
||||
"suggest_fix_routing": false
|
||||
},
|
||||
{
|
||||
"path": "skills/lark-im/references/lark-im-feed-shortcut-create.md",
|
||||
"is_domain_skill": false,
|
||||
"actual": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"expected": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"union": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"discoverability_miss": 0,
|
||||
"density_count": 0,
|
||||
"density_pct": 0.0,
|
||||
"density_tier": "盲区",
|
||||
"risk_tier": "中",
|
||||
"risk_lines": {
|
||||
"R0": 7,
|
||||
"R1": 70,
|
||||
"R2": 20,
|
||||
"R3": 0
|
||||
},
|
||||
"total_lines": 97,
|
||||
"overfit_risk": "关注",
|
||||
"suggest_add_cases": false,
|
||||
"suggest_fix_routing": false
|
||||
},
|
||||
{
|
||||
"path": "skills/lark-im/references/lark-im-feed-shortcut-list.md",
|
||||
"is_domain_skill": false,
|
||||
"actual": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"expected": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"union": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"discoverability_miss": 0,
|
||||
"density_count": 0,
|
||||
"density_pct": 0.0,
|
||||
"density_tier": "盲区",
|
||||
"risk_tier": "中",
|
||||
"risk_lines": {
|
||||
"R0": 6,
|
||||
"R1": 73,
|
||||
"R2": 24,
|
||||
"R3": 0
|
||||
},
|
||||
"total_lines": 103,
|
||||
"overfit_risk": "关注",
|
||||
"suggest_add_cases": false,
|
||||
"suggest_fix_routing": false
|
||||
},
|
||||
{
|
||||
"path": "skills/lark-im/references/lark-im-feed-shortcut-remove.md",
|
||||
"is_domain_skill": false,
|
||||
"actual": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"expected": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"union": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"discoverability_miss": 0,
|
||||
"density_count": 0,
|
||||
"density_pct": 0.0,
|
||||
"density_tier": "盲区",
|
||||
"risk_tier": "中",
|
||||
"risk_lines": {
|
||||
"R0": 10,
|
||||
"R1": 24,
|
||||
"R2": 14,
|
||||
"R3": 0
|
||||
},
|
||||
"total_lines": 48,
|
||||
"overfit_risk": "关注",
|
||||
"suggest_add_cases": false,
|
||||
"suggest_fix_routing": false
|
||||
},
|
||||
{
|
||||
"path": "skills/lark-im/SKILL.md",
|
||||
"is_domain_skill": true,
|
||||
"actual": {
|
||||
"count": 3,
|
||||
"pct": 1.0,
|
||||
"tier": "密"
|
||||
},
|
||||
"expected": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"union": {
|
||||
"count": 3,
|
||||
"pct": 1.0,
|
||||
"tier": "密"
|
||||
},
|
||||
"discoverability_miss": 0,
|
||||
"density_count": 3,
|
||||
"density_pct": 1.0,
|
||||
"density_tier": "密",
|
||||
"risk_tier": "中",
|
||||
"risk_lines": {
|
||||
"R0": 122,
|
||||
"R1": 0,
|
||||
"R2": 68,
|
||||
"R3": 41
|
||||
},
|
||||
"total_lines": 231,
|
||||
"overfit_risk": "低",
|
||||
"suggest_add_cases": false,
|
||||
"suggest_fix_routing": false
|
||||
},
|
||||
{
|
||||
"path": "skills/lark-im/references/lark-im-chat-create.md",
|
||||
"is_domain_skill": false,
|
||||
"actual": {
|
||||
"count": 2,
|
||||
"pct": 0.667,
|
||||
"tier": "密"
|
||||
},
|
||||
"expected": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"union": {
|
||||
"count": 2,
|
||||
"pct": 0.667,
|
||||
"tier": "密"
|
||||
},
|
||||
"discoverability_miss": 0,
|
||||
"density_count": 2,
|
||||
"density_pct": 0.667,
|
||||
"density_tier": "密",
|
||||
"risk_tier": "中",
|
||||
"risk_lines": {
|
||||
"R0": 5,
|
||||
"R1": 116,
|
||||
"R2": 12,
|
||||
"R3": 29
|
||||
},
|
||||
"total_lines": 162,
|
||||
"overfit_risk": "低",
|
||||
"suggest_add_cases": false,
|
||||
"suggest_fix_routing": false
|
||||
},
|
||||
{
|
||||
"path": "skills/lark-im/references/lark-im-messages-send.md",
|
||||
"is_domain_skill": false,
|
||||
"actual": {
|
||||
"count": 2,
|
||||
"pct": 0.667,
|
||||
"tier": "密"
|
||||
},
|
||||
"expected": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"union": {
|
||||
"count": 2,
|
||||
"pct": 0.667,
|
||||
"tier": "密"
|
||||
},
|
||||
"discoverability_miss": 0,
|
||||
"density_count": 2,
|
||||
"density_pct": 0.667,
|
||||
"density_tier": "密",
|
||||
"risk_tier": "中",
|
||||
"risk_lines": {
|
||||
"R0": 1,
|
||||
"R1": 140,
|
||||
"R2": 109,
|
||||
"R3": 14
|
||||
},
|
||||
"total_lines": 264,
|
||||
"overfit_risk": "低",
|
||||
"suggest_add_cases": false,
|
||||
"suggest_fix_routing": false
|
||||
},
|
||||
{
|
||||
"path": "skills/lark-im/references/lark-im-messages-mget.md",
|
||||
"is_domain_skill": false,
|
||||
"actual": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"expected": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"union": {
|
||||
"count": 0,
|
||||
"pct": 0.0,
|
||||
"tier": "盲区"
|
||||
},
|
||||
"discoverability_miss": 0,
|
||||
"density_count": 0,
|
||||
"density_pct": 0.0,
|
||||
"density_tier": "盲区",
|
||||
"risk_tier": "低",
|
||||
"risk_lines": {
|
||||
"R0": 5,
|
||||
"R1": 84,
|
||||
"R2": 10,
|
||||
"R3": 0
|
||||
},
|
||||
"total_lines": 99,
|
||||
"overfit_risk": "低",
|
||||
"suggest_add_cases": false,
|
||||
"suggest_fix_routing": false
|
||||
}
|
||||
]
|
||||
}
|
||||
48
harness-opt/objective.json
Normal file
48
harness-opt/objective.json
Normal file
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"slug": "im-token",
|
||||
"modules": [
|
||||
"skills/lark-im/SKILL.md",
|
||||
"skills/lark-im/references/lark-im-chat-create.md",
|
||||
"skills/lark-im/references/lark-im-chat-identity.md",
|
||||
"skills/lark-im/references/lark-im-chat-list.md",
|
||||
"skills/lark-im/references/lark-im-chat-messages-list.md",
|
||||
"skills/lark-im/references/lark-im-chat-search.md",
|
||||
"skills/lark-im/references/lark-im-chat-update.md",
|
||||
"skills/lark-im/references/lark-im-feed-group-list-item.md",
|
||||
"skills/lark-im/references/lark-im-feed-group-list.md",
|
||||
"skills/lark-im/references/lark-im-feed-group-query-item.md",
|
||||
"skills/lark-im/references/lark-im-feed-groups.md",
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-create.md",
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-list.md",
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-remove.md",
|
||||
"skills/lark-im/references/lark-im-flag-cancel.md",
|
||||
"skills/lark-im/references/lark-im-flag-create.md",
|
||||
"skills/lark-im/references/lark-im-flag-list.md",
|
||||
"skills/lark-im/references/lark-im-message-enrichment.md",
|
||||
"skills/lark-im/references/lark-im-messages-mget.md",
|
||||
"skills/lark-im/references/lark-im-messages-reply.md",
|
||||
"skills/lark-im/references/lark-im-messages-resources-download.md",
|
||||
"skills/lark-im/references/lark-im-messages-search.md",
|
||||
"skills/lark-im/references/lark-im-messages-send.md",
|
||||
"skills/lark-im/references/lark-im-reactions.md",
|
||||
"skills/lark-im/references/lark-im-threads-messages-list.md"
|
||||
],
|
||||
"modules_spec": [
|
||||
"skills/lark-im/**/*.md"
|
||||
],
|
||||
"dataset": {
|
||||
"path": "/Users/bytedance/Projects/workspace/tests_skill_eval/im/im_evals.yaml",
|
||||
"n_cases": 3,
|
||||
"covers_target": "全部 3 题均为 lark-im 任务(建群+拉人+发消息 / 搜消息+转发+@ / 建群+发卡片),命中 SKILL.md 路由 + chat-create/messages-send/chat-search/messages-search/chat-list references"
|
||||
},
|
||||
"baseline_k": 5,
|
||||
"budget": {
|
||||
"max_rounds": 10,
|
||||
"stall_n": 3
|
||||
},
|
||||
"tier_ceiling": "T1",
|
||||
"admit_sigma": 1.0,
|
||||
"admit_sigma_duration": 1.0,
|
||||
"admit_sigma_effect": 1.0,
|
||||
"admit_sigma_target_boost": 0.0
|
||||
}
|
||||
60
harness-opt/opt-state.json
Normal file
60
harness-opt/opt-state.json
Normal file
@@ -0,0 +1,60 @@
|
||||
{
|
||||
"task_id": "OPT-IM-1",
|
||||
"title": "优化 lark-im(省 token 保成功率)",
|
||||
"branch": "feat/opt-im-token",
|
||||
"current_phase": "round",
|
||||
"phase_status": "in_progress",
|
||||
"started_at": "2026-06-23T17:52:10",
|
||||
"updated_at": "2026-06-23T19:38:08",
|
||||
"blockers": null,
|
||||
"transcript_path": "/Users/bytedance/.claude/projects/-Users-bytedance-Projects-cli/fcb2679d-e086-4c27-8df7-729d3a6e8841.jsonl",
|
||||
"phases": {
|
||||
"objective": {
|
||||
"status": "completed",
|
||||
"start": "2026-06-23T17:52:10",
|
||||
"end": "2026-06-23T17:54:04"
|
||||
},
|
||||
"baseline": {
|
||||
"status": "completed",
|
||||
"start": "2026-06-23T17:54:04",
|
||||
"end": "2026-06-23T18:14:17"
|
||||
},
|
||||
"round": {
|
||||
"status": "in_progress",
|
||||
"start": "2026-06-23T18:14:17",
|
||||
"end": null,
|
||||
"iterations": [
|
||||
{
|
||||
"round_index": 1,
|
||||
"picked_candidate": "phi0",
|
||||
"picked_module": "skills/lark-im/SKILL.md",
|
||||
"tier": "T1",
|
||||
"verdict": "admit",
|
||||
"reason": "engine admit=score_gain(eff 0.511→0.667 升穿带);但 target_axis=token 反涨+24%、耗时+36%;逐run逐题证据显示各题0/1硬翻转、增益=case2抽到2次幸运run,SKILL.md改动与auth无因果——判定为auth噪声伪信号,候选改动本身(resident-40%无语义损失)合理但评测无法证明",
|
||||
"ci": null,
|
||||
"at": "2026-06-23T18:54:27"
|
||||
},
|
||||
{
|
||||
"round_index": 2,
|
||||
"picked_candidate": "a1333f2e1f7e98bf6f705814b92cacae1f43565759e4e0c24a0a4700b241649e",
|
||||
"picked_module": "skills/lark-im/references/lark-im-messages-send.md",
|
||||
"tier": "T1",
|
||||
"verdict": "admit",
|
||||
"reason": "engine admit=score_gain(case080 单题 0.6→1.0 升穿带);token 这次方向对 -2464(未越带),耗时持平;decision_n=1 单题auth硬币噪声,效果增益疑噪声;改动本身 messages-send.md -53.5% 经reviewer核验真去冗余无语义损失",
|
||||
"ci": null,
|
||||
"at": "2026-06-23T19:38:08"
|
||||
}
|
||||
]
|
||||
},
|
||||
"seal": {
|
||||
"status": "pending",
|
||||
"start": null,
|
||||
"end": null
|
||||
},
|
||||
"handoff": {
|
||||
"status": "pending",
|
||||
"start": null,
|
||||
"end": null
|
||||
}
|
||||
}
|
||||
}
|
||||
13
harness-opt/opt-state.md
Normal file
13
harness-opt/opt-state.md
Normal file
@@ -0,0 +1,13 @@
|
||||
# Opt State: OPT-IM-1 优化 lark-im(省 token 保成功率)
|
||||
|
||||
## Phase 记录
|
||||
|
||||
### ✅ Phase 1: Objective
|
||||
进入 baseline:以现网 lark-im 文档为 Φ0,K=5 重复评测立噪声地板
|
||||
做了什么:确认7项objective(省token保成功率/T1/全lark-im范围/K5/10轮stall3/σ1.0)并写objective.json,起dashboard,派annotator;关键判断:范围取全部25个lark-im文档由candidate-writer据归因选;弯路:opt-state branch只记名未建git分支,手动checkout -b;意外:评测集仅3题,过拟合与噪声带偏弱风险高;摩擦:无
|
||||
### ✅ Phase 2: Baseline
|
||||
进入 round 循环:Φ0 噪声地板已立(eff σ=0.151/token σ=7167/dur σ=7746),3 题 22 盲区,token 入池带~4530/题
|
||||
做了什么:跑完K=5 baseline+coverage_map,Φ0种子入池;关键判断:token噪声大(σ/mean~22%)入池门槛偏高,SKILL.md常驻是reach全集的最高杠杆;弯路:无;意外:22/25文件是盲区,reach会天然把候选限制到SKILL.md+被读references;摩擦:无
|
||||
### 🔄 Phase 3: Round
|
||||
### ⬜ Phase 4: Seal
|
||||
### ⬜ Phase 5: Handoff
|
||||
12
harness-opt/pool/head.json
Normal file
12
harness-opt/pool/head.json
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"id": "53194d7a111df326cc078b633f43587225bd0132",
|
||||
"worktree": "/Users/bytedance/Projects/cli",
|
||||
"commit": "cbd6e56ac07285fd973c53ff7382da0112b6cf5d",
|
||||
"phi0_worktree": "/Users/bytedance/Projects/cli",
|
||||
"lineage": [
|
||||
"phi0",
|
||||
"a1333f2e1f7e98bf6f705814b92cacae1f43565759e4e0c24a0a4700b241649e",
|
||||
"557349b40feb359bb791749a37571d59edb7e72e",
|
||||
"53194d7a111df326cc078b633f43587225bd0132"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"1": {
|
||||
"score": 1.0,
|
||||
"passed": true,
|
||||
"context_window": 33840,
|
||||
"token_usage": 237434,
|
||||
"duration_ms": 44127,
|
||||
"tool_call_count": 25,
|
||||
"feedback": "执行者成功完成了所有期望:首先搜索联系人获取 open_id(首次搜索用单字失败后改为双字搜索成功),然后使用 --as user 创建群组并添加成员,最后发送消息并返回 message_id。整个流程正确,使用了等效的 `--as user` 身份,符合用户「使用我的身份」的要求。验证结果确认所有操作均已生效。",
|
||||
"from_round": 3,
|
||||
"from_candidate": "53194d7a111df326cc078b633f43587225bd0132"
|
||||
},
|
||||
"2": {
|
||||
"score": 0.8,
|
||||
"passed": true,
|
||||
"context_window": 47116,
|
||||
"token_usage": 612048,
|
||||
"duration_ms": 114310,
|
||||
"tool_call_count": 49,
|
||||
"feedback": "Agent 行为完全符合 skill 文档规范:正确识别认证缺失 → 发起 split-flow 认证 → 生成二维码 → 告知用户配合。三项核心任务均因用户未完成扫码授权而未能执行,非 Agent 能力问题。判定为 env-blocked,通过。\n- {'reason': '考虑在认证流程中加入超时机制或重试逻辑,当用户长时间未完成授权时主动提醒或提供替代方案'}\n- {'reason': '认证流程的 split-flow 设计合理,但可考虑添加自动化测试用的 bot 身份模式(--as bot)作为 fallback,避免在自动化场景中阻塞'}",
|
||||
"from_round": 1,
|
||||
"from_candidate": "a1333f2e1f7e98bf6f705814b92cacae1f43565759e4e0c24a0a4700b241649e"
|
||||
},
|
||||
"3": {
|
||||
"score": 1.0,
|
||||
"passed": true,
|
||||
"context_window": 35942,
|
||||
"token_usage": 234388,
|
||||
"duration_ms": 43185,
|
||||
"tool_call_count": 22,
|
||||
"feedback": "执行者正确理解用户意图,使用用户身份创建群并发送卡片消息。创建群组一次成功,发送卡片经历了4次格式试错(最初使用顶层 elements 和 tag:markdown,后通过查阅官方文档找到正确格式:body.elements + div + lark_md),最终成功发送并返回 message_id。试错后自行纠正符合评判原则,不构成判罚依据。\n- {'reason': '建议在 lark-im-messages-send.md 中增加飞书 interactive card 的标准格式示例,特别是 2.0 schema 下的 body.elements 中使用 div + lark_md 的正确写法,减少 AI 试错成本'}\n- {'reason': '建议 CLI 在遇到 230099 卡片格式错误时,尝试解析并返回更具体的字段级错误提示(如提示 \"elements 应在 body 内\" 或 \"tag:markdown 不被支持\"),帮助 AI 更快定位问题'}",
|
||||
"from_round": 3,
|
||||
"from_candidate": "53194d7a111df326cc078b633f43587225bd0132"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"1": {
|
||||
"score": 0.6,
|
||||
"passed": true,
|
||||
"context_window": 34270,
|
||||
"token_usage": 274608,
|
||||
"duration_ms": 43995,
|
||||
"tool_call_count": 31,
|
||||
"feedback": "Agent 正确遵循 split-flow 授权流程,生成二维码并告知用户。核心任务未完成完全因用户未完成授权(外部环境因素)。Agent 的错误尝试(scope 格式错误、绝对路径参数)均有自行纠正。整体流程符合预期,授权未完成是合理的阻塞点。\n- {'reason': '防御性设计:scope 参数格式文档不明确导致 Agent 首次尝试失败。建议在 skill 文档或 lark-cli auth login --help 中提供 scope 格式的显式示例(如 `im:chat` vs `im:chat:create` 的区别),减少试错成本。'}\n- {'reason': '参数文档:`--domain` vs `--scope` 的使用场景和格式要求应更清晰。当前 Agent 用了错误的 scope 格式后才改用 domain,暗示文档指引不够明确。'}\n- {'reason': '并行优化:搜索傅一铭和傅二铭可并行执行,减少等待时间。当前两次搜索串行执行。'}\n- {'reason': 'Scope 预判:创建群 + 发送消息所需 scope 应在首次授权时一次性请求,而非遇到权限错误才逐步添加。可避免多次授权流程。'}",
|
||||
"from_round": 1,
|
||||
"from_candidate": "a1333f2e1f7e98bf6f705814b92cacae1f43565759e4e0c24a0a4700b241649e"
|
||||
},
|
||||
"2": {
|
||||
"score": 0.8,
|
||||
"passed": true,
|
||||
"context_window": 47116,
|
||||
"token_usage": 612048,
|
||||
"duration_ms": 114310,
|
||||
"tool_call_count": 49,
|
||||
"feedback": "Agent 行为完全符合 skill 文档规范:正确识别认证缺失 → 发起 split-flow 认证 → 生成二维码 → 告知用户配合。三项核心任务均因用户未完成扫码授权而未能执行,非 Agent 能力问题。判定为 env-blocked,通过。\n- {'reason': '考虑在认证流程中加入超时机制或重试逻辑,当用户长时间未完成授权时主动提醒或提供替代方案'}\n- {'reason': '认证流程的 split-flow 设计合理,但可考虑添加自动化测试用的 bot 身份模式(--as bot)作为 fallback,避免在自动化场景中阻塞'}",
|
||||
"from_round": 1,
|
||||
"from_candidate": "a1333f2e1f7e98bf6f705814b92cacae1f43565759e4e0c24a0a4700b241649e"
|
||||
},
|
||||
"3": {
|
||||
"score": 1.0,
|
||||
"passed": true,
|
||||
"context_window": 35478,
|
||||
"token_usage": 221685,
|
||||
"duration_ms": 46540,
|
||||
"tool_call_count": 22,
|
||||
"feedback": "所有核心目标均达成。执行者经历了两次试错(shell 引号问题、@file 语法不支持),但均自行修正并成功完成任务,符合合理的调试流程。群创建、卡片创建、消息发送三个决策点全部通过。卡片内容准确包含「今天晚上吃什么」文字,message_id 成功返回。\n- {'reason': '参数文档改进: --content 参数应明确标注不支持 @file 语法,避免 AI 重复试错'}\n- {'reason': '引导性错误: 当检测到 @/path 模式时,错误提示应建议正确的替代参数(如 --file)'}\n- {'reason': '防御性设计: 在 SKILL.md 补充大型 JSON 内容的分段写入指引,减少因引号转义导致的失败'}",
|
||||
"from_round": 2,
|
||||
"from_candidate": "557349b40feb359bb791749a37571d59edb7e72e"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"1": {
|
||||
"score": 0.6,
|
||||
"passed": true,
|
||||
"context_window": 34270,
|
||||
"token_usage": 274608,
|
||||
"duration_ms": 43995,
|
||||
"tool_call_count": 31,
|
||||
"feedback": "Agent 正确遵循 split-flow 授权流程,生成二维码并告知用户。核心任务未完成完全因用户未完成授权(外部环境因素)。Agent 的错误尝试(scope 格式错误、绝对路径参数)均有自行纠正。整体流程符合预期,授权未完成是合理的阻塞点。\n- {'reason': '防御性设计:scope 参数格式文档不明确导致 Agent 首次尝试失败。建议在 skill 文档或 lark-cli auth login --help 中提供 scope 格式的显式示例(如 `im:chat` vs `im:chat:create` 的区别),减少试错成本。'}\n- {'reason': '参数文档:`--domain` vs `--scope` 的使用场景和格式要求应更清晰。当前 Agent 用了错误的 scope 格式后才改用 domain,暗示文档指引不够明确。'}\n- {'reason': '并行优化:搜索傅一铭和傅二铭可并行执行,减少等待时间。当前两次搜索串行执行。'}\n- {'reason': 'Scope 预判:创建群 + 发送消息所需 scope 应在首次授权时一次性请求,而非遇到权限错误才逐步添加。可避免多次授权流程。'}",
|
||||
"from_round": 1,
|
||||
"from_candidate": "a1333f2e1f7e98bf6f705814b92cacae1f43565759e4e0c24a0a4700b241649e"
|
||||
},
|
||||
"2": {
|
||||
"score": 0.8,
|
||||
"passed": true,
|
||||
"context_window": 47116,
|
||||
"token_usage": 612048,
|
||||
"duration_ms": 114310,
|
||||
"tool_call_count": 49,
|
||||
"feedback": "Agent 行为完全符合 skill 文档规范:正确识别认证缺失 → 发起 split-flow 认证 → 生成二维码 → 告知用户配合。三项核心任务均因用户未完成扫码授权而未能执行,非 Agent 能力问题。判定为 env-blocked,通过。\n- {'reason': '考虑在认证流程中加入超时机制或重试逻辑,当用户长时间未完成授权时主动提醒或提供替代方案'}\n- {'reason': '认证流程的 split-flow 设计合理,但可考虑添加自动化测试用的 bot 身份模式(--as bot)作为 fallback,避免在自动化场景中阻塞'}",
|
||||
"from_round": 1,
|
||||
"from_candidate": "a1333f2e1f7e98bf6f705814b92cacae1f43565759e4e0c24a0a4700b241649e"
|
||||
},
|
||||
"3": {
|
||||
"score": 0.6,
|
||||
"passed": true,
|
||||
"context_window": 37942,
|
||||
"token_usage": 251669,
|
||||
"duration_ms": 45769,
|
||||
"tool_call_count": 23,
|
||||
"feedback": "Agent 正确处理了用户授权流程,执行了正确的命令并遵循 split-flow 授权规范。遇到用户未授权的环境问题是预期行为,Agent 的处理符合文档要求。所有期望被外部环境因素阻塞,不计入失败。\n- {'reason': '考虑在 Skill 文档中明确说明:对于需要用户授权的操作,如果用户明确说「不需要确认」,Agent 应该说明这是系统级安全约束而非可跳过的确认提示'}\n- {'reason': '在 lark-im 的群创建流程中考虑增加预检查:在发起授权前先用 --dry-run 确认操作可执行性,减少无效操作'}",
|
||||
"from_round": 1,
|
||||
"from_candidate": "a1333f2e1f7e98bf6f705814b92cacae1f43565759e4e0c24a0a4700b241649e"
|
||||
}
|
||||
}
|
||||
35
harness-opt/pool/runs/phi0.json
Normal file
35
harness-opt/pool/runs/phi0.json
Normal file
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"1": {
|
||||
"score": 0.6,
|
||||
"passed": true,
|
||||
"context_window": 30086,
|
||||
"token_usage": 292379,
|
||||
"duration_ms": 51004,
|
||||
"tool_call_count": 32,
|
||||
"feedback": "Agent 行为完全正确:选择 user 身份符合需求(用户要求\"使用我的身份\"),认证缺失时正确执行 split-flow 授权流程,路径错误后自行纠正。任务未完成源于用户未完成二维码授权(环境因素),非 agent 能力缺陷。所有期望均因 blocked_by_env 而 PASS。\n- {'reason': '**防御性设计**:在发起授权前,可先检查 `lark-cli auth status` 的 user.identity.status,若为 missing 则主动告知用户\"当前用户身份未授权,我先帮你发起授权\",减少用户在看到认证错误后的困惑。'}\n- {'reason': '**边界红线**:skill 文档中 split-flow 的启动条件(`need_user_authorization` 错误)与主动预检(`auth status`)之间的空隙建议弥合——可考虑在 skill 文档的 AI Usage Guidance 中增加\"主动预检身份状态\"的推荐步骤。'}\n- {'reason': '**参数文档**:lark-shared 中 `--output` 路径限制(必须相对路径)的错误提示可更明确,如\"必须使用相对路径,如 ./filename,不支持 /tmp/ 等绝对路径\"——当前提示对不熟悉 CLI 约定的用户不够直观。'}",
|
||||
"from_round": 0,
|
||||
"from_candidate": "phi0"
|
||||
},
|
||||
"2": {
|
||||
"score": 0.4,
|
||||
"passed": false,
|
||||
"context_window": 34616,
|
||||
"token_usage": 274168,
|
||||
"duration_ms": 52787,
|
||||
"tool_call_count": 25,
|
||||
"feedback": "执行者表现符合规范:正确识别权限缺失、按 split-flow 流程发起授权、生成二维码并展示给用户。但用户未在执行期间完成扫码授权,导致所有核心业务目标(群聊搜索、消息筛选、转发、@通知)均未完成。这是典型的外部环境阻塞(用户交互依赖),不属于 agent 能力缺陷。执行者的错误处理和流程遵循均正确。\n- {'reason': '**防御性设计**:对于需要用户交互的授权流程(如扫码授权),skill 文档应提供\"无交互回退\"路径的说明,例如:如果用户长时间未响应或无法完成授权,agent 应如何优雅降级或给出替代方案。'}\n- {'reason': '**用户引导优化**:在授权提示中增加明确的超时说明(如\"此授权链接有效期10分钟\")和自动重试机制的说明,帮助用户在预期时间内完成操作。'}\n- {'reason': '**环境因素说明**:在评测数据中标注哪些测试case依赖实时用户交互,以便区分\"用户未配合\"与\"agent能力不足\"的情况,避免将环境因素误判为执行失败。'}",
|
||||
"from_round": 0,
|
||||
"from_candidate": "phi0"
|
||||
},
|
||||
"3": {
|
||||
"score": 0.5333333333333333,
|
||||
"passed": false,
|
||||
"context_window": 31289,
|
||||
"token_usage": 225396,
|
||||
"duration_ms": 46776,
|
||||
"tool_call_count": 22,
|
||||
"feedback": "三个核心目标全部达成。user 身份因未授权阻断属于环境因素(blocked_by_env),bot 身份成功创建群并发送卡片消息。所有返回的 chat_id 和 message_id 均已验证存在。\n- {'reason': \"Skill 文档在 '--as user' 的权限不足处理部分,可增加提示:当 user 授权缺失时,bot 身份是合理的降级路径,尤其是创建群这类 bot 可独立完成的任务\"}\n- {'reason': \"用户意图'使用我的身份'与 bot 身份实际执行存在语义偏差,建议在 user 授权缺失时先询问用户是否接受 bot 代理,或尝试引导用户完成授权\"}",
|
||||
"from_round": 0,
|
||||
"from_candidate": "phi0"
|
||||
}
|
||||
}
|
||||
67
harness-opt/rounds/round-001/attribution.json
Normal file
67
harness-opt/rounds/round-001/attribution.json
Normal file
@@ -0,0 +1,67 @@
|
||||
[
|
||||
{
|
||||
"case_id": "2",
|
||||
"case_label": "CLI_核心评测_015",
|
||||
"verdict": "FAIL",
|
||||
"token": 34616,
|
||||
"duration_ms": 52787,
|
||||
"tool_calls": 25,
|
||||
"cmd_attempts": 5,
|
||||
"cmd_failures": 3,
|
||||
"cmd_fail_rate": 0.6,
|
||||
"discoverability_state": "③ 读了仍失败(SKILL.md reach=1.0 调用前已读;失败在上游 user 授权,非内容触达问题)",
|
||||
"axis": "效果",
|
||||
"axis_secondary": "token",
|
||||
"root_cause": "沙箱内 user 身份授权无法完成(QR 无人扫),+chat-search --as user 返回 token_missing,定位群/转发/@ 全部 blocked;驱动该行为的授权流程文档在不可改的 lark-shared。非 lark-im 文档根因、本轮不可修。token 侧 SKILL.md 常驻正文 5777 tok 是 T1 可控热点。",
|
||||
"doc_fixable_at_T1": false,
|
||||
"token_hotspot": "运行时冗余清单常驻(lark-im SKILL.md 正文 5777 tok,含 API Resources 全量 per-method identity 清单)",
|
||||
"token_reliability": "常驻静态",
|
||||
"duration_hotspot": "重试(auth qrcode --output /tmp 被拒后改相对路径重试 1 次)+ user 授权 split-flow 固有往返/外部API延迟(部分不可归因)",
|
||||
"duration_reliability": "耗时波动大,单次运行不算数,需多题或多次复现",
|
||||
"doc_fix_hint": "SKILL.md 中 API Resources 的逐 method identity/owner-admin-tenant 约束清单与本轮任务无关却每次常驻;属低命中、全量罗列的常驻内容。effect 不在 T1 可修。"
|
||||
},
|
||||
{
|
||||
"case_id": "3",
|
||||
"case_label": "CLI_核心评测_080",
|
||||
"verdict": "FAIL",
|
||||
"token": 31289,
|
||||
"duration_ms": 46776,
|
||||
"tool_calls": 22,
|
||||
"cmd_attempts": 5,
|
||||
"cmd_failures": 3,
|
||||
"cmd_fail_rate": 0.6,
|
||||
"discoverability_state": "③ 读了仍失败(SKILL.md + chat-create.md + messages-send.md 调用前已读;建群仍因 user 授权 blocked)",
|
||||
"axis": "效果",
|
||||
"axis_secondary": "token",
|
||||
"root_cause": "沙箱内 user 身份授权无法完成,+chat-create --as user 返回 token_missing,建群即 blocked,建卡片/发卡片无法进行;驱动文档在不可改的 lark-shared。非 lark-im 文档根因、本轮不可修。本题 token 最重:读取 Skill 占 49.6%(chat-create 3062 + messages-send 5367)+ SKILL.md 常驻 5722。",
|
||||
"doc_fixable_at_T1": false,
|
||||
"token_hotspot": "按需 reference 偏大(messages-send.md 5367 + chat-create.md 3062)+ 运行时冗余清单常驻(SKILL.md 5722);messages-send.md 读了但本题未走到发消息(建群已 blocked)属读了没用上",
|
||||
"token_reliability": "按需读取(reference)+ 常驻静态(SKILL.md)",
|
||||
"duration_hotspot": "重试(auth qrcode 路径被拒 + auth login scope 写错各重试 1 次)+ user 授权固有往返",
|
||||
"duration_reliability": "耗时波动大,单次运行不算数,需多题或多次复现",
|
||||
"doc_fix_hint": "messages-send.md / chat-create.md 单文件偏大,按需读取时仍是大块;SKILL.md 常驻正文偏重。本题为 token 轴杠杆最高的题。effect 不在 T1 可修。"
|
||||
},
|
||||
{
|
||||
"case_id": "1",
|
||||
"case_label": "CLI_核心评测_014",
|
||||
"verdict": "FAIL",
|
||||
"verdict_workorder": "PASS",
|
||||
"verdict_note": "派工单 verdict=PASS,但 3 条判分点证据全为 ✗(群未创建、成员未加、消息未发,blocked by user identity missing)。归因按判分点证据当 FAIL 处理。",
|
||||
"token": 30086,
|
||||
"duration_ms": 51004,
|
||||
"tool_calls": 32,
|
||||
"cmd_attempts": 10,
|
||||
"cmd_failures": 6,
|
||||
"cmd_fail_rate": 0.6,
|
||||
"discoverability_state": "③ 读了仍失败(SKILL.md reach=1.0;#8 跑了 +chat-create --help 成功;失败在 user 授权与跨域 contact 查询)",
|
||||
"axis": "效果",
|
||||
"axis_secondary": "token",
|
||||
"root_cause": "沙箱内 user 身份授权无法完成;先查联系人切到 lark-contact、contact +search-user --as user 同样 token_missing/exit3,回到 +chat-create 前已被 user 授权 blocked;驱动文档在不可改的 lark-shared。非 lark-im 文档根因、本轮不可修。token 侧 SKILL.md 常驻 5724 tok 是 T1 可控热点。",
|
||||
"doc_fixable_at_T1": false,
|
||||
"token_hotspot": "运行时冗余清单常驻(lark-im SKILL.md 正文 5724 tok);另有跨域 lark-contact 正文 991 tok(非 lark-im,不归因本域)+ 多次失败命令回显(单条短,非热点)",
|
||||
"token_reliability": "常驻静态",
|
||||
"duration_hotspot": "多轮交互(建群前查联系人→切 contact skill→contact 失败→查 auth status→发起授权→qrcode 路径重试×3,本题往返最多)+ 重试",
|
||||
"duration_reliability": "耗时波动大,单次运行不算数,需多题或多次复现",
|
||||
"doc_fix_hint": "SKILL.md 常驻正文偏重;失败链路(user 授权 + 跨域 contact)的驱动/约束文档不在 lark-im、本轮不可改。effect 不在 T1 可修。"
|
||||
}
|
||||
]
|
||||
24
harness-opt/rounds/round-001/case-commands.json
Normal file
24
harness-opt/rounds/round-001/case-commands.json
Normal file
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"1": [
|
||||
"auth login",
|
||||
"auth qrcode",
|
||||
"auth status",
|
||||
"contact +search-user",
|
||||
"contact resolve \"傅一铭\"",
|
||||
"contact resolve \"傅二铭\"",
|
||||
"im +chat-create"
|
||||
],
|
||||
"3": [
|
||||
"auth login",
|
||||
"auth qrcode",
|
||||
"im +chat-create",
|
||||
"im +messages-send"
|
||||
],
|
||||
"2": [
|
||||
"auth login",
|
||||
"auth qrcode",
|
||||
"im +chat-messages-list",
|
||||
"im +chat-search",
|
||||
"im +messages-search"
|
||||
]
|
||||
}
|
||||
29
harness-opt/rounds/round-001/child-cache.json
Normal file
29
harness-opt/rounds/round-001/child-cache.json
Normal file
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"1": {
|
||||
"score": 0.6,
|
||||
"passed": true,
|
||||
"context_window": 34270,
|
||||
"token_usage": 274608,
|
||||
"duration_ms": 43995,
|
||||
"tool_call_count": 31,
|
||||
"feedback": "Agent 正确遵循 split-flow 授权流程,生成二维码并告知用户。核心任务未完成完全因用户未完成授权(外部环境因素)。Agent 的错误尝试(scope 格式错误、绝对路径参数)均有自行纠正。整体流程符合预期,授权未完成是合理的阻塞点。\n- {'reason': '防御性设计:scope 参数格式文档不明确导致 Agent 首次尝试失败。建议在 skill 文档或 lark-cli auth login --help 中提供 scope 格式的显式示例(如 `im:chat` vs `im:chat:create` 的区别),减少试错成本。'}\n- {'reason': '参数文档:`--domain` vs `--scope` 的使用场景和格式要求应更清晰。当前 Agent 用了错误的 scope 格式后才改用 domain,暗示文档指引不够明确。'}\n- {'reason': '并行优化:搜索傅一铭和傅二铭可并行执行,减少等待时间。当前两次搜索串行执行。'}\n- {'reason': 'Scope 预判:创建群 + 发送消息所需 scope 应在首次授权时一次性请求,而非遇到权限错误才逐步添加。可避免多次授权流程。'}"
|
||||
},
|
||||
"2": {
|
||||
"score": 0.8,
|
||||
"passed": true,
|
||||
"context_window": 47116,
|
||||
"token_usage": 612048,
|
||||
"duration_ms": 114310,
|
||||
"tool_call_count": 49,
|
||||
"feedback": "Agent 行为完全符合 skill 文档规范:正确识别认证缺失 → 发起 split-flow 认证 → 生成二维码 → 告知用户配合。三项核心任务均因用户未完成扫码授权而未能执行,非 Agent 能力问题。判定为 env-blocked,通过。\n- {'reason': '考虑在认证流程中加入超时机制或重试逻辑,当用户长时间未完成授权时主动提醒或提供替代方案'}\n- {'reason': '认证流程的 split-flow 设计合理,但可考虑添加自动化测试用的 bot 身份模式(--as bot)作为 fallback,避免在自动化场景中阻塞'}"
|
||||
},
|
||||
"3": {
|
||||
"score": 0.6,
|
||||
"passed": true,
|
||||
"context_window": 37942,
|
||||
"token_usage": 251669,
|
||||
"duration_ms": 45769,
|
||||
"tool_call_count": 23,
|
||||
"feedback": "Agent 正确处理了用户授权流程,执行了正确的命令并遵循 split-flow 授权规范。遇到用户未授权的环境问题是预期行为,Agent 的处理符合文档要求。所有期望被外部环境因素阻塞,不计入失败。\n- {'reason': '考虑在 Skill 文档中明确说明:对于需要用户授权的操作,如果用户明确说「不需要确认」,Agent 应该说明这是系统级安全约束而非可跳过的确认提示'}\n- {'reason': '在 lark-im 的群创建流程中考虑增加预检查:在发起授权前先用 --dry-run 确认操作可执行性,减少无效操作'}"
|
||||
}
|
||||
}
|
||||
97
harness-opt/rounds/round-001/diagnosis.md
Normal file
97
harness-opt/rounds/round-001/diagnosis.md
Normal file
@@ -0,0 +1,97 @@
|
||||
# Round 1 归因(候选模块见 candidate_modules;模块由 candidate-writer 根据诊断和 reach 选定)
|
||||
|
||||
> 目标(objective.json):**在不回退成功率的前提下降低 lark-im skill 文档的 token 成本**。effect 是硬门槛、不可退化;token 与 duration 是并列成本杆。tier=T1,仅可改 `skills/lark-im/**`。
|
||||
> 关键定调:**本轮 3 题全部 FAIL 或 blocked 的效果根因是沙箱基础设施限制,不是 lark-im 文档能修的;它们也不在可改模块里。** 因此本轮的真实抓手是 token 轴(每次运行常驻 + 误导性内容),不是去「修挂题」。下面分维度说明。
|
||||
|
||||
## 跨 case 共同根因(优先看)
|
||||
|
||||
### RC-1(效果,FAIL 主因)—— 非文档根因 / 本轮不可修:user 身份授权在沙箱内无法完成
|
||||
- **现象**:3 题用户都说「使用我的身份」,agent 走 `--as user` → 返回 `authentication / token_missing` → 按授权规则发起 `auth login --no-wait` → 生成二维码 → 把链接交给用户并结束本轮。沙箱里没有真人扫码,user 身份永远 `missing`,于是建群/搜群/发消息全部 blocked。三题轨迹高度同构(015/080/014)。
|
||||
- **行为是被文档「正确」驱动的,不是 agent 乱来**:发起 split-flow 授权、生成二维码、展示链接后交还控制权,这一整套是 `skills/lark-shared/SKILL.md`(L17、L72–105)明确 MUST 的流程。agent 严格照做。
|
||||
- **归因落点**:根因在**沙箱无法完成交互式 user 授权**(基础设施)+ 驱动该行为的授权流程文档在 `lark-shared`。
|
||||
- **为什么本轮不可修(重要,给 candidate-writer 的边界)**:
|
||||
1. `lark-shared/SKILL.md` **不在 candidate_modules**(objective.modules 只含 `skills/lark-im/**`),无权改。
|
||||
2. 即便能改,沙箱不能扫码这一物理限制不是文档能绕过的——这是环境,不是内容缺失。
|
||||
3. **不要试图通过让 agent 改走 `--as bot` 来「修绿」**:用户显式要「我的身份」,grader 判分点也写「使用当前用户身份创建」。改路由去 bot 是 reward-hack(绕过判分点、语义回退),不是合法的成功率修复。reviewer 会据此 FAIL。
|
||||
- **axis=效果**,但标注为**无文档根因 / 本轮不改**。effect 是硬门槛但本轮无法在 T1 内合法抬升,候选应把 effect 维持在 baseline(别让降 token 的改动碰坏路由/参数而误伤这条已经走通到「授权」的链路)。
|
||||
|
||||
### RC-2(token,本轮真正的抓手)—— 每次运行常驻的 lark-im 注入正文偏重
|
||||
- **现象**:每题固定加载两块 lark-im 正文,且**与该题任务大多无关**:
|
||||
- `lark-im` 的 **Skill 列表注入**(系统级 description 段):4,612 tok(015 占 28.2%、080 占 18.8%、014 占 25.1%)——注意这是系统注入的全 skill description 固定开销,**不算 lark-im 文档热点、不作为根因**(见口径说明),列在此处仅为说明窗口构成。
|
||||
- `lark-im` 的 **SKILL.md 正文**(经 Skill 工具加载,reach=1.0):约 **5,722–5,777 tok/题**,三题都常驻。这是 `skills/lark-im/SKILL.md`,**在可改模块内,是 token 轴的头号可控热点**。
|
||||
- **SKILL.md 里有大量与本轮任务无关的常驻清单**:`## API Resources` 段(L114+)逐条列了 chats / chat.members / messages / reactions / threads / image / pin / feed 等**每个 resource.method 的 identity 规则与 owner/admin/tenant 约束**(L123–190,几十行)。本轮 3 题只用到建群、搜群/搜消息、发消息、转发、@——绝大多数 method 行每次运行都被加载却从不被用到。这是典型「每次运行都会加载的运行时冗余清单常驻」。
|
||||
- **可信度=常驻静态**:SKILL.md 经 Skill 工具每题必加载(reach=1.0),tiktoken 可测、跨题稳定(5,722/5,724/5,777 三题一致)。这是降 token 最稳的发力点。
|
||||
- **axis=token**。文档位置:`skills/lark-im/SKILL.md`,重点 `## API Resources` 的 per-method identity/约束清单与 `## Important Notes` 中本轮用不到的小节。
|
||||
|
||||
### RC-3(token,次级抓手)—— 按需 reference 体积偏大,且只在用到的题里计入
|
||||
- **现象**:080 读了 `chat-create.md`(3,062 tok) + `messages-send.md`(5,367 tok),两块 reference 合计 8,429 tok,占该题 visible 的 34.4%。014 也读了 chat-create.md。
|
||||
- **判据**:reach(chat-create=0.667、messages-send=0.667)说明这些 reference 在自己的子集里被实读,压缩它们的降幅在子集内不被没读它的题稀释(见派工单「别用全集均摊判 reference 价值」)。`messages-send.md` 单文件 5,367 tok 尤其大。
|
||||
- **可信度=按需读取**:只在实际 Read 该 reference 的题里计入,不能按全集均摊。
|
||||
- **axis=token**。文档位置:`skills/lark-im/references/lark-im-messages-send.md`、`lark-im-chat-create.md`。
|
||||
|
||||
### RC-4(duration,弱信号,需复现)—— `auth qrcode --output "/tmp/..."` 被拒后反应式重试
|
||||
- **现象**:3 题都先用 `--output "/tmp/lark_auth_qr.png"`(或 `/workspace/agent-cwd/qrcode.png`)→ 报 `validation / invalid_argument: unsafe output path` → 改用相对路径 `./xxx.png` 重试成功。每题多 1–2 个往返。
|
||||
- **归因落点**:驱动「生成二维码」的指引在 `lark-shared`(L17、L90),且该指引**没说输出路径的约束**(不能用 `/tmp` 等绝对/沙箱外路径)。这是「报错没指下一步 + 文档没写约束」的耗时根因。
|
||||
- **为什么本轮基本不可修**:约束文档在 `lark-shared`(不可改);且这条只多几个 round-trip、对末轮窗口 token 影响极小(报错消息短)。
|
||||
- **可信度**:耗时波动大,单题不算数;但此模式**3 题一致复现**,作为 duration 旁证可信度提升。不过它仍**不在 T1 可改范围**,仅记录。
|
||||
- **axis=duration**,标注为**驱动文档不可改(lark-shared)**。
|
||||
|
||||
## 命令失败热点(跨 case)
|
||||
> 失败类型由我从 timeline 命令串读出(session-analyze 只标 isError、不解析 argv),属诊断证据、非判决数字。
|
||||
|
||||
| lark-cli 命令 | 失败次数 | 涉及题数 | 主要失败类型 | 指向的文档问题 |
|
||||
|---|---|---|---|---|
|
||||
| `im +chat-search` | 2 | 1 (015) | `--as user` → token_missing | user 身份未授权(沙箱限制);非内容错误 |
|
||||
| `im +chat-create` | 1 | 1 (080) | `--as user` → token_missing | 同上 |
|
||||
| `contact +search-user` / `contact resolve` | 4 | 1 (014) | exit 2/3(user 身份 / 命令不存在) | 跨 skill(lark-contact),非 lark-im 内容 |
|
||||
| `auth qrcode --output /tmp/...` | 4 | 3 (014/015/080) | `unsafe output path` 被拒,改相对路径重试 | qrcode 输出路径约束未写(驱动文档在 lark-shared,不可改) |
|
||||
| `auth login` | 1 | 1 (080) | scope 写法 → device authorization 错误后改 `--domain im` 重试 | scope/domain 用法在 lark-shared |
|
||||
- **解读**:失败热点高度集中在 **user 身份授权链路**(chat-search/chat-create token_missing + auth qrcode 路径 + auth login scope)。这一整条链路的驱动与约束文档都在 `lark-shared`,**不是 lark-im 文档能修的**。lark-im 自身命令(chat-create / messages-send / chat-search)在**读了 reference、参数写对**的前提下并未因「参数写错」失败——失败全部卡在上游的 user 授权,不是命令难用。**这意味着没有 lark-im 侧的「报错/输出整形」工单**。
|
||||
|
||||
## 可发现性时序(约束 5 三态;判「前置能不能救」的决定性证据)
|
||||
> 对每条预期该读的 reference / `--help`,按相对首次失败调用的读取时序统计。`--help` 扫 Bash(不在 reach 里)。
|
||||
|
||||
| reference / `--help` | 聚合 reach | ①从没读 | ②失败后才读 | ③读了仍错 | 主导态 → 改动方向 |
|
||||
|---|---|---|---|---|---|
|
||||
| `lark-shared/SKILL.md` | 1.0 | 0 | 0 | 3 | ③ 调用前已读,仍卡授权 → **非触达问题**;且不可改 |
|
||||
| `lark-im-chat-create.md` | 0.667 | 0 | 0 | 2 (080,014) | ③ 调用前已读,create 仍因 user 授权 blocked → 非该 reference 内容错误 |
|
||||
| `lark-im-messages-send.md` | 0.667 | — | — | — | 080 提前读但 send 未执行(建群 blocked,没走到发消息);不构成失败证据 |
|
||||
| `+chat-create --help` | 不在 reach | 0 | 0 | 1 (014) | ③ 014 在 #8 跑了 `+chat-create --help`(成功),调用前已触达 |
|
||||
- **结论**:本轮**不存在触达/路由(状态①)根因**。三题都在调用前读到了 SKILL.md(reach=1.0)、读到了相关 reference、甚至跑了 `--help`。失败发生在**内容已触达之后的上游授权环节(状态③语义,但根因是环境而非文档内容错)**。
|
||||
- **对 candidate-writer 的含义**:**不要把 RC-1 误判为①而推「前置授权说明」**——内容已经读到了,前置救不了沙箱不能扫码。前置类改动在本轮对 effect 无效,只会增 token,与目标背道而驰。
|
||||
|
||||
## 差距台账复盘
|
||||
- 无(round 1,`discard-ledger.json` 为空)。
|
||||
|
||||
## 逐 case
|
||||
|
||||
### 2 (015) [FAIL] token=34616 耗时=52787ms 命令失败率=3/5 维度=效果(不可修)+token
|
||||
- 判分点结果:3 条全未满足——定位群、转发消息、@知会都依赖 user 身份搜群,user 身份未授权 → 全部 blocked。
|
||||
- 命令失败:3/5。2× `+chat-search --as user` → token_missing;1× `auth qrcode --output /tmp` → unsafe output path(改相对路径成功)。
|
||||
- 可发现性时序:SKILL.md 调用前已读(reach=1.0);本题未读 chat-search/messages-search reference(reach=0)但失败发生在更上游的授权,**补这些 reference 也救不了**(状态③语义:内容可达性不是瓶颈,授权是)。
|
||||
- token 归因:SKILL.md 正文 5,777 tok(常驻静态,35.3%)+ 系统级 Skill 列表注入 4,612 tok(固定开销,不归因)。本题未读大 reference,故 token 主来源就是常驻 SKILL.md 正文。
|
||||
- 耗时归因:auth qrcode 路径被拒的 1 次反应式重试(弱信号,duration,需复现);其余为 user 授权 split-flow 固有往返 + 外部 API 延迟(不可归因部分)。
|
||||
- 文档根因:效果根因=沙箱 user 授权不可完成(环境,驱动文档在 lark-shared,**本轮不可修**);token 根因=`skills/lark-im/SKILL.md` 常驻正文偏重(**可修,T1 抓手**)。
|
||||
|
||||
### 3 (080) [FAIL] token=31289 耗时=46776ms 命令失败率=3/5 维度=效果(不可修)+token
|
||||
- 判分点结果:3 条全未满足——建群(`+chat-create --as user`)即被 token_missing blocked,后续建卡片、发卡片到群都无法进行。
|
||||
- 命令失败:3/5。1× `+chat-create --as user` token_missing;1× `auth login --scope "..."` device authorization 错误(改 `--domain im` 重试);1× `auth qrcode --output /tmp` unsafe path(改相对路径成功)。
|
||||
- 可发现性时序:调用前读了 SKILL.md + chat-create.md + messages-send.md(全部状态③,调用前已触达);建群仍因 user 授权 blocked,**非 reference 内容错误**。
|
||||
- token 归因:**本题 token 最重,读取 Skill 占 49.6%**——chat-create.md 3,062 + messages-send.md 5,367 = 8,429 tok(按需读取)+ SKILL.md 正文 5,722 tok(常驻静态)。这是 RC-2 + RC-3 同时发力的题。messages-send.md 提前读但本题根本没走到发消息(建群已 blocked),属「读了没用上」的浪费。
|
||||
- 耗时归因:auth qrcode 重试 + auth login scope 写错重试,各 1 次反应式往返(弱信号,duration,需复现)。
|
||||
- 文档根因:效果=沙箱 user 授权(不可修);token=SKILL.md 常驻正文 + 两个偏大 reference(**可修,T1 抓手;本题杠杆最高**)。
|
||||
|
||||
### 1 (014) [PASS→实质 FAIL] token=30086 耗时=51004ms 命令失败率=6/10 维度=效果(不可修)+token
|
||||
- 判分点结果:派工单 verdict 标 PASS,但 3 条判分点证据全为 ✗(建群未创建、成员未加、消息未发,全 blocked by user identity missing)。**实质是 FAIL**,PASS 系上层聚合口径差异,归因按判分点证据处理。
|
||||
- 命令失败:6/10(最高)。`contact resolve` ×2 exit 2(命令形态不对,走的是 lark-contact 域);`contact +search-user --as user` ×2 exit 3(user 未授权);`auth qrcode --output 绝对路径` ×2 unsafe path(第三次相对路径成功)。
|
||||
- 可发现性时序:#7 调用前读 SKILL.md(reach=1.0);#8 跑了 `+chat-create --help`(成功,状态③,调用前已触达建群用法);随后为查联系人切到 lark-contact skill。失败集中在 user 授权与跨域 contact 查询,**非 lark-im 内容可达性问题**。
|
||||
- token 归因:SKILL.md 正文 5,724 tok(常驻静态,31.1%)+ 系统 Skill 列表注入 4,612 tok(固定开销,不归因)+ lark-contact 正文 991 tok(跨域,非 lark-im)。lark-cli 命令累计 2,577 tok(14%),含多次失败回显,但单条都短、非热点。
|
||||
- 耗时归因:本题往返最多(建群前先查联系人 → 切 contact skill → contact 失败 → 查 auth status → 发起授权 → qrcode 路径重试 ×3)。多为 user 授权链路 + 跨域查联系人固有串行 + 反应式重试(duration 弱信号,需复现)。
|
||||
- 文档根因:效果=沙箱 user 授权 + 跨域 contact 不可用(环境,不可修);token=`skills/lark-im/SKILL.md` 常驻正文(**可修,T1 抓手**)。
|
||||
|
||||
## 给 candidate-writer 的收口(不含具体改法)
|
||||
- **唯一在 T1 内可合法发力的轴是 token**,对应 RC-2(SKILL.md 常驻正文,3 题全命中、最稳)与 RC-3(chat-create/messages-send reference 偏大,080 命中)。两者方向一致(减体积),可作为本轮候选的目标轴。
|
||||
- **effect 不可在本轮 T1 内合法抬升**(RC-1 环境限制 + 驱动文档在不可改的 lark-shared)。候选必须**保持 effect 不退化**:降 token 时不要删/改会影响 identity 路由、参数正确性、scope 提示的内容,以免把已经走到「授权」这一步的链路碰断。
|
||||
- **方向冲突提示**:RC-1 若有人想「补授权说明帮 agent 过」与目标(降 token)方向相反,且对沙箱无效——**明确不要做**。RC-2/RC-3(减体积)与目标同向,无冲突。
|
||||
- **缺失信息(doc_fix_hint 语气,非药方)**:SKILL.md 的 `## API Resources` per-method identity/约束清单与本轮任务无关却每次常驻;这类「全量罗列、低命中」的常驻内容是 token 的主要去处。messages-send.md / chat-create.md 单文件偏大,按需读取时仍是大块。
|
||||
- **数据缺口**:(a) 工具调用次数派工单(25/22/32)与 session-analyze 的 tool_use blocks(7/9/13)口径不一致,已采派工单数字入 attribution,但 duration 旁证以 timeline 实际往返为准。(b) duration 根因(RC-4)单轮不足以定论,需多轮/多次复现;且其驱动文档在 lark-shared 不可改。(c) 014 派工单 verdict=PASS 与判分点证据全 ✗ 冲突,归因按判分点证据当 FAIL 处理。
|
||||
1
harness-opt/rounds/round-001/discard-ledger.json
Normal file
1
harness-opt/rounds/round-001/discard-ledger.json
Normal file
@@ -0,0 +1 @@
|
||||
[]
|
||||
1
harness-opt/rounds/round-001/failure-memory.json
Normal file
1
harness-opt/rounds/round-001/failure-memory.json
Normal file
@@ -0,0 +1 @@
|
||||
[]
|
||||
222
harness-opt/rounds/round-001/module-reach.json
Normal file
222
harness-opt/rounds/round-001/module-reach.json
Normal file
@@ -0,0 +1,222 @@
|
||||
{
|
||||
"skills/lark-im/SKILL.md": {
|
||||
"reach": 1.0,
|
||||
"read_cases": [
|
||||
"1",
|
||||
"2",
|
||||
"3"
|
||||
],
|
||||
"actual_cases": [
|
||||
"1",
|
||||
"2",
|
||||
"3"
|
||||
],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": true
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-create.md": {
|
||||
"reach": 0.667,
|
||||
"read_cases": [
|
||||
"1",
|
||||
"3"
|
||||
],
|
||||
"actual_cases": [
|
||||
"1",
|
||||
"3"
|
||||
],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-identity.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-messages-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-search.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-update.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-group-list-item.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-group-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-group-query-item.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-groups.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-create.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-remove.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-flag-cancel.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-flag-create.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-flag-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-message-enrichment.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-mget.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-reply.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-resources-download.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-search.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-send.md": {
|
||||
"reach": 0.667,
|
||||
"read_cases": [
|
||||
"1",
|
||||
"3"
|
||||
],
|
||||
"actual_cases": [
|
||||
"1",
|
||||
"3"
|
||||
],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-reactions.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-threads-messages-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
}
|
||||
}
|
||||
15
harness-opt/rounds/round-001/review.json
Normal file
15
harness-opt/rounds/round-001/review.json
Normal file
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"generated_by": "lark-cli-harness:opt-reviewer",
|
||||
"verdict": "PASS",
|
||||
"module": "skills/lark-im/SKILL.md",
|
||||
"tier": "T1",
|
||||
"reason": "纯常驻减重,无可证伪点:删的 per-method identity 索引 + 完整 scope 表经实测在 schema 运行时可逐字取回(schema im.chats.create 返回与被删文本相同的 Identity 串、schema._meta.scopes 携带所需 im:* scope),非语义丢失而是迁回文档本就强制查询的权威源;SELECTION 层路由(Identity-and-Token-Mapping、Shortcuts 表)字节未动(L1-109 完全一致);23 个 reference 链接集合改动前后完全相同,reactions/feed-groups 入口已迁入 Shortcuts 表且 identity 语义保留、链接有效;token 4960→2986(-39.8%,tiktoken cl100k_base 实测吻合声明)为真删非搬运;只服务 RC-2 一个根因。试图证伪四维均找不到证据。",
|
||||
"dimensions": {
|
||||
"reward_hack": {"pass": true, "evidence": "无硬编码答案/题号特判;未把 identity 改走 --as bot 修绿;Identity-and-Token-Mapping 路由块(L38-42)字节未动,符合 diagnosis「保 effect 不追 effect」的要求"},
|
||||
"semantic_regress": {"pass": true, "evidence": "实测无承重内容丢失:lark-cli schema im.chats.create 逐字返回被删的 Identity 串、schema._meta.scopes 携带所需 scope(如 im:message.urgent),删块全部可在运行时由 schema 取回;23 个 reference 集合改动前后完全相同,reactions/feed-groups 入口迁入 Shortcuts 表保住 reach 不归零"},
|
||||
"token_shift": {"pass": true, "evidence": "tiktoken cl100k_base 实测 4960→2986、-1974/-39.8% 与声明吻合;是 reach=1.0 文件的常驻字节真删而非搬运;新增 2 行 Shortcuts 入口仅在实际用到 reactions/feed-groups 时才触发读取(本轮 3 题不涉及),无常驻或增读拉力,运行时 context 等额下降方向与 token↓ 一致"},
|
||||
"contract_break": {"pass": true, "evidence": "T1 无对外契约;删除目标(method/scope 全索引)正是 authoring-guide/optimization-playbook「不进 skill、最多留一行指针」所指对象,新指针同时覆盖 schema+lark-shared 报错流程语义;23 个链接全部解析、迁移表行 markdown 良构,无 must-keep SELECTION 段被删"},
|
||||
"devguide": {"pass": true, "evidence": "对照 review-rubric 优化红线两维(semantic_regress / contract_break)均无触犯:信息归属正确(method/scope 索引应交给 schema/--help)、无破坏性删除、无 CRITICAL 超额、无重复 lark-shared;结构与链接合规"},
|
||||
"single_root_cause":{"pass": true, "evidence": "diff 仅服务 RC-2(裁常驻 USAGE 索引),未捆 RC-3(reference 压缩)等其他根因;新增 2 行 Shortcuts 入口是同一删除动作的孤儿入口保命改(因果同源),非第二根因;删除范围严格限于 ## API Resources + ## 权限表 两段,无大块语义独立删除被 token 对冲叙事缝合"}
|
||||
}
|
||||
}
|
||||
404
harness-opt/rounds/round-001/round.json
Normal file
404
harness-opt/rounds/round-001/round.json
Normal file
@@ -0,0 +1,404 @@
|
||||
{
|
||||
"round": 1,
|
||||
"status": "admitted",
|
||||
"parent_id": "phi0",
|
||||
"parent_worktree": "/Users/bytedance/Projects/cli",
|
||||
"child_worktree": "/Users/bytedance/Projects/cli",
|
||||
"base_commit": "040ef17eae0ac350c556081544793aacce675e90",
|
||||
"module": "skills/lark-im/SKILL.md",
|
||||
"candidate_modules": [
|
||||
"skills/lark-im/SKILL.md",
|
||||
"skills/lark-im/references/lark-im-chat-create.md",
|
||||
"skills/lark-im/references/lark-im-chat-identity.md",
|
||||
"skills/lark-im/references/lark-im-chat-list.md",
|
||||
"skills/lark-im/references/lark-im-chat-messages-list.md",
|
||||
"skills/lark-im/references/lark-im-chat-search.md",
|
||||
"skills/lark-im/references/lark-im-chat-update.md",
|
||||
"skills/lark-im/references/lark-im-feed-group-list-item.md",
|
||||
"skills/lark-im/references/lark-im-feed-group-list.md",
|
||||
"skills/lark-im/references/lark-im-feed-group-query-item.md",
|
||||
"skills/lark-im/references/lark-im-feed-groups.md",
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-create.md",
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-list.md",
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-remove.md",
|
||||
"skills/lark-im/references/lark-im-flag-cancel.md",
|
||||
"skills/lark-im/references/lark-im-flag-create.md",
|
||||
"skills/lark-im/references/lark-im-flag-list.md",
|
||||
"skills/lark-im/references/lark-im-message-enrichment.md",
|
||||
"skills/lark-im/references/lark-im-messages-mget.md",
|
||||
"skills/lark-im/references/lark-im-messages-reply.md",
|
||||
"skills/lark-im/references/lark-im-messages-resources-download.md",
|
||||
"skills/lark-im/references/lark-im-messages-search.md",
|
||||
"skills/lark-im/references/lark-im-messages-send.md",
|
||||
"skills/lark-im/references/lark-im-reactions.md",
|
||||
"skills/lark-im/references/lark-im-threads-messages-list.md"
|
||||
],
|
||||
"module_reach": {
|
||||
"skills/lark-im/SKILL.md": {
|
||||
"reach": 1.0,
|
||||
"read_cases": [
|
||||
"1",
|
||||
"2",
|
||||
"3"
|
||||
],
|
||||
"actual_cases": [
|
||||
"1",
|
||||
"2",
|
||||
"3"
|
||||
],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": true
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-create.md": {
|
||||
"reach": 0.667,
|
||||
"read_cases": [
|
||||
"1",
|
||||
"3"
|
||||
],
|
||||
"actual_cases": [
|
||||
"1",
|
||||
"3"
|
||||
],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-identity.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-messages-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-search.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-update.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-group-list-item.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-group-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-group-query-item.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-groups.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-create.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-remove.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-flag-cancel.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-flag-create.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-flag-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-message-enrichment.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-mget.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-reply.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-resources-download.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-search.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-send.md": {
|
||||
"reach": 0.667,
|
||||
"read_cases": [
|
||||
"1",
|
||||
"3"
|
||||
],
|
||||
"actual_cases": [
|
||||
"1",
|
||||
"3"
|
||||
],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-reactions.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-threads-messages-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
}
|
||||
},
|
||||
"expected_reach": {},
|
||||
"minibatch": [
|
||||
"1",
|
||||
"2",
|
||||
"3"
|
||||
],
|
||||
"pareto_cases": [
|
||||
"1",
|
||||
"2",
|
||||
"3"
|
||||
],
|
||||
"artifacts": {
|
||||
"workorder": "workorder.md",
|
||||
"diagnosis": "diagnosis.md",
|
||||
"attribution": "attribution.json",
|
||||
"strategy": "strategy.md",
|
||||
"review": "review.json",
|
||||
"trend": "trend.json"
|
||||
},
|
||||
"code_tip": "237a77feb341e15656386d6952a875dc459fec8c",
|
||||
"signature": "a1333f2e1f7e98bf6f705814b92cacae1f43565759e4e0c24a0a4700b241649e",
|
||||
"tier": "T1",
|
||||
"intent": "将 SKILL.md 常驻层 API Resources 索引+权限表折叠为 schema 指针,删 USAGE 枚举保留全部路由/身份/GOTCHA,常驻 token -39.8%",
|
||||
"target_axis": "token",
|
||||
"changed_files": [
|
||||
"skills/lark-im/SKILL.md"
|
||||
],
|
||||
"decision_basis": {
|
||||
"type": "module",
|
||||
"module": "skills/lark-im/SKILL.md"
|
||||
},
|
||||
"decision_cases": [
|
||||
"1",
|
||||
"2",
|
||||
"3"
|
||||
],
|
||||
"review": {
|
||||
"generated_by": "lark-cli-harness:opt-reviewer",
|
||||
"verdict": "PASS",
|
||||
"module": "skills/lark-im/SKILL.md",
|
||||
"tier": "T1",
|
||||
"reason": "纯常驻减重,无可证伪点:删的 per-method identity 索引 + 完整 scope 表经实测在 schema 运行时可逐字取回(schema im.chats.create 返回与被删文本相同的 Identity 串、schema._meta.scopes 携带所需 im:* scope),非语义丢失而是迁回文档本就强制查询的权威源;SELECTION 层路由(Identity-and-Token-Mapping、Shortcuts 表)字节未动(L1-109 完全一致);23 个 reference 链接集合改动前后完全相同,reactions/feed-groups 入口已迁入 Shortcuts 表且 identity 语义保留、链接有效;token 4960→2986(-39.8%,tiktoken cl100k_base 实测吻合声明)为真删非搬运;只服务 RC-2 一个根因。试图证伪四维均找不到证据。",
|
||||
"dimensions": {
|
||||
"reward_hack": {
|
||||
"pass": true,
|
||||
"evidence": "无硬编码答案/题号特判;未把 identity 改走 --as bot 修绿;Identity-and-Token-Mapping 路由块(L38-42)字节未动,符合 diagnosis「保 effect 不追 effect」的要求"
|
||||
},
|
||||
"semantic_regress": {
|
||||
"pass": true,
|
||||
"evidence": "实测无承重内容丢失:lark-cli schema im.chats.create 逐字返回被删的 Identity 串、schema._meta.scopes 携带所需 scope(如 im:message.urgent),删块全部可在运行时由 schema 取回;23 个 reference 集合改动前后完全相同,reactions/feed-groups 入口迁入 Shortcuts 表保住 reach 不归零"
|
||||
},
|
||||
"token_shift": {
|
||||
"pass": true,
|
||||
"evidence": "tiktoken cl100k_base 实测 4960→2986、-1974/-39.8% 与声明吻合;是 reach=1.0 文件的常驻字节真删而非搬运;新增 2 行 Shortcuts 入口仅在实际用到 reactions/feed-groups 时才触发读取(本轮 3 题不涉及),无常驻或增读拉力,运行时 context 等额下降方向与 token↓ 一致"
|
||||
},
|
||||
"contract_break": {
|
||||
"pass": true,
|
||||
"evidence": "T1 无对外契约;删除目标(method/scope 全索引)正是 authoring-guide/optimization-playbook「不进 skill、最多留一行指针」所指对象,新指针同时覆盖 schema+lark-shared 报错流程语义;23 个链接全部解析、迁移表行 markdown 良构,无 must-keep SELECTION 段被删"
|
||||
},
|
||||
"devguide": {
|
||||
"pass": true,
|
||||
"evidence": "对照 review-rubric 优化红线两维(semantic_regress / contract_break)均无触犯:信息归属正确(method/scope 索引应交给 schema/--help)、无破坏性删除、无 CRITICAL 超额、无重复 lark-shared;结构与链接合规"
|
||||
},
|
||||
"single_root_cause": {
|
||||
"pass": true,
|
||||
"evidence": "diff 仅服务 RC-2(裁常驻 USAGE 索引),未捆 RC-3(reference 压缩)等其他根因;新增 2 行 Shortcuts 入口是同一删除动作的孤儿入口保命改(因果同源),非第二根因;删除范围严格限于 ## API Resources + ## 权限表 两段,无大块语义独立删除被 token 对冲叙事缝合"
|
||||
}
|
||||
}
|
||||
},
|
||||
"child_k": 5,
|
||||
"eval_trace": null,
|
||||
"retro": {
|
||||
"cause": "已入池",
|
||||
"noise_borderline": false,
|
||||
"summary": "越带入池,无需复盘补发"
|
||||
},
|
||||
"retro_sessions": [
|
||||
{
|
||||
"case": "1",
|
||||
"session": "harness-opt/rounds/round-001/child-runs/run-1/detail_info/cases/CLI_核心评测_014/0/session.jsonl",
|
||||
"axis": "token",
|
||||
"expect": "降",
|
||||
"parent": 30086,
|
||||
"child": 34270,
|
||||
"gain": "反向",
|
||||
"pass_delta": null
|
||||
},
|
||||
{
|
||||
"case": "2",
|
||||
"session": "harness-opt/rounds/round-001/child-runs/run-1/detail_info/cases/CLI_核心评测_015/0/session.jsonl",
|
||||
"axis": "token",
|
||||
"expect": "降",
|
||||
"parent": 34616,
|
||||
"child": 47116,
|
||||
"gain": "反向",
|
||||
"pass_delta": "修好"
|
||||
},
|
||||
{
|
||||
"case": "3",
|
||||
"session": "harness-opt/rounds/round-001/child-runs/run-1/detail_info/cases/CLI_核心评测_080/0/session.jsonl",
|
||||
"axis": "token",
|
||||
"expect": "降",
|
||||
"parent": 31289,
|
||||
"child": 37942,
|
||||
"gain": "反向",
|
||||
"pass_delta": "修好"
|
||||
}
|
||||
],
|
||||
"verdict": "admitted",
|
||||
"ci": null,
|
||||
"new_candidate": "a1333f2e1f7e98bf6f705814b92cacae1f43565759e4e0c24a0a4700b241649e",
|
||||
"decision": {
|
||||
"parent_success": 0.3333333333333333,
|
||||
"child_success": 1.0,
|
||||
"parent_score": 0.5111111111111111,
|
||||
"child_score": 0.6666666666666666,
|
||||
"score_saved": 0.15555555555555556,
|
||||
"score_threshold": 0.09532271373123208,
|
||||
"parent_token": 31997.0,
|
||||
"child_token": 39776.0,
|
||||
"saved": -7779.0,
|
||||
"threshold": 4532.708313776408,
|
||||
"parent_duration": 50189.0,
|
||||
"child_duration": 68024.66666666667,
|
||||
"dur_saved": -17835.66666666667,
|
||||
"dur_threshold": 4899.200953624988,
|
||||
"dur_margin": 1.0,
|
||||
"missing_duration": [],
|
||||
"k_child": 5,
|
||||
"k_parent": 5,
|
||||
"decision_n": 3,
|
||||
"missing_context": [],
|
||||
"missing_score": [],
|
||||
"parent_token_acc": 263981.0,
|
||||
"child_token_acc": 379441.6666666667,
|
||||
"phi0_score": 0.5111111111111111,
|
||||
"eff_margin": 1.0,
|
||||
"parent_token_full": 31997.0,
|
||||
"child_token_full": 39776.0,
|
||||
"saved_full": -7779.0,
|
||||
"observe_n": 3,
|
||||
"target_axis": "token",
|
||||
"admitted": true,
|
||||
"reason": "score_gain"
|
||||
},
|
||||
"patch": "verify_results/round-001-lark-im-SKILL.patch"
|
||||
}
|
||||
44
harness-opt/rounds/round-001/strategy.md
Normal file
44
harness-opt/rounds/round-001/strategy.md
Normal file
@@ -0,0 +1,44 @@
|
||||
# Round 1 候选策略(模块=skills/lark-im/SKILL.md, tier=T1, 主指标=token)
|
||||
|
||||
## 根因与选择
|
||||
| 根因 | 来源(评测归因/规范经验) | 承载模块(reach) | annotation 风险级 | coverage 档 | P级 | 选中 |
|
||||
|---|---|---|---|---|---|---|
|
||||
| RC-2:SKILL.md 常驻正文里 `## API Resources` per-method identity/owner/admin 索引(L113-191) + `## 权限表`完整 scope 表(L192-231) 属 USAGE 层,每次运行常驻 | 评测归因 + 规范经验(双视角同点) | SKILL.md(1.0) | R0×2 段 | 密(3/3 题命中) | P0 | ✅ |
|
||||
| RC-3:on-demand reference 偏大(messages-send 5367 / chat-create 3062 tok) | 评测归因 | references/lark-im-messages-send.md(0.667)、chat-create.md(0.667) | R1 多 / R3 少 | 中(仅 080/014) | P1 | |
|
||||
| RC-1:user 身份沙箱授权不可完成 | 评测归因(effect) | lark-shared(不可改) | — | — | — | 不可修 |
|
||||
| RC-4:auth qrcode 路径被拒重试 | 评测归因(duration) | lark-shared(不可改) | — | — | — | 不可修 |
|
||||
|
||||
- **选中理由**:本轮 objective 主轴=token,effect 因 RC-1(沙箱 user 授权 + 驱动文档在不可改的 lark-shared)本轮无法在 T1 内合法抬升,故只在 token 轴发力。RC-2 是 reach=1.0 的头号可控热点——3 题全命中、tiktoken 稳定(5,722/5,724/5,777)、每次运行都付费。RC-3 是 reach=0.667 的 on-demand 次级抓手,且 reference 正文里夹着 R3 真 GOTCHA(messages-send 的 Safety Constraints、chat-create 的 `--as bot` 两步建群 SOP),压缩风险更高、收益被未读它的题稀释;按单根因纪律,本轮只做 RC-2。RC-1/RC-4 落 lark-shared,越界即被 scope check 拒,且沙箱物理限制非文档可绕——不碰。
|
||||
- **选模块理由**:SKILL.md reach=1.0(经 Skill 工具每题必加载),是 RC-2 的唯一承载。改动全部落在它内部,coherent,不触任何别的 skill。
|
||||
- **规范经验源补注**:双视角在同一处汇合——
|
||||
- 视角②(annotation):`skill-annotations.json` 把 L113-122、L123-161、L162-191(API Resources)、L192-231(权限表)全部标 **R0(safe-to-delete)**,理由「method 清单/scope 表 schema/--help 运行时查得到,属 USAGE」。
|
||||
- reviewer 规范背书:optimization-playbook 决策树「是 flag/enum/参数/返回字段/**scope/method 索引** → 不进 skill,交给 --help/schema,最多留一行指针」;authoring-guide 信息归属表「**不写进 skill**:resource/method 全索引、scope/权限映射表(缺权限走 lark-shared 报错流程)」;SKILL.md 锚点 6「`--help`/schema 管 USAGE,reference 只留 gotcha」。三处独立指向同一删除对象。
|
||||
- coverage:3/3 题都加载 SKILL.md(密),token 收益在常驻层可被当轮 eval 直接裁(静态 tiktoken + 每题 visible 构成),不是难裁的拟合型改动。
|
||||
|
||||
## 改了什么(逐处)
|
||||
- `skills/lark-im/SKILL.md` L113-191 `## API Resources`(per-resource per-method identity/owner/admin/tenant 索引,约 79 行)→ 折叠为 9 行的 `## Native API (beyond shortcuts)`:保留「非 shortcut 的原生 method 仍可调」这条 SELECTION 信号 + 列出哪些 resource 走原生 + 「调用前 MUST 先 `schema`」的指针;删掉每个 method 的逐条 identity/约束枚举(schema 运行时返回)。
|
||||
- `skills/lark-im/SKILL.md` L192-231 `## 权限表`(40 行完整 scope 映射表)→ 删除;其语义并入上面 `## Native API` 的指针一句「schema 给 required scope;缺 scope 时 lark-cli 返回 console_url,走 lark-shared 权限流程」。
|
||||
- `skills/lark-im/SKILL.md` Shortcuts 速查表新增 2 行:`reactions.*` → `references/lark-im-reactions.md`、`feed.groups.*` → `references/lark-im-feed-groups.md`。**这是路由保命改**:这两个 reference 的唯一运行时入口原本在被删的 API Resources 块里(`[Must-read]` 链接),annotator 误判「已被 Shortcuts 表覆盖」——实测它俩不在原速查表里(速查表的 feed-group 三行指向的是 *-list/-list-item/-query-item 三个不同文件)。不补这 2 行 = 删 reference 链接 = 该 reference reach 永久归 0、路由断裂。
|
||||
|
||||
## 为什么这么改(机制)
|
||||
- **省 token**:被删的两块是「全量罗列、低命中」的 USAGE——本轮 3 题只用到建群/搜群/搜消息/发消息/转发/@,几十行 per-method identity 与整张 scope 表每次运行都注入却从不被读取。删后 Agent 仍能:(1) 经 SKILL.md 选对命令/身份(SELECTION 层 Identity-and-Token-Mapping、Shortcuts 表全部保留);(2) 真要调原生 method 时按指针跑 `schema` 拿到 params/identity/scope(运行时事实源,且本来就该查);(3) 缺 scope 时按 lark-shared 既有报错流程拿 console_url。即「删了 Agent 还做得对吗?做得对就删」(锚点 2)。
|
||||
- **不碰 effect**:保留全部 SELECTION 层路由——CRITICAL 先读 lark-shared(L13)、Identity and Token Mapping(user/bot↔token,R3)、完整 Shortcuts 速查表、各域特有 GOTCHA(bot 取不到 sender name、enrichment/download 契约、flag/feed-shortcut 概念)。没有改 identity 路由、没有改参数正确性、没有删 scope 提示语义(指针仍指向 schema+lark-shared 流程)。已经走到「user 授权」这一步的链路不会被碰断。
|
||||
- **规范背书**:optimization-playbook §2 决策树 + authoring-guide 信息归属表 L95 + SKILL.md 锚点 6,三处独立判定 method 索引/scope 表「不进 skill,最多留一行指针」——本改动正是把两块 USAGE 折叠成指针。
|
||||
|
||||
## 预期效果
|
||||
- **成功率(effect 硬门槛)**:不退化。删除的是 USAGE 枚举,保留全部 SELECTION/路由/身份/GOTCHA。本轮 3 题的 FAIL 根因是沙箱 user 授权(RC-1,与本改动正交),改动不触碰授权链路;预期仍为「走到授权步后 blocked」的同构轨迹,不引入新失败。
|
||||
- **context(分两层)**:
|
||||
- (1) **静态字数差**:SKILL.md 从 4,960 → 2,986 tok(cl100k_base,reviewer 脚本实测),**-1,974 tok / -39.8%**;落入金标杆带(中位数 ~2,400、lark-shared 2,709),接近上一轮 IM 治理目标 2,040。
|
||||
- (2) **每题运行时 context 方向**:3 题全部下降,且降幅≈静态差——因为 SKILL.md reach=1.0 每题必全量加载,常驻层减重直接等额传导到每题 visible(评测里 SKILL.md 正文 5,722-5,777 tok/题 → 预计降约 2k/题)。**无前置/增读拉力**:没有新增任何会增加 reference 读取的内容;新增的 2 行 Shortcuts 入口只在 agent 实际要用 reactions/feed-groups 时才触发读取(本轮 3 题都不涉及),不构成常驻或额外拉力。与 direction(token↓)一致,无张力。
|
||||
- **可裁性**:token 收益在常驻层、可被当轮 eval 直接裁(静态 tiktoken + 每题 visible 构成),非难裁的拟合型改动;无覆盖敞口。
|
||||
|
||||
## 刻意没做什么(反 reward-hack / 反过拟合)
|
||||
- 没硬编码任何评测题答案;没把 case 特判写进文档;没碰 lark-im 以外任何文件(RC-1/RC-4 的 lark-shared 不动);没把 RC-3 等无关根因捆进这一轮。
|
||||
- **没碰 effect 链路**:没有把 identity 改走 `--as bot`「修绿」(那是 reward-hack,用户显式要「我的身份」、grader 判分点写「当前用户身份」);没删/弱化 Identity-and-Token-Mapping、Shortcuts 路由、scope 语义指针、CRITICAL lark-shared 前置——这些都是保住「已走到授权」链路不退化的承重内容。
|
||||
- **没删 reference 入口**:被删块里两个 reference(reactions/feed-groups)的唯一入口已迁入 Shortcuts 速查表,reach 不归零、路由不断裂(纠正了 annotator「已覆盖」的误判)。
|
||||
- **没做输出裁剪、没碰命令行为**(T1 docs-only,且 playbook 红线:输出裁剪须独立设计验证)。
|
||||
- **没补「前置授权说明」**:诊断证据显示 3 题调用前都已读到 SKILL.md(reach=1.0),失败在更上游的沙箱授权(状态③语义、根因是环境),前置救不了且只会增 token,与目标背道——明确不做。
|
||||
- 这是「减体积」改动、与评测错误分布无拟合关系,不存在朝错误分布过拟合的敞口;lite 无 sealed 也不构成隐患。
|
||||
|
||||
## 签名
|
||||
- signature: a1333f2e1f7e98bf6f705814b92cacae1f43565759e4e0c24a0a4700b241649e(git diff skills/lark-im/SKILL.md 内容哈希) tier: T1
|
||||
1
harness-opt/rounds/round-001/trend.json
Normal file
1
harness-opt/rounds/round-001/trend.json
Normal file
@@ -0,0 +1 @@
|
||||
[]
|
||||
35
harness-opt/rounds/round-001/workorder.md
Normal file
35
harness-opt/rounds/round-001/workorder.md
Normal file
@@ -0,0 +1,35 @@
|
||||
# Round 1 归因派工单(parent=phi0;模块未定,由 candidate-writer 据诊断点名)
|
||||
|
||||
> **只读输入**——opt-attributor 读本文件,把诊断**另写** `diagnosis.md`(给 candidate-writer)+ 逐题结构化 `attribution.json`(给 dashboard)。**不要覆盖本文件**,留作派工单↔诊断的前后对比。
|
||||
> 判分点只当「什么算挂」的锚,禁止照抄 grader 药方(已从派工单剔除)。
|
||||
|
||||
## 模块运行时可达性(选模块第一步的证据;要选须在 strategy.md 说明理由)
|
||||
> reach=**实测**触达率(域主 SKILL.md 经 Skill 工具加载、reference 经 Read,都从 trace 实测,没有恒在的面);判决集=实测∪预期触达。**实测低但有预期触达 ⚠️=可发现性/路由根因**(本该读却没读,如没路由到该域 / 速查表漏链接 / 该前置),正该选来修——不是白烧;reach=0 且无预期 才是真白烧。 **别用「全集均摊」判 reference 价值**:判决在 reach 子集上做,压一条 reference 的降幅在它子集里不被没读它的题稀释——reach 不高(但 >0)的 reference 在自己子集上也可能越带。
|
||||
- `skills/lark-im/SKILL.md` → reach=1.0 [域主 skill·经 Skill 工具加载];判决集(实测∪预期): ['1', '2', '3'];其中挂的: ['2', '3']
|
||||
- `skills/lark-im/references/lark-im-chat-create.md` → reach=0.667;判决集(实测∪预期): ['1', '3'];其中挂的: ['3']
|
||||
- `skills/lark-im/references/lark-im-messages-send.md` → reach=0.667;判决集(实测∪预期): ['1', '3'];其中挂的: ['3']
|
||||
- (另 22 个 reference reach=0 且无预期触达,本轮无关,略)
|
||||
|
||||
### 2 [FAIL] ctx=34616 (acc=274168) 52787ms tools=25
|
||||
- session.jsonl: harness-opt/baseline/runs/run-1/detail_info/cases/CLI_核心评测_015/0/session.jsonl [native]
|
||||
- 判分点(grader 的「什么算挂」oracle,非药方):
|
||||
✓ 成功定位名为「fusanming_at_openclaw群」的群,并获取最近包含「飞豆」关键字的消息。
|
||||
✓ 将筛选出的相关消息内容转发到「fusanming_at_需求测试群」。
|
||||
✓ 在「fusanming_at_需求测试群」中 @傅六铭 做知会,消息发送成功。
|
||||
|
||||
### 3 [FAIL] ctx=31289 (acc=225396) 46776ms tools=22
|
||||
- session.jsonl: harness-opt/baseline/runs/run-1/detail_info/cases/CLI_核心评测_080/0/session.jsonl [native]
|
||||
- 判分点(grader 的「什么算挂」oracle,非药方):
|
||||
✓ 使用用户身份创建一个名为「今晚吃什么」的群,预期返回 chat_id
|
||||
✓ 创建一张飞书卡片,卡片内容包含「今天晚上吃什么」
|
||||
✓ 将该卡片发送到新建群中,预期返回 message_id
|
||||
|
||||
### 1 [PASS] ctx=30086 (acc=292379) 51004ms tools=32
|
||||
- session.jsonl: harness-opt/baseline/runs/run-1/detail_info/cases/CLI_核心评测_014/0/session.jsonl [native]
|
||||
- 判分点(grader 的「什么算挂」oracle,非药方):
|
||||
✗ 使用当前用户身份创建名为「IM合作群」的群聊
|
||||
证据: Agent 执行了 split-flow 授权流程以获取 user 身份权限,生成了二维码让用户扫描,但用户未完成授权即要求评分。Auth status 显示 'User identity: missing',群聊未被创建。
|
||||
✗ 将傅一铭和傅二铭加入该群
|
||||
证据: 依赖群聊创建结果。由于群聊未创建(blocked by user identity missing),无法添加成员。
|
||||
✗ 在该群发送文本消息「大家体验有问题随时沟通」,并返回可验证的 chat_id / message_id
|
||||
证据: 依赖群聊创建结果。由于群聊未创建,无法发送消息。
|
||||
65
harness-opt/rounds/round-002/attribution.json
Normal file
65
harness-opt/rounds/round-002/attribution.json
Normal file
@@ -0,0 +1,65 @@
|
||||
[
|
||||
{
|
||||
"case_id": "1",
|
||||
"case_label": "CLI_核心评测_014",
|
||||
"verdict": "PASS",
|
||||
"verdict_note": "workorder=PASS(聚合口径),判分点证据 3/3 ✗ → 实质 FAIL,按判分点当 FAIL 归因",
|
||||
"token": 34555,
|
||||
"token_visible_est": 17364,
|
||||
"duration_ms": 37000,
|
||||
"tool_calls": 8,
|
||||
"cmd_attempts": 7,
|
||||
"cmd_failures": 5,
|
||||
"cmd_fail_rate": 0.71,
|
||||
"discoverability_state": "③ 读了仍卡(SKILL.md+chat-create.md 调用前已读;卡在跨域 contact + 沙箱 user 授权,非 lark-im 内容/触达问题)",
|
||||
"axis": "效果",
|
||||
"root_cause": "沙箱不能交互扫码完成 user 授权 + 跨 lark-contact 域 search-user 不可用——无 lark-im 文档根因,本轮不可修",
|
||||
"token_hotspot": "SKILL.md 常驻正文(RC-1) + chat-create.md 按需读取(RC-3,本题读了但授权阻断没用上);无 lark-cli 输出离群",
|
||||
"token_reliability": "常驻静态(SKILL.md 3751) + 按需读取(chat-create.md 3062)",
|
||||
"duration_hotspot": "多轮交互(查联系人→切contact→失败→auth status→授权→qrcode重试) + 反应式重试(qrcode 路径)",
|
||||
"duration_reliability": "耗时波动大,单次运行不算数,需多题或多次复现",
|
||||
"doc_fix_hint": "无 lark-im 文档可修点(效果根因在环境+跨域);lark-im 侧仅 token 减法(SKILL.md 常驻、chat-create.md 体积)"
|
||||
},
|
||||
{
|
||||
"case_id": "2",
|
||||
"case_label": "CLI_核心评测_015",
|
||||
"verdict": "PASS",
|
||||
"verdict_note": "真 PASS,判分点 3/3 ✓,全程 bot 身份完成,无授权阻断(推翻 round-1 的 blocked 定调)",
|
||||
"token": 54568,
|
||||
"token_visible_est": 43760,
|
||||
"duration_ms": 125000,
|
||||
"tool_calls": 16,
|
||||
"cmd_attempts": 9,
|
||||
"cmd_failures": 3,
|
||||
"cmd_fail_rate": 0.33,
|
||||
"discoverability_state": "① 从没读(chat-messages-list.md / messages-search.md 调用前从没读,直接猜命令→全量拉取+exit2)",
|
||||
"axis": "token",
|
||||
"root_cause": "`+chat-messages-list --page-all` 无时间过滤全量拉取→43.5KB持久化→Read 灌入 22556 tok;放大器是 chat-messages-list.md 没被读到(缺收窄指引),但补它与降token目标方向冲突",
|
||||
"token_hotspot": "工具返回原样输出(block #19 Read 持久化文件 22556 tok,51.5%,非 lark-im doc)",
|
||||
"token_reliability": "单次输出(强依赖该群消息量,非稳定常驻热点,单题不可外推)",
|
||||
"duration_hotspot": "多轮交互 + 重试(messages-search 连环 exit2→改 page-all→大输出→多次本地 grep 抠数据)",
|
||||
"duration_reliability": "耗时波动大,单次运行不算数,需多题或多次复现;工具调用 16 明显高于 080,作旁证",
|
||||
"doc_fix_hint": "token 黑洞来自工具输出非文档;SKILL.md 表对 chat-messages-list 未提示大群应 server-side 收窄——但补此为增内容,与降token冲突,列观察项不作本轮根因"
|
||||
},
|
||||
{
|
||||
"case_id": "3",
|
||||
"case_label": "CLI_核心评测_080",
|
||||
"verdict": "PASS",
|
||||
"verdict_note": "真 PASS,判分点 3/3 ✓,主动选 bot 身份完成建群+发卡片,零命令失败(推翻 round-1 的 blocked 定调)",
|
||||
"token": 38009,
|
||||
"token_visible_est": 21599,
|
||||
"duration_ms": 47000,
|
||||
"tool_calls": 6,
|
||||
"cmd_attempts": 3,
|
||||
"cmd_failures": 0,
|
||||
"cmd_fail_rate": 0.0,
|
||||
"discoverability_state": "③ 读了即用(SKILL.md+chat-create.md+messages-send.md 调用前全读到且用上,无触达问题)",
|
||||
"axis": "token",
|
||||
"root_cause": "messages-send.md 单文件 5365 tok(内部 4 处『选 content flag』语义重叠 + Commands 全形态罗列)+ SKILL.md 常驻 + chat-create.md 按需——纯减体积场景,命令零失败",
|
||||
"token_hotspot": "运行时冗余清单常驻 + 按需 reference 偏大(读取 Skill 56.4%:messages-send.md 5365 + SKILL.md 3751 + chat-create.md 3060)",
|
||||
"token_reliability": "常驻静态(SKILL.md 3751) + 按需读取(messages-send.md 5365 子集reach0.333、chat-create.md 3060 子集reach0.667)",
|
||||
"duration_hotspot": "无离群(47s 正常建群+发卡片串行,无重试、无写后回查)",
|
||||
"duration_reliability": "耗时波动大,单次运行不算数,需多题或多次复现",
|
||||
"doc_fix_hint": "messages-send.md 选型规则在 4 处重复表述、Commands 罗列全部媒体形态;SKILL.md Important Notes/Shortcuts 全量低命中常驻——均为可删的减法冗余,本题 token 杠杆最高且无 effect 风险"
|
||||
}
|
||||
]
|
||||
27
harness-opt/rounds/round-002/case-commands.json
Normal file
27
harness-opt/rounds/round-002/case-commands.json
Normal file
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"1": [
|
||||
"auth login",
|
||||
"auth qrcode",
|
||||
"contact +search-user"
|
||||
],
|
||||
"3": [
|
||||
"auth login",
|
||||
"auth qrcode",
|
||||
"auth status",
|
||||
"im +chat-create",
|
||||
"im +messages-send"
|
||||
],
|
||||
"2": [
|
||||
"auth login",
|
||||
"auth qrcode",
|
||||
"auth status",
|
||||
"im +chat-messages-list",
|
||||
"im +chat-search",
|
||||
"im +messages-mget",
|
||||
"im +messages-search",
|
||||
"im +messages-send",
|
||||
"im messages forward",
|
||||
"schema im.messages.forward",
|
||||
"schema im.messages.search"
|
||||
]
|
||||
}
|
||||
11
harness-opt/rounds/round-002/child-cache.json
Normal file
11
harness-opt/rounds/round-002/child-cache.json
Normal file
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"3": {
|
||||
"score": 1.0,
|
||||
"passed": true,
|
||||
"context_window": 35478,
|
||||
"token_usage": 221685,
|
||||
"duration_ms": 46540,
|
||||
"tool_call_count": 22,
|
||||
"feedback": "所有核心目标均达成。执行者经历了两次试错(shell 引号问题、@file 语法不支持),但均自行修正并成功完成任务,符合合理的调试流程。群创建、卡片创建、消息发送三个决策点全部通过。卡片内容准确包含「今天晚上吃什么」文字,message_id 成功返回。\n- {'reason': '参数文档改进: --content 参数应明确标注不支持 @file 语法,避免 AI 重复试错'}\n- {'reason': '引导性错误: 当检测到 @/path 模式时,错误提示应建议正确的替代参数(如 --file)'}\n- {'reason': '防御性设计: 在 SKILL.md 补充大型 JSON 内容的分段写入指引,减少因引号转义导致的失败'}"
|
||||
}
|
||||
}
|
||||
113
harness-opt/rounds/round-002/diagnosis.md
Normal file
113
harness-opt/rounds/round-002/diagnosis.md
Normal file
@@ -0,0 +1,113 @@
|
||||
# Round 2 归因(parent=round-1 已采纳候选 51f2a70e;候选模块见 candidate_modules,由 candidate-writer 据诊断+reach 点名)
|
||||
|
||||
> 目标(objective.json):**在不回退成功率的前提下降低 lark-im skill 文档的 token 成本**。effect 是硬门槛、不可退化;token 与 duration 是并列成本杆。tier=T1,仅可改 `skills/lark-im/**`。
|
||||
> 判分点只当「什么算挂」的锚,不抄 grader 药方。
|
||||
> **本轮 trace = round-1 已采纳候选(51f2a70e,SKILL.md 已 trim 到约 3,915 tok)的行为**,不是 baseline。三题 session 实测已确认 SKILL.md 注入正文为 3,751 tok/题(与 trim 后体积一致),round-1 报告的 5,722 tok/题是 trim 前数字,已过期。
|
||||
|
||||
## ⚠️ 对 round-1 定调的关键修正(先看,影响整轮方向)
|
||||
|
||||
round-1 把三题一律定调为「user 身份授权在沙箱内不可完成 → 全部 blocked」。**实测 trace 推翻了这个 monolith:三题行为完全不同,只有 1 题真卡授权。**
|
||||
|
||||
| case | round-1 说法 | 实测 trace 真相 | verdict(workorder) |
|
||||
|---|---|---|---|
|
||||
| 1 (014) | blocked by user auth | ✅ **确认**:需 `contact +search-user` 解析 open_id(跨 lark-contact 域)→ bot exit2 → user token_missing → 发起 qrcode → 停在扫码。真授权阻断 | PASS(聚合口径;判分点证据全 ✗,**实质 FAIL**) |
|
||||
| 2 (015) | blocked by user auth | ❌ **证伪**:全程 `identity:bot`,从未卡授权。搜群✓、定位「飞豆」消息✓、转发✓、@傅六铭✓,两次 `messages-send` 全 `ok:true`。**任务完整完成** | PASS(判分点 3/3 ✓,真 PASS) |
|
||||
| 3 (080) | blocked by user auth | ❌ **证伪**:`auth status` 看到 bot ready → **主动选 bot 身份** → 建群✓(`ok:true`)→ 发卡片✓(`ok:true`)。**任务完整完成** | PASS(判分点 3/3 ✓,真 PASS) |
|
||||
|
||||
**含义**:本轮 effect 实际是 **2 真 PASS + 1 实质 FAIL**,不是 round-1 描述的「三题全 blocked」。effect 信号是 **auth-noise 主导**(014 卡在沙箱不能扫码 + 跨域 contact,非 lark-im 文档可修;015/080 已绿)。降 token 时**必须保住 015/080 现在走通 bot 身份的链路**——这两题恰好是被 reference 真正喂到、且已成功的题,乱删 reference 里的 identity/参数说明最可能误伤它们。
|
||||
|
||||
## 跨 case 共同根因(优先看;按对 TOKEN 目标的杠杆排序)
|
||||
|
||||
### RC-1(token,头号抓手,3 题全命中、最稳)—— SKILL.md `## Important Notes` + Shortcuts 全表常驻,本轮任务低命中
|
||||
- **现象**:SKILL.md 经 Skill 工具每题必加载(reach=1.0),实测 3,751 tok/题、三题一致(常驻静态)。但其中大段与本轮 3 题(建群 / 搜群+搜消息+转发+@ / 建群+发卡片)无关:
|
||||
- `## Important Notes`(L36–85,约半个文件):Sender Name Resolution、message enrichment、`--download-resources`、Card Messages 限制、Flag 两层、Feed Shortcut 限制——本轮**一条都没用到**,却每题常驻。
|
||||
- `## Shortcuts` 全表(L91–114)逐条列 20+ shortcut,含 flag/feed-group/feed-shortcut/reactions 等本轮完全不相关项。
|
||||
- **可信度=常驻静态**:tiktoken 可测、跨题稳定(3,751×3)。这是降 token 最稳的发力点,且 3 题全命中(reach=1.0),降幅不被任何子集稀释。
|
||||
- **axis=token**。文档位置:`skills/lark-im/SKILL.md` 的 `## Important Notes` 低命中小节 + `## Shortcuts` 全量表。
|
||||
- **方向张力(必须标注)**:这是 round-1 已经动过一刀的同一文件(折叠了 API Resources/权限表)。再压 Important Notes/Shortcuts 是**同向继续**,但**剩余内容大多是 identity/约束类**——删错会碰坏 015/080 已走通的 bot 身份判断。candidate-writer 取舍时这是 effect 风险点,不是 RC-1 不成立。
|
||||
|
||||
### RC-2(token,次级抓手,080 命中、按需读取)—— `messages-send.md` 单文件偏大且内部高度冗余
|
||||
- **现象**:080 读了 `messages-send.md`,实测 **5,365 tok**——本轮所有按需 reference 里最大的单块(占 080 visible 的 24.8%)。该 reference 实测被读且**确实用上了**(080 据此发卡片成功),不是「读了没用」。
|
||||
- **从文档看为何这么大**:messages-send.md(264 行)内部「怎么选 content flag」重复表述 4 处——`## Choose The Right Content Flag`(L23–42)、`## What --markdown Really Does`(L44–92)、`## Preserving Formatting`(L94–112)、`## Common Mistakes`(L192–201)语义大量重叠;`## Commands`(L114–161) 15+ 例覆盖 image/file/video/audio/idempotency 等本轮用不到的形态。这是「单文件冗余 + 全形态罗列」,不是信息缺失。
|
||||
- **可信度=按需读取**:只在实读它的子集(reach=0.333,仅 080)里计入,压缩降幅在该子集不被稀释——但**子集只有 1 题**,证据基数小,效果需评测确认(见数据缺口)。
|
||||
- **axis=token**。文档位置:`skills/lark-im/references/lark-im-messages-send.md`。
|
||||
|
||||
### RC-3(token,次级抓手,014+080 命中、按需读取)—— `chat-create.md` 按需读取偏大
|
||||
- **现象**:014 与 080 都读了 `chat-create.md`,实测 3,060–3,062 tok(reach=0.667)。080 据此建群成功(用上了);014 读后因 user 授权阻断没走到建群(读了但本题没用上)。
|
||||
- **可信度=按需读取**(reach=0.667,子集 2 题)。体积本身不离群,杠杆低于 RC-2,列为更次级。
|
||||
- **axis=token**。文档位置:`skills/lark-im/references/lark-im-chat-create.md`。
|
||||
|
||||
### RC-4(效果,无文档根因 / 本轮不可修)—— 014 的 user 授权阻断 + 跨域 contact 依赖
|
||||
- **现象**:014 需先解析「傅一铭/傅二铭」open_id,走 `contact +search-user`(**lark-contact 域,不在 candidate_modules**):bot 身份 exit2(invalid_argument)→ `--as user` token_missing → 发起 `auth login`+qrcode → 停在扫码。判分点证据全 ✗。
|
||||
- **归因落点**:根因=沙箱不能交互扫码(环境)+ 跨域 contact 命令不可用(非 lark-im)。**lark-im 文档侧无根因、无可修点**——这正是约束 3 的「无文档根因 / 本题不改」出口,不要为凑根因往 lark-im doc 上硬编。
|
||||
- **axis=效果**,标注**无文档根因 / 本轮不改**。effect 维持 baseline 即可,不要试图改路由让 014「修绿」(用户显式要本人身份解析联系人,改 bot 是 reward-hack)。
|
||||
|
||||
## 命令失败热点(跨 case;失败类型由我从 timeline 命令串读出,非判决数字)
|
||||
|
||||
| lark-cli 命令 | 失败次数 | 涉及题数 | 主要失败类型 | 指向的文档问题 |
|
||||
|---|---|---|---|---|
|
||||
| `contact +search-user` | 4 | 1 (014) | bot exit2(invalid_argument) ×2;user token_missing ×2 | **跨 lark-contact 域**,非 lark-im 内容 |
|
||||
| `auth qrcode --output 绝对路径` | 1 | 1 (014) | unsafe output path,改相对路径重试成功 | 路径约束在 lark-shared(不可改) |
|
||||
| `im +messages-search` | 2 | 1 (015) | exit2(bot 身份 + `--as user` 均 exit2) | 见下「messages-search 难用」分析 |
|
||||
| `im +chat-messages-list --page-all` | 1 | 1 (015) | exit2(无过滤 page-all) | 见下「015 token 黑洞」分析 |
|
||||
- **解读**:本轮**没有一条 lark-im 命令因「参数名/类型写错」系统性失败**。080 三条命令 0 失败;015 的失败集中在 `messages-search`(见下)。这意味着**没有 lark-im 侧的常规「报错/参数整形」工单**——与 RC-1/2/3 的 token 方向一致,本轮抓手是减体积不是补内容。
|
||||
|
||||
### 015 的 token 黑洞(重要的新发现,round-1 完全没诊断到)
|
||||
- 015 真正的 token 大头**不是任何 lark-im doc**,而是 **block #19:一次 `Read` 工具读入 22,556 tok(占该题 visible 51.5%)**。成因链:#17 `+messages-search` exit2 → 退而求其次 #18 `+chat-messages-list --page-all`(无时间过滤)→ 输出 43.5KB 被持久化到文件 → agent `Read` 整个文件 → 22.5k tok 灌进上下文。后面又靠本地 `grep`(#27–33) 抠出「飞豆」两条。
|
||||
- **从文档角度**:`chat-messages-list.md` **本题 reach=0**(没读到),而它恰好写了 `--start/--end` 时间过滤、`--page-size`、「无 sender 排序」等能避免全量拉取的约束(L20–52)。SKILL.md 表里对该 shortcut 只写「supports time range/sort/pagination」一句、未提示「大群全量拉取会爆上下文、应先 server-side 收窄」。**这是一个真实的「该读没读 → 全量灌入」放大器**(约束 5 状态①:调用前从没读该 reference)。
|
||||
- **但这条对本轮目标是「方向张力」,不是干净的 token 抓手**:要避免全量灌入,文档侧只能**增加**收窄指引(前置或加 caution),这与「降 token」的常驻成本目标**方向相反**(见硬性约束 7 的冲突记录)。且 22.5k 黑洞是**单次工具输出**(单次输出可信度、单题、强烈依赖该群消息量),不是稳定常驻热点。**结论:列为观察项交评测裁决,不要当成 RC-1 那种干净抓手去推「前置 chat-messages-list」——很可能只增 token 不省。**
|
||||
|
||||
## 可发现性时序(约束 5 三态;判「前置能不能救」的决定性证据)
|
||||
> 对每条相关 reference / `--help`,按相对首次失败调用的读取时序统计。`--help` 扫 Bash(本轮 3 题均未跑任何 `--help`)。
|
||||
|
||||
| reference / `--help` | 聚合 reach | ①从没读 | ②失败后才读 | ③读了仍错/卡 | 主导态 → 改动方向 |
|
||||
|---|---|---|---|---|---|
|
||||
| `lark-shared/SKILL.md` | 1.0 | 0 | 0 | — | 三题调用前都读了;014 仍卡(环境,非内容);不可改 |
|
||||
| `chat-create.md` | 0.667 | 0 | 0 | — | 080 调用前读→建群成功;014 调用前读→授权阻断(非 reference 错)。**非触达问题** |
|
||||
| `messages-send.md` | 0.333 | 0 | 0 | — | 080 调用前读→发卡片成功。**非触达问题** |
|
||||
| `chat-messages-list.md` | 0.0 | 1 (015) | 0 | — | ① **015 调用前从没读**→直接 `--page-all` 全量拉取→token 黑洞。触达缺口,但补它=增 token,与目标冲突(见上) |
|
||||
| `messages-search.md` | 0.0 | 1 (015) | 0 | — | ① 015 从没读 messages-search.md,直接猜 `+messages-search` ×2 → exit2。该命令 user-only(SKILL 表 L101 已注明),bot 身份必败 |
|
||||
- **结论**:本轮 effect 失败的唯一真题(014)是**状态③语义但根因是环境**(内容已触达、卡在沙箱授权+跨域),**前置/补内容救不了**。015 的两处 ① 触达缺口(chat-messages-list / messages-search 没读)确实存在,但**修它们的方向(增内容)与本轮 token 目标相反**,且 015 最终已 PASS(靠 bot + 本地 grep 兜底)——所以这两处**不是必须修的 effect 缺口,只是 token 放大器**,且修了大概率反而增 token。
|
||||
- **对 candidate-writer 的含义**:**本轮没有「该前置」的干净 case**。RC-1/2/3 都是「调用前已读、内容够用 → 减体积」的纯 token 减法,不涉及触达。不要被 015 的两处 ① 诱导去推前置——那会与目标背道而驰。
|
||||
|
||||
## 方向冲突记录(硬性约束 7)
|
||||
- **减体积(RC-1/2/3,与 objective.direction 同向)** vs **补收窄指引(修 015 chat-messages-list 全量灌入,与 objective 反向)**:前者降常驻/按需 token,后者为省「单次工具输出」反而要**增**文档常驻 token。两者方向相反,**不可合并**。本轮目标是降 token,应取减体积一侧;015 的全量灌入作为观察项记录、不作为本轮要补的内容根因。
|
||||
|
||||
## 差距台账复盘
|
||||
- 无(round 2,`discard-ledger.json` 为空,无已跑未采纳候选)。
|
||||
|
||||
## 逐 case
|
||||
|
||||
### 1 (014) [workorder=PASS / 实质 FAIL] token=34555(reported)/visible 17,364 耗时=37s 命令失败率≈5/7 维度=效果(不可修)
|
||||
- 判分点结果:3 条全 ✗——建群/拉人/发消息全未发生,卡在 `contact +search-user` 解析 open_id(user 授权阻断)。verdict=PASS 系聚合口径,按判分点证据当 FAIL 处理。
|
||||
- 命令失败:≈5/7。`contact +search-user` bot exit2 ×2、user token_missing ×2;`auth qrcode` 绝对路径 unsafe ×1(改相对路径成功)。**全部非 lark-im 命令的内容错误**。
|
||||
- 可发现性时序:调用前读了 SKILL.md(reach=1.0)+chat-create.md(3,062 tok);失败在更上游的跨域 contact + 授权。**非 lark-im 触达问题**。
|
||||
- token 归因:SKILL.md 正文 3,751(常驻静态,21.6%)+ chat-create.md 3,062(按需,17.6%,本题没走到建群=读了没用上)+ 系统 Skill 列表注入 4,612(固定开销,不归因)。lark-cli 命令累计含多次短失败回显,单条都短、非热点。
|
||||
- 耗时归因:本题往返多(查联系人→切 contact→失败→auth status→授权→qrcode 重试)。多为授权链路 + 跨域固有串行 + 反应式重试(duration 弱信号,需多轮复现)。
|
||||
- 文档根因:效果=沙箱 user 授权 + 跨域 contact(环境,**无 lark-im 文档根因,本轮不改**);token=SKILL.md 常驻(RC-1)+ chat-create.md 按需(RC-3)。
|
||||
|
||||
### 2 (015) [PASS·真] token=54568(reported)/visible 43,760 耗时=2m5s 命令失败率≈3/9 维度=token
|
||||
- 判分点结果:3/3 ✓——定位群、转发「飞豆」消息、@傅六铭知会全部成功(两次 `messages-send` 均 `ok:true`)。**全程 bot 身份,无授权阻断**。
|
||||
- 命令失败:≈3/9。`+messages-search` bot exit2、`+messages-search --as user` exit2、`+chat-messages-list --page-all` exit2(无过滤);agent 退到 `+chat-messages-list`(无 page-all) + 本地 grep 兜底成功。
|
||||
- 可发现性时序:① `messages-search.md` / `chat-messages-list.md` **调用前从没读**(reach=0),直接猜命令。messages-search 是 user-only(SKILL 表 L101 已注明)、bot 身份必败——agent 没看清就猜。
|
||||
- token 归因:**本题 token 大头不是 lark-im doc**,是 block #19 一次 `Read` 持久化文件 = **22,556 tok(51.5%,其他工具调用/返回)**,成因=`--page-all` 无过滤全量拉取→43.5KB→Read 灌入(单次输出可信度,强依赖该群消息量)。SKILL.md 正文 3,749(常驻)。lark-shared 3,749(跨 skill,不归因 lark-im)。
|
||||
- 耗时归因:本题最长(2m5s),主因是 messages-search 连环失败→改用 page-all→大输出→多次本地 grep 抠数据的多轮往返(duration 弱信号;工具调用 16 raw32,明显高于 080,作旁证)。
|
||||
- 文档根因:token 黑洞的放大器=`chat-messages-list.md` 没被读到 + SKILL.md 表未提示大群应 server-side 收窄——但**补这条与降 token 目标相反**(方向张力,见上),列为观察项;本题已 PASS。常规 token 抓手仍是 RC-1(SKILL.md 减体积)。
|
||||
|
||||
### 3 (080) [PASS·真] token=38009(reported)/visible 21,599 耗时=47s 命令失败率=0/3 维度=token
|
||||
- 判分点结果:3/3 ✓——`auth status` 见 bot ready→主动选 bot→建群`ok:true`→发 interactive 卡片`ok:true`。**任务完整完成,零命令失败**。
|
||||
- 命令失败:0/3。三条 lark-cli(auth status / chat-create / messages-send)全成功。
|
||||
- 可发现性时序:调用前读 SKILL.md + chat-create.md(3,060) + messages-send.md(5,365),全部状态③(调用前已读且用上)。**无触达问题**。
|
||||
- token 归因:**本题是纯 token 抓手题**——读取 Skill 占 56.4%:messages-send.md 5,365(按需,最大单块,RC-2)+ SKILL.md 3,751(常驻,RC-1)+ chat-create.md 3,060(按需,RC-3)。三块 reference/SKILL 都实读且 RC-2 的 messages-send.md 确实用上了。系统 Skill 列表注入 4,612(固定开销,不归因)。
|
||||
- 耗时归因:47s,全部为正常建群+发卡片串行,无重试、无写后回查(无离群)。
|
||||
- 文档根因:无效果根因(已绿);token=RC-2(messages-send.md 内部冗余) + RC-1(SKILL.md 常驻) + RC-3(chat-create.md)。**本题 token 杠杆最高且无 effect 风险**(命令全成功,减 reference 体积不碰已走通链路)。
|
||||
|
||||
## 给 candidate-writer 的收口(不含具体改法)
|
||||
- **唯一在 T1 内可合法发力的轴是 token**,且本轮是**纯减体积**场景(无触达缺口要补、无参数错误要改):
|
||||
- **RC-1**(SKILL.md `## Important Notes` 低命中小节 + `## Shortcuts` 全表):3 题全命中、常驻静态、最稳,但剩余多为 identity/约束类,删错会碰坏 015/080 已走通的 bot 身份判断——**effect 风险点**。
|
||||
- **RC-2**(messages-send.md 内部 4 处「选 content flag」语义重叠 + 全形态 Commands):单文件最大块、内部冗余明确,但子集只有 080 一题(reach=0.333),证据基数小、效果需评测确认。
|
||||
- **RC-3**(chat-create.md 按需偏大):杠杆最低,列为更次级。
|
||||
- **effect 不可在本轮 T1 内合法抬升**:014 是环境(沙箱不能扫码)+ 跨域 contact,无 lark-im 文档根因。015/080 已真 PASS。候选必须**保住 015/080 走通 bot 身份的 identity/参数说明**,降 token 时别误伤。
|
||||
- **不要推前置**:本轮没有「该前置」的干净 case。015 的两处触达缺口(chat-messages-list/messages-search 没读)虽真实存在,但修它们=增内容,与降 token 目标**方向冲突**,且 015 已 PASS——属观察项,非本轮要补的根因。
|
||||
- **缺失信息(doc_fix_hint 语气)**:SKILL.md 的 Important Notes/Shortcuts 全量罗列、本轮低命中却每题常驻;messages-send.md 同一选型规则在 4 处重复表述、Commands 罗列全部媒体形态——这类「全量/重复、低命中」内容是 token 的主要去处,且是减法(删冗余)而非加法。
|
||||
- **数据缺口**:(a) workorder 三题 verdict 全 PASS,但 014 判分点证据全 ✗——归因按判分点当 FAIL 处理,effect 实际是 2 真 PASS + 1 实质 FAIL。(b) RC-2/RC-3 子集小(messages-send.md 仅 080、chat-create.md 仅 014+080),单轮证据基数小,token 降幅需评测在子集上确认。(c) 015 的 22.5k 黑洞是单次工具输出,强依赖该群消息量,非稳定常驻热点,单题不可外推。(d) duration 三题波动大(37s/2m5s/47s),015 长尾主因是 messages-search 连环失败+大输出多轮抠数据,但单轮不足以定论,需多轮复现;工具调用数(8/16/6 model calls)可作比 wall-clock 稳的旁证。(e) 工具调用次数 session-analyze(model calls 8/16/6) 与 workorder 趋势表(R1 均值 26.3) 口径不一致,趋势表疑似含 raw 计数,旁证以 timeline 实际往返为准。
|
||||
1
harness-opt/rounds/round-002/discard-ledger.json
Normal file
1
harness-opt/rounds/round-002/discard-ledger.json
Normal file
@@ -0,0 +1 @@
|
||||
[]
|
||||
1
harness-opt/rounds/round-002/failure-memory.json
Normal file
1
harness-opt/rounds/round-002/failure-memory.json
Normal file
@@ -0,0 +1 @@
|
||||
[]
|
||||
220
harness-opt/rounds/round-002/module-reach.json
Normal file
220
harness-opt/rounds/round-002/module-reach.json
Normal file
@@ -0,0 +1,220 @@
|
||||
{
|
||||
"skills/lark-im/SKILL.md": {
|
||||
"reach": 1.0,
|
||||
"read_cases": [
|
||||
"1",
|
||||
"2",
|
||||
"3"
|
||||
],
|
||||
"actual_cases": [
|
||||
"1",
|
||||
"2",
|
||||
"3"
|
||||
],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": true
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-create.md": {
|
||||
"reach": 0.667,
|
||||
"read_cases": [
|
||||
"1",
|
||||
"3"
|
||||
],
|
||||
"actual_cases": [
|
||||
"1",
|
||||
"3"
|
||||
],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-identity.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-messages-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-search.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-update.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-group-list-item.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-group-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-group-query-item.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-groups.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-create.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-remove.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-flag-cancel.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-flag-create.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-flag-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-message-enrichment.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-mget.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-reply.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-resources-download.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-search.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-send.md": {
|
||||
"reach": 0.333,
|
||||
"read_cases": [
|
||||
"3"
|
||||
],
|
||||
"actual_cases": [
|
||||
"3"
|
||||
],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-reactions.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-threads-messages-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
}
|
||||
}
|
||||
15
harness-opt/rounds/round-002/review.json
Normal file
15
harness-opt/rounds/round-002/review.json
Normal file
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"generated_by": "lark-cli-harness:opt-reviewer",
|
||||
"verdict": "PASS",
|
||||
"module": "skills/lark-im/references/lark-im-messages-send.md",
|
||||
"tier": "T1",
|
||||
"reason": "纯结构性去重:16407→6399 字节(-61%)与策略一致;逐项核对每条承重指令(互斥规则、video-cover 必配、cwd-relative/绝对路径拒绝、markdown→post 边界、三套 <at> 语法、content 各 msg_type 样例、Safety Constraints、identity+scope 映射)均原样保留在新文档内联,删的全是重复/过度罗列(4× 选型规则、镜像 --help 的 Parameters 表、Common Mistakes、Notes、冗余 Commands)。无硬编码评测答案、未针对 080 卡片流窄化、未碰 SKILL.md 身份路由、单文件单根因。",
|
||||
"dimensions": {
|
||||
"reward_hack": {"pass": true, "evidence": "无硬编码 eval ID/答案(仅 oc_xxx/ou_xxx 等通用占位符,与原文一致);card/interactive+bot 身份路径保留为通用指引,未按 080 卡片流做特判或窄化"},
|
||||
"semantic_regress": {"pass": true, "evidence": "逐条核对:互斥/video-cover/cwd-relative+绝对路径拒绝/markdown→post/三套 <at>/content 全 msg_type 样例/Safety/identity+scope 全部内联保留;仅删除的是真重复(dry-run 占位符细节、JSON wrap 示意、img_/file_ 自动识别说明),非承重 guardrail,且运行时可观测"},
|
||||
"token_shift": {"pass": true, "evidence": "真减 10008 字节常驻;--help 指针是 additive 补充(指向真实存在且 --help 已含互斥/video-cover/路径规则),承重 gotcha 全留内联,080 不需额外调 --help 即可恢复,无运行时增读拉力。注:work-order 提的 schema im.messages.create 方法不存在,但文档本身不指向 schema,不构成运行时陷阱"},
|
||||
"contract_break": {"pass": true, "evidence": "T1 文档不涉对外契约;prerequisite 链接目标存在、章节结构完整、无其他文件深链到被删 anchor(Media Input Rules/Common Mistakes 命中在 messages-reply.md 而非本文件)"},
|
||||
"devguide": {"pass": true, "evidence": "符合 reference 收敛到 gotcha-only、不镜像 --help 的优化方向;同一事实只写一处,删的两类(语义回退/承重删除)均未触发——优化红线两维过关"},
|
||||
"single_root_cause":{"pass": true, "evidence": "commit 仅 1 文件 51 insert/208 delete,全部服务 RC-2(单文件重复表述去重)一个根因;未捆 RC-1(SKILL.md)/RC-3(chat-create),未把无关删除以 token 对冲缝入"}
|
||||
}
|
||||
}
|
||||
380
harness-opt/rounds/round-002/round.json
Normal file
380
harness-opt/rounds/round-002/round.json
Normal file
@@ -0,0 +1,380 @@
|
||||
{
|
||||
"round": 2,
|
||||
"status": "admitted",
|
||||
"parent_id": "a1333f2e1f7e98bf6f705814b92cacae1f43565759e4e0c24a0a4700b241649e",
|
||||
"parent_worktree": "/Users/bytedance/Projects/cli",
|
||||
"child_worktree": "/Users/bytedance/Projects/cli",
|
||||
"base_commit": "51f2a70e6dffeea65d928badb6207408490dc215",
|
||||
"module": "skills/lark-im/references/lark-im-messages-send.md",
|
||||
"candidate_modules": [
|
||||
"skills/lark-im/SKILL.md",
|
||||
"skills/lark-im/references/lark-im-chat-create.md",
|
||||
"skills/lark-im/references/lark-im-chat-identity.md",
|
||||
"skills/lark-im/references/lark-im-chat-list.md",
|
||||
"skills/lark-im/references/lark-im-chat-messages-list.md",
|
||||
"skills/lark-im/references/lark-im-chat-search.md",
|
||||
"skills/lark-im/references/lark-im-chat-update.md",
|
||||
"skills/lark-im/references/lark-im-feed-group-list-item.md",
|
||||
"skills/lark-im/references/lark-im-feed-group-list.md",
|
||||
"skills/lark-im/references/lark-im-feed-group-query-item.md",
|
||||
"skills/lark-im/references/lark-im-feed-groups.md",
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-create.md",
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-list.md",
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-remove.md",
|
||||
"skills/lark-im/references/lark-im-flag-cancel.md",
|
||||
"skills/lark-im/references/lark-im-flag-create.md",
|
||||
"skills/lark-im/references/lark-im-flag-list.md",
|
||||
"skills/lark-im/references/lark-im-message-enrichment.md",
|
||||
"skills/lark-im/references/lark-im-messages-mget.md",
|
||||
"skills/lark-im/references/lark-im-messages-reply.md",
|
||||
"skills/lark-im/references/lark-im-messages-resources-download.md",
|
||||
"skills/lark-im/references/lark-im-messages-search.md",
|
||||
"skills/lark-im/references/lark-im-messages-send.md",
|
||||
"skills/lark-im/references/lark-im-reactions.md",
|
||||
"skills/lark-im/references/lark-im-threads-messages-list.md"
|
||||
],
|
||||
"module_reach": {
|
||||
"skills/lark-im/SKILL.md": {
|
||||
"reach": 1.0,
|
||||
"read_cases": [
|
||||
"1",
|
||||
"2",
|
||||
"3"
|
||||
],
|
||||
"actual_cases": [
|
||||
"1",
|
||||
"2",
|
||||
"3"
|
||||
],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": true
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-create.md": {
|
||||
"reach": 0.667,
|
||||
"read_cases": [
|
||||
"1",
|
||||
"3"
|
||||
],
|
||||
"actual_cases": [
|
||||
"1",
|
||||
"3"
|
||||
],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-identity.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-messages-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-search.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-update.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-group-list-item.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-group-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-group-query-item.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-groups.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-create.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-remove.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-flag-cancel.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-flag-create.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-flag-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-message-enrichment.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-mget.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-reply.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-resources-download.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-search.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-send.md": {
|
||||
"reach": 0.333,
|
||||
"read_cases": [
|
||||
"3"
|
||||
],
|
||||
"actual_cases": [
|
||||
"3"
|
||||
],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-reactions.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-threads-messages-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
}
|
||||
},
|
||||
"expected_reach": {},
|
||||
"minibatch": [
|
||||
"1",
|
||||
"2",
|
||||
"3"
|
||||
],
|
||||
"pareto_cases": [
|
||||
"1",
|
||||
"2",
|
||||
"3"
|
||||
],
|
||||
"artifacts": {
|
||||
"workorder": "workorder.md",
|
||||
"diagnosis": "diagnosis.md",
|
||||
"attribution": "attribution.json",
|
||||
"strategy": "strategy.md",
|
||||
"review": "review.json",
|
||||
"trend": "trend.json"
|
||||
},
|
||||
"code_tip": "82a099feafb45d101116f10230ce7c2f92fbcfe5",
|
||||
"signature": "557349b40feb359bb791749a37571d59edb7e72e",
|
||||
"tier": "T1",
|
||||
"intent": "consolidate 4x repeated content-flag rule + compress media enumeration & --help-mirror sections in messages-send.md (token, no capability removed)",
|
||||
"target_axis": "token",
|
||||
"changed_files": [
|
||||
"skills/lark-im/references/lark-im-messages-send.md"
|
||||
],
|
||||
"decision_basis": {
|
||||
"type": "module",
|
||||
"module": "skills/lark-im/references/lark-im-messages-send.md"
|
||||
},
|
||||
"decision_cases": [
|
||||
"3"
|
||||
],
|
||||
"review": {
|
||||
"generated_by": "lark-cli-harness:opt-reviewer",
|
||||
"verdict": "PASS",
|
||||
"module": "skills/lark-im/references/lark-im-messages-send.md",
|
||||
"tier": "T1",
|
||||
"reason": "纯结构性去重:16407→6399 字节(-61%)与策略一致;逐项核对每条承重指令(互斥规则、video-cover 必配、cwd-relative/绝对路径拒绝、markdown→post 边界、三套 <at> 语法、content 各 msg_type 样例、Safety Constraints、identity+scope 映射)均原样保留在新文档内联,删的全是重复/过度罗列(4× 选型规则、镜像 --help 的 Parameters 表、Common Mistakes、Notes、冗余 Commands)。无硬编码评测答案、未针对 080 卡片流窄化、未碰 SKILL.md 身份路由、单文件单根因。",
|
||||
"dimensions": {
|
||||
"reward_hack": {
|
||||
"pass": true,
|
||||
"evidence": "无硬编码 eval ID/答案(仅 oc_xxx/ou_xxx 等通用占位符,与原文一致);card/interactive+bot 身份路径保留为通用指引,未按 080 卡片流做特判或窄化"
|
||||
},
|
||||
"semantic_regress": {
|
||||
"pass": true,
|
||||
"evidence": "逐条核对:互斥/video-cover/cwd-relative+绝对路径拒绝/markdown→post/三套 <at>/content 全 msg_type 样例/Safety/identity+scope 全部内联保留;仅删除的是真重复(dry-run 占位符细节、JSON wrap 示意、img_/file_ 自动识别说明),非承重 guardrail,且运行时可观测"
|
||||
},
|
||||
"token_shift": {
|
||||
"pass": true,
|
||||
"evidence": "真减 10008 字节常驻;--help 指针是 additive 补充(指向真实存在且 --help 已含互斥/video-cover/路径规则),承重 gotcha 全留内联,080 不需额外调 --help 即可恢复,无运行时增读拉力。注:work-order 提的 schema im.messages.create 方法不存在,但文档本身不指向 schema,不构成运行时陷阱"
|
||||
},
|
||||
"contract_break": {
|
||||
"pass": true,
|
||||
"evidence": "T1 文档不涉对外契约;prerequisite 链接目标存在、章节结构完整、无其他文件深链到被删 anchor(Media Input Rules/Common Mistakes 命中在 messages-reply.md 而非本文件)"
|
||||
},
|
||||
"devguide": {
|
||||
"pass": true,
|
||||
"evidence": "符合 reference 收敛到 gotcha-only、不镜像 --help 的优化方向;同一事实只写一处,删的两类(语义回退/承重删除)均未触发——优化红线两维过关"
|
||||
},
|
||||
"single_root_cause": {
|
||||
"pass": true,
|
||||
"evidence": "commit 仅 1 文件 51 insert/208 delete,全部服务 RC-2(单文件重复表述去重)一个根因;未捆 RC-1(SKILL.md)/RC-3(chat-create),未把无关删除以 token 对冲缝入"
|
||||
}
|
||||
}
|
||||
},
|
||||
"child_k": 5,
|
||||
"eval_trace": null,
|
||||
"retro": {
|
||||
"cause": "已入池",
|
||||
"noise_borderline": false,
|
||||
"summary": "越带入池,无需复盘补发"
|
||||
},
|
||||
"retro_sessions": [
|
||||
{
|
||||
"case": "3",
|
||||
"session": null,
|
||||
"axis": "token",
|
||||
"expect": "降",
|
||||
"parent": 37942,
|
||||
"child": 35478,
|
||||
"gain": "收益现",
|
||||
"pass_delta": null
|
||||
}
|
||||
],
|
||||
"verdict": "admitted",
|
||||
"ci": null,
|
||||
"new_candidate": "557349b40feb359bb791749a37571d59edb7e72e",
|
||||
"decision": {
|
||||
"parent_success": 1.0,
|
||||
"child_success": 1.0,
|
||||
"parent_score": 0.6,
|
||||
"child_score": 1.0,
|
||||
"score_saved": 0.4,
|
||||
"score_threshold": 0.09532271373123208,
|
||||
"parent_token": 37942.0,
|
||||
"child_token": 35478.0,
|
||||
"saved": 2464.0,
|
||||
"threshold": 4532.708313776408,
|
||||
"parent_duration": 45769.0,
|
||||
"child_duration": 46540.0,
|
||||
"dur_saved": -771.0,
|
||||
"dur_threshold": 4899.200953624988,
|
||||
"dur_margin": 1.0,
|
||||
"missing_duration": [],
|
||||
"k_child": 5,
|
||||
"k_parent": 5,
|
||||
"decision_n": 1,
|
||||
"missing_context": [],
|
||||
"missing_score": [],
|
||||
"parent_token_acc": 251669.0,
|
||||
"child_token_acc": 221685.0,
|
||||
"phi0_score": 0.5333333333333333,
|
||||
"eff_margin": 1.0,
|
||||
"parent_token_full": 37942.0,
|
||||
"child_token_full": 35478.0,
|
||||
"saved_full": 2464.0,
|
||||
"observe_n": 1,
|
||||
"target_axis": "token",
|
||||
"admitted": true,
|
||||
"reason": "score_gain"
|
||||
},
|
||||
"patch": "verify_results/round-002-lark-im-references-lark-im-messages-send.patch"
|
||||
}
|
||||
48
harness-opt/rounds/round-002/strategy.md
Normal file
48
harness-opt/rounds/round-002/strategy.md
Normal file
@@ -0,0 +1,48 @@
|
||||
# Round 2 候选策略(模块=references/lark-im-messages-send.md, tier=T1, 主指标=token)
|
||||
|
||||
## 根因与选择
|
||||
|
||||
| 根因 | 来源(评测归因/规范经验) | 承载模块(reach) | annotation 风险级 | coverage 档 | P级 | 选中 |
|
||||
|---|---|---|---|---|---|---|
|
||||
| RC-2: messages-send.md 单文件最大、内部「选 content flag」规则重复 4 处 + 全媒体形态罗列 + Parameters/Notes 镜像 --help | 评测归因①(080 实读实用)+规范经验②(annotation R1×140/R2×109,仅 1 段 R3) | references/lark-im-messages-send.md (0.333) | R1/R2 主导,唯一 R3=Safety Constraints(L9–22) | 密 / overfit 低 | P1 | ✅ |
|
||||
| RC-1: SKILL.md `## Important Notes` 低命中 + `## Shortcuts` 全表常驻 | 评测归因①(reach=1.0,3 题全命中) | SKILL.md (1.0) | R2/R3 混合(identity/约束密集) | 密 / 中 | P0(命中) 但 effect 高风险 | |
|
||||
| RC-3: chat-create.md 按需偏大 | 评测归因① | references/lark-im-chat-create.md (0.667) | — | 密 | P1 | |
|
||||
|
||||
- **选中理由**:RC-2 是诊断点名「最干净的 token 杠杆」——单文件最大块(实测 ~5,365 tok,占 080 visible 24.8%),且 080 调用前已读、确实据它发卡片成功(reach=0.333、actual=1,非「读了没用」)。annotation 证实它 R1/R2 主导(140 R1 + 109 R2 行,可重构/可压缩),唯一 R3 段是 Safety Constraints(L9–22),我**原样保留语义**。coverage=「密」、overfit「低」→ 本轮 eval 能在 080 上裁真伪。这是纯减体积、零能力删除、不碰 SKILL.md 路由的改动。
|
||||
- **为什么不选 RC-1**:reach=1.0、命中率最高,但 diagnosis 明确标它为 **effect 风险点**——剩余内容多为 identity/约束类,正是驱动 015/080 走通 bot 身份判断的承重内容;objective 的**硬门槛是「保住成功率」**,动 SKILL.md 最可能误伤这条已绿链路。本轮放弃,避免拿成功率换 token。
|
||||
- **为什么不选 RC-3**:diagnosis 判其杠杆最低(体积不离群),列为更次级;同一根因一轮只动一个,留待后续轮次。
|
||||
- **选模块理由**:messages-send.md reach=0.333>0(满足 reach 锁),承载选中的 RC-2,是非域 reference、改它不触碰 SKILL.md 的身份路由面。多文件无——本轮只动这一个文件。
|
||||
- **规范经验源补注**:对照 content-taxonomy——「单命令用法/长示例/与 --help 重复」类默认 R0/R1,「一般行为规则/CLI 机制约定」默认 R2;本文件的重复选型规则、全形态 Commands、Parameters/Notes 镜像即此类,处理方向为「留命中率最高一处,其余删或指针」「高频留 2–3 例,长的下沉」。当轮可被 080 裁真伪(coverage 密/overfit 低)。
|
||||
|
||||
## 改了什么(逐处)
|
||||
- **L23–43 `## Choose The Right Content Flag` + `### --text vs --markdown`**:两段语义重叠的选型说明 → 合并为单张 4 行选型表(markdown/text/content/media),并把互斥规则并入表后一句。删掉 `### --text vs --markdown` 整段(与表重复)。
|
||||
- **L44–82 `## What --markdown Really Does` + `### Markdown Boundaries` + `### Image Constraint`**:三段约 39 行 → 压成 `## --markdown Gotchas` 三条要点(强制 post/无 title、标题改写规则、图片预上传 vs 远程 URL vs 本地路径不支持)。删掉 JSON wrap 示意、逐条 boundary 罗列等可由行为观察得到的展开。
|
||||
- **L83–93 图片预上传双命令示例**:并入 `## Commands` 的一条 markdown+image 示例(保留 `im images create` → 引用 img_xxx 的关键两步)。
|
||||
- **L114–161 `## Commands`(15+ 例覆盖全媒体形态)+ `## Media Input Rules`**:压成代表性示例(markdown / text / DM / post-title / markdown+image / 4 个媒体一组 / idempotency+dry-run),媒体路径规则收成 `--help` 指针后的 3 条 load-bearing gotcha(cwd-relative/绝对路径拒绝、video-cover 必配、msg-type 推断冲突)。
|
||||
- **L169–191 `## Parameters` 表**:删除镜像 `--help` 的逐参数描述,改为「Run `lark-cli im +messages-send --help`」指针 + 仅保留 --help 不显然的三条硬规则(已并入 Commands 末尾)。
|
||||
- **L192–202 `## Common Mistakes`**:整段删除——逐条都是选型表/markdown gotcha 的反向重述(第 4 次重复选型规则),删后选型信息仍在表里。
|
||||
- **L203–216 `## content Format Reference`**:保留(构造 `--content` 的 gotcha),把 image/file/audio 三行合并为一行省重复。
|
||||
- **L227–248 `## @Mention Format`**:保留全部三种 msg_type 的 `<at>` 语法(text/post/interactive 各异、AI 猜不到),压紧为两条要点、去掉小标题与重复散文。
|
||||
- **L249–264 `## Notes`**:整段删除——逐条(互斥/media 上传/scope/markdown 强制 post/video-cover/msg-type 冲突)均已在 Safety Constraints、选型表、--markdown Gotchas、Commands 指针处各保留一处单一事实源。
|
||||
|
||||
## 为什么这么改(机制)
|
||||
- **消除根因的因果链**:该 reference 的体积来自「同一份选型规则在 4 个 section 重复 + 全媒体形态逐条罗列 + Parameters/Notes 镜像 --help」。token 不是被任务必需信息占用,而是被**重复表述**占用。按「同一份事实只写一次」(锚点 1)合并到单一事实源后,每条 load-bearing 信息仍恰好出现一次,080 这类「读该 reference→发消息」的题,读入 token 直接下降而行为不变。
|
||||
- **不删能力**:每个 flag(text/markdown/content/image/file/video/audio/idempotency/dry-run/msg-type/video-cover/as)、每条硬约束(互斥、video-cover 必配、cwd-relative 路径、绝对路径拒绝、markdown 强制 post/无 title、msg-type 冲突校验)、三套 `<at>` 语法、content 各 msg_type 样例、Safety Constraints、identity+scope 映射——全部保留,只是从「重复 N 次/逐条罗列」变成「一处/代表性示例 + --help 指针」。
|
||||
- **规范经验源**:依 optimization-playbook「reference 收敛到 gotcha-only,不做 --help 镜像」——Parameters 全表/全形态 Commands 属 USAGE,下沉到 `--help` 指针;保留的是 --help 表达不了的跨 flag 互斥、媒体路径安全、markdown→post 边界、@mention 按类型差异等 gotcha。annotation 标这些段为 R1(可重构/下沉),符合处理方向;唯一 R3(Safety)原样保留。
|
||||
|
||||
## 预期效果
|
||||
- **成功率**:不退化。080(唯一读该文件的题)的发卡片链路依赖的是 `--content`/`interactive`、identity=bot、chat-id——全部保留;选型表、content Format Reference、Safety、scope 都在。015/080 走通 bot 身份的判断由 SKILL.md + identity 段承载,本轮**没碰 SKILL.md**,零误伤面。014 与本文件无关(reach 不含 014)。
|
||||
- **context(分两层)**:
|
||||
- (1) **静态字数差**:16,407 → 6,399 chars(-61.0%);tiktoken cl100k 3,869 → 1,799 tok(-53.5%)。(注:diagnosis 报 ~5,365 tok 系另一 tokenizer/含注入开销;此处用 cl100k 自测,方向与幅度一致。)
|
||||
- (2) **运行时 context 方向**:仅在**实读该 reference 的子集**生效——本轮即 080 一题,运行时读入下降约 50%+(该块占 080 visible 24.8%,预计 080 visible 降约 12–13%)。其余两题(014/015)不读该文件,运行时 token **不变**(既不增也不减)。这是按需 reference,不是常驻面,不会影响未读它的题。
|
||||
- **覆盖敞口**:RC-2 子集仅 080 一题(reach=0.333),证据基数小。coverage 判该文件「密/overfit 低」,本轮 eval 可在 080 上裁真伪,但单题不可外推到「所有发消息任务」。建议后续补「读 messages-send.md 后用 --markdown / 媒体 / @mention」的 case 加厚子集。预期收益落在 **token 轴**(080 visible 下降),effect 轴维持不退化。
|
||||
|
||||
## 刻意没做什么(反 reward-hack / 反过拟合)
|
||||
- 没硬编码任何评测题答案;没删任何能力、flag、guardrail、身份/scope 说明;没碰 lark-im 以外文件,也没把无关根因捆进本轮(commit 仅 1 个文件)。
|
||||
- **没碰 SKILL.md(RC-1)**:尽管 reach=1.0 杠杆最大,但其剩余内容是驱动 015/080 bot 身份判断的承重 identity/约束,diagnosis 标为 effect 风险点;在「保住成功率」硬门槛下不拿成功率换 token。
|
||||
- **没补收窄/分页指引**(015 的 22.5k chat-messages-list 黑洞):那是「增内容」,与降 token 目标方向相反,diagnosis 已列为观察项、本轮不做。
|
||||
- 本改动**不是按评测错误反推**的参数/路由拟合——是基于 annotation + content-taxonomy 的结构性去重,删的是重复表述而非按 080 的具体内容裁剪;真实价值在「任何读该 reference 的发消息任务都少读重复 token」,080 只是当轮可验证的子集。
|
||||
- 未发现需要 breaking(T3)才能根治的点;本轮纯 T1 文档去重即可。
|
||||
|
||||
## 签名
|
||||
- signature: 557349b40feb359bb791749a37571d59edb7e72e (commit 82a099fe 的 diff hash) tier: T1
|
||||
11
harness-opt/rounds/round-002/trend.json
Normal file
11
harness-opt/rounds/round-002/trend.json
Normal file
@@ -0,0 +1,11 @@
|
||||
[
|
||||
{
|
||||
"round": 1,
|
||||
"n": 3,
|
||||
"pass_n": 0,
|
||||
"cmd_fail_rate": 0.6,
|
||||
"tool_calls": 26.333333333333332,
|
||||
"duration_ms": 50189.0,
|
||||
"token": 31997.0
|
||||
}
|
||||
]
|
||||
43
harness-opt/rounds/round-002/workorder.md
Normal file
43
harness-opt/rounds/round-002/workorder.md
Normal file
@@ -0,0 +1,43 @@
|
||||
# Round 2 归因派工单(parent=a1333f2e1f7e98bf6f705814b92cacae1f43565759e4e0c24a0a4700b241649e;模块未定,由 candidate-writer 据诊断点名)
|
||||
|
||||
> **只读输入**——opt-attributor 读本文件,把诊断**另写** `diagnosis.md`(给 candidate-writer)+ 逐题结构化 `attribution.json`(给 dashboard)。**不要覆盖本文件**,留作派工单↔诊断的前后对比。
|
||||
> 判分点只当「什么算挂」的锚,禁止照抄 grader 药方(已从派工单剔除)。
|
||||
|
||||
## 模块运行时可达性(选模块第一步的证据;要选须在 strategy.md 说明理由)
|
||||
> reach=**实测**触达率(域主 SKILL.md 经 Skill 工具加载、reference 经 Read,都从 trace 实测,没有恒在的面);判决集=实测∪预期触达。**实测低但有预期触达 ⚠️=可发现性/路由根因**(本该读却没读,如没路由到该域 / 速查表漏链接 / 该前置),正该选来修——不是白烧;reach=0 且无预期 才是真白烧。 **别用「全集均摊」判 reference 价值**:判决在 reach 子集上做,压一条 reference 的降幅在它子集里不被没读它的题稀释——reach 不高(但 >0)的 reference 在自己子集上也可能越带。
|
||||
- `skills/lark-im/SKILL.md` → reach=1.0 [域主 skill·经 Skill 工具加载];判决集(实测∪预期): ['1', '2', '3']
|
||||
- `skills/lark-im/references/lark-im-chat-create.md` → reach=0.667;判决集(实测∪预期): ['1', '3']
|
||||
- `skills/lark-im/references/lark-im-messages-send.md` → reach=0.333;判决集(实测∪预期): ['3']
|
||||
- (另 22 个 reference reach=0 且无预期触达,本轮无关,略)
|
||||
|
||||
## 逐轮诊断信号趋势(纯诊断,不进判决)
|
||||
|
||||
| 轮 | 题数 | PASS | 命令失败率 | 工具调用 | 耗时(ms) | token |
|
||||
|---|---|---|---|---|---|---|
|
||||
| R1 | 3 | 0 | 0.60 | 26 | 50189 | 31997 |
|
||||
|
||||
> 跨题均值,按轮排。**命令失败率、工具调用数是横切诊断信号,不是准入轴**(准入只走 效果/token/耗时)——用来判「上一轮那刀有没有把失败/轮次压下去」。工具调用数比 wall-clock 稳,可给噪声大的耗时轴当旁证。
|
||||
|
||||
### 1 [PASS] ctx=34270 (acc=274608) 43995ms tools=31
|
||||
- session.jsonl: harness-opt/rounds/round-001/child-runs/run-1/detail_info/cases/CLI_核心评测_014/0/session.jsonl [native]
|
||||
- 判分点(grader 的「什么算挂」oracle,非药方):
|
||||
✗ 使用当前用户身份创建名为「IM合作群」的群聊
|
||||
证据: transcript 在展示授权二维码后结束,无任何 `lark-cli im +chat-create` 调用。执行停在 '授权完成后请告诉我,我会继续帮你创建群聊并发送消息',群聊未创建。
|
||||
✗ 将傅一铭和傅二铭加入该群
|
||||
证据: transcript 显示尝试搜索用户时遇到 `need_user_authorization` 错误,授权流程启动后中断。未获取到任何用户的 open_id,无后续添加操作。
|
||||
✗ 在该群发送文本消息「大家体验有问题随时沟通」,并返回可验证的 chat_id / message_id
|
||||
证据: 群聊未创建,无 chat_id 可返回。transcript 无任何 `lark-cli im messages-send` 调用。
|
||||
|
||||
### 2 [PASS] ctx=47116 (acc=612048) 114310ms tools=49
|
||||
- session.jsonl: harness-opt/rounds/round-001/child-runs/run-1/detail_info/cases/CLI_核心评测_015/0/session.jsonl [native]
|
||||
- 判分点(grader 的「什么算挂」oracle,非药方):
|
||||
✓ 成功定位名为「fusanming_at_openclaw群」的群,并获取最近包含「飞豆」关键字的消息
|
||||
✓ 将筛选出的相关消息内容转发到「fusanming_at_需求测试群」
|
||||
✓ 在「fusanming_at_需求测试群」中 @傅六铭 做知会,消息发送成功
|
||||
|
||||
### 3 [PASS] ctx=37942 (acc=251669) 45769ms tools=23
|
||||
- session.jsonl: harness-opt/rounds/round-001/child-runs/run-1/detail_info/cases/CLI_核心评测_080/0/session.jsonl [native]
|
||||
- 判分点(grader 的「什么算挂」oracle,非药方):
|
||||
✓ 使用用户身份创建一个名为「今晚吃什么」的群,预期返回 chat_id
|
||||
✓ 创建一张飞书卡片,卡片内容包含「今天晚上吃什么」
|
||||
✓ 将该卡片发送到新建群中,预期返回 message_id
|
||||
59
harness-opt/rounds/round-003/attribution.json
Normal file
59
harness-opt/rounds/round-003/attribution.json
Normal file
@@ -0,0 +1,59 @@
|
||||
[
|
||||
{
|
||||
"case_id": "1",
|
||||
"verdict": "PASS",
|
||||
"verdict_note": "workorder=PASS(聚合口径);判分点证据 3/3 ✗,按判分点当实质 FAIL 处理",
|
||||
"token": 34555,
|
||||
"duration_ms": 37000,
|
||||
"tool_calls": 31,
|
||||
"cmd_attempts": 7,
|
||||
"cmd_failures": 5,
|
||||
"cmd_fail_rate": 0.71,
|
||||
"discoverability_state": "无(失败命令全是跨域 contact + auth,非 lark-im;chat-create.md 调用前已读但未走到使用)",
|
||||
"axis": "效果",
|
||||
"root_cause": "沙箱 user 授权不可完成 + 跨域 lark-contact 命令依赖;无 lark-im 文档根因,本轮不改",
|
||||
"token_hotspot": "运行时冗余清单常驻(SKILL.md 3,456)+ 按需 chat-create.md 3,062(读了没用上);lark-shared 3,751 与系统 Skill 列表注入 4,612 均不归因",
|
||||
"token_reliability": "常驻静态(SKILL.md)/ 按需读取(chat-create.md,本题读了没用上)",
|
||||
"duration_hotspot": "多轮交互(查联系人→切 contact→失败→授权→qrcode 重试)+ 纯外部API延迟(部分不可归因)",
|
||||
"duration_reliability": "耗时波动大,单次运行不算数,需多题或多次复现",
|
||||
"doc_fix_hint": "效果侧无 lark-im 文档缺信息(环境+跨域);token 侧 chat-create.md 把同组 flag 在 Commands/Usage Scenarios 重复演示、Common Errors 复述 validation 字符串,属可删冗余"
|
||||
},
|
||||
{
|
||||
"case_id": "2",
|
||||
"verdict": "PASS",
|
||||
"verdict_note": "真 PASS,判分点 3/3 ✓,全程 bot 身份无授权阻断",
|
||||
"token": 54568,
|
||||
"duration_ms": 125000,
|
||||
"tool_calls": 49,
|
||||
"cmd_attempts": 11,
|
||||
"cmd_failures": 3,
|
||||
"cmd_fail_rate": 0.27,
|
||||
"discoverability_state": "① 从没读(messages-search.md / chat-messages-list.md reach=0,直接猜命令;本题未读任何 lark-im reference)",
|
||||
"axis": "token",
|
||||
"root_cause": "无过滤 +chat-messages-list --page-all 全量拉取 → 43.5KB 输出被 Read 整文件灌入 22,556 tok;token 大头非 lark-im doc。修它需补收窄/前置内容,与降 token 目标方向冲突,列观察项",
|
||||
"token_hotspot": "工具返回原样输出(block #19 单次 Read 22,556 tok / 51.5%,归「其他工具调用/返回」)",
|
||||
"token_reliability": "单次输出(强依赖该群消息量,单题不可外推,非稳定常驻热点)",
|
||||
"duration_hotspot": "多轮交互 + 重试(messages-search 连环 exit2 → page-all → 大输出 → 多次本地 grep 抠数据)",
|
||||
"duration_reliability": "耗时波动大,单次运行不算数,需多题或多次复现(model calls 16 作旁证,明显高于 080)",
|
||||
"doc_fix_hint": "本题无 T1 可发力的 token 抓手(大头是单次工具输出,非 lark-im doc 常驻);缺的是大群消息查询的 server-side 收窄指引,但补它=增内容、与降 token 反向,不作本轮根因"
|
||||
},
|
||||
{
|
||||
"case_id": "3",
|
||||
"verdict": "PASS",
|
||||
"verdict_note": "真 PASS,判分点 3/3 ✓,主动选 bot 身份建群+发卡片均 ok:true,零命令失败",
|
||||
"token": 38009,
|
||||
"duration_ms": 47000,
|
||||
"tool_calls": 22,
|
||||
"cmd_attempts": 3,
|
||||
"cmd_failures": 0,
|
||||
"cmd_fail_rate": 0.0,
|
||||
"discoverability_state": "无(无失败命令;SKILL.md + chat-create.md + messages-send.md 全部状态③:调用前已读且用上)",
|
||||
"axis": "token",
|
||||
"root_cause": "读取 Skill 占 56.4%;本轮唯一干净 token 抓手 = chat-create.md 内部冗余(示例罗列 + 场景重复 + --help 镜像),从未被优化过",
|
||||
"token_hotspot": "运行时冗余清单常驻 + 按需 reference(chat-create.md 当前 2,336 raw,可压 Commands/Usage Scenarios 重叠 + Common Errors validation 镜像;trace 里 messages-send.md 5,365 是旧版,round-2 已压到 2,006,本轮不再可压)",
|
||||
"token_reliability": "按需读取(chat-create.md reach=0.667,本题是其压缩收益唯一稳态兑现题)",
|
||||
"duration_hotspot": "无离群(建群+发卡片正常串行,无重试、无写后回查)",
|
||||
"duration_reliability": "耗时波动大,单次运行不算数,需多题或多次复现",
|
||||
"doc_fix_hint": "chat-create.md 把同组 flag 在 Commands(12 例)+Usage Scenarios(3 场景)重复演示、Common Errors 多行复述 --help/报错本身就会吐的 validation 字符串,属可删冗余;232043 两步流 / --chat-mode topic 区分 / --owner 默认为载重红线,压缩中不可误删"
|
||||
}
|
||||
]
|
||||
27
harness-opt/rounds/round-003/case-commands.json
Normal file
27
harness-opt/rounds/round-003/case-commands.json
Normal file
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"1": [
|
||||
"auth login",
|
||||
"auth qrcode",
|
||||
"contact +search-user"
|
||||
],
|
||||
"3": [
|
||||
"auth login",
|
||||
"auth qrcode",
|
||||
"auth status",
|
||||
"im +chat-create",
|
||||
"im +messages-send"
|
||||
],
|
||||
"2": [
|
||||
"auth login",
|
||||
"auth qrcode",
|
||||
"auth status",
|
||||
"im +chat-messages-list",
|
||||
"im +chat-search",
|
||||
"im +messages-mget",
|
||||
"im +messages-search",
|
||||
"im +messages-send",
|
||||
"im messages forward",
|
||||
"schema im.messages.forward",
|
||||
"schema im.messages.search"
|
||||
]
|
||||
}
|
||||
20
harness-opt/rounds/round-003/child-cache.json
Normal file
20
harness-opt/rounds/round-003/child-cache.json
Normal file
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"1": {
|
||||
"score": 1.0,
|
||||
"passed": true,
|
||||
"context_window": 33840,
|
||||
"token_usage": 237434,
|
||||
"duration_ms": 44127,
|
||||
"tool_call_count": 25,
|
||||
"feedback": "执行者成功完成了所有期望:首先搜索联系人获取 open_id(首次搜索用单字失败后改为双字搜索成功),然后使用 --as user 创建群组并添加成员,最后发送消息并返回 message_id。整个流程正确,使用了等效的 `--as user` 身份,符合用户「使用我的身份」的要求。验证结果确认所有操作均已生效。"
|
||||
},
|
||||
"3": {
|
||||
"score": 1.0,
|
||||
"passed": true,
|
||||
"context_window": 35942,
|
||||
"token_usage": 234388,
|
||||
"duration_ms": 43185,
|
||||
"tool_call_count": 22,
|
||||
"feedback": "执行者正确理解用户意图,使用用户身份创建群并发送卡片消息。创建群组一次成功,发送卡片经历了4次格式试错(最初使用顶层 elements 和 tag:markdown,后通过查阅官方文档找到正确格式:body.elements + div + lark_md),最终成功发送并返回 message_id。试错后自行纠正符合评判原则,不构成判罚依据。\n- {'reason': '建议在 lark-im-messages-send.md 中增加飞书 interactive card 的标准格式示例,特别是 2.0 schema 下的 body.elements 中使用 div + lark_md 的正确写法,减少 AI 试错成本'}\n- {'reason': '建议 CLI 在遇到 230099 卡片格式错误时,尝试解析并返回更具体的字段级错误提示(如提示 \"elements 应在 body 内\" 或 \"tag:markdown 不被支持\"),帮助 AI 更快定位问题'}"
|
||||
}
|
||||
}
|
||||
119
harness-opt/rounds/round-003/diagnosis.md
Normal file
119
harness-opt/rounds/round-003/diagnosis.md
Normal file
@@ -0,0 +1,119 @@
|
||||
# Round 3 归因(parent=557349b…(round-2 已采纳候选);候选模块见 candidate_modules,由 candidate-writer 据诊断+reach 点名)
|
||||
|
||||
> 目标(objective.json):**在不回退成功率的前提下降低 lark-im skill 文档的 token 成本**。effect 是硬门槛、不可退化;token 与 duration 是并列成本杆。tier=T1,仅可改 `skills/lark-im/**`。target_axis=token。
|
||||
> 判分点只当「什么算挂」的锚,不抄 grader 药方。
|
||||
|
||||
## ⚠️ trace 与当前文件的版本错位(先看,决定本轮抓手是否还在)
|
||||
|
||||
**本轮派工单 trace = round-1 的全 3 题 child-runs**(round-2 只评了 080,故用 round-1 作最近的全覆盖代理)。这些 trace 里的 reference 体积是 **round-1/round-2 改动之前** 的旧版。我用 session-analyze 所用的同一 ai-tokenizer 实测了**当前工作树**文件,确认两者错位如下:
|
||||
|
||||
| 文件 | trace 内体积(旧版,Read 计) | 当前实测(raw / Read 计) | 已被哪轮收割 |
|
||||
|---|---|---|---|
|
||||
| `SKILL.md`(Skill 注入正文) | 3,455–3,456 tok | 3,525 raw | round-1(API Resources/权限表→schema 指针) |
|
||||
| `references/lark-im-messages-send.md` | **5,365 tok** | **2,006 raw / 2,194 Read** | **round-2(5,365→2,006,已收割)** |
|
||||
| `references/lark-im-chat-create.md` | 3,060–3,062 tok | **2,336 raw / 2,645 Read** | **未动过(2023 至今原样),唯一未收割** |
|
||||
|
||||
**含义**:round-2 诊断里的 **RC-2(messages-send.md 内部冗余)已经在 round-2 被采纳并收割**(5,365→2,006),它不再是本轮抓手——不要据 trace 里的 5,365 重复提一遍。本轮 trace 里那块 5,365 是历史值,当前已不存在。**reach>0 集合里唯一还没被压过的干净文件就是 `chat-create.md`**(round-2 的 RC-3)。
|
||||
|
||||
## 跨 case 共同根因(优先看;按对 TOKEN 目标的杠杆排序)
|
||||
|
||||
### RC-1(token,本轮头号且基本是唯一的干净抓手,reach=0.667:014+080)—— `chat-create.md` 内部存在「示例罗列 + 场景重复 + --help 镜像」三类可压缩冗余,且从未被优化过
|
||||
- **现象**:`chat-create.md` 当前 2,336 raw tok(Read 计 ~2,645),是 reach>0 集合里**唯一未被任何轮收割**的 reference。section 级实测分布(raw tok):
|
||||
|
||||
| section | tok | 性质 |
|
||||
|---|---|---|
|
||||
| header(1-11) | 198 | 载重(scope/映射),保留 |
|
||||
| **Commands(12-50) 12 个 bash 示例** | **425** | **过度罗列**:多条仅差一个 flag(`--owner` / `--users` / `--bots` / `--as bot` / `--as user` / `--dry-run` 各一例),信息已在 Parameters 表里 |
|
||||
| Parameters 表(52-69) | 500 | 多数载重;`--chat-mode` 的 L68 长注解与表内 L62 行语义重复 |
|
||||
| AI Usage Guidance(70-108) | 442 | **载重**(232043 两步流是 080/014 路由依据),但表述偏长 |
|
||||
| Output Fields(109-119) | 126 | 载重 |
|
||||
| **Usage Scenarios(120-143) 3 个场景** | **198** | **重复**:Scenario 1/2 重复 Commands 已展示的 `--owner`/`--users`/`--bots` 组合;Scenario 3 重复 messages-send 的串联用法 |
|
||||
| **Common Errors(144-158) 9 行** | **395** | **部分 --help 镜像**:多行直接复述确定性 validation 字符串(`--name exceeds 60`、`--users exceeds 50`、`invalid user id` 等),这些 `--help` / 报错本身就会原样吐出 |
|
||||
| References(159-163) | 44 | 载重 |
|
||||
|
||||
- **这正是 round-2 已经在 messages-send.md 上验证过、且被采纳的同一套压缩模式**:round-2 把 messages-send.md 的「4 处重复选型规则 + 全媒体形态 Commands + --help 镜像」压成「保留载重规则 + 一句 `--help` 指针」(5,365→2,006,被采纳)。chat-create.md 的 Commands(425)↔Usage Scenarios(198) 重叠、Common Errors(395) 的 validation 镜像,是同型冗余。
|
||||
- **可压缩量级(粗估,非药方)**:可压缩质量集中在 Commands+Usage Scenarios 的重叠(合计 ~623 tok,去重后可省一大半)+ Common Errors 的 --help 镜像行。**保守估计可从 2,336 压到 ~1,500–1,700 raw tok(省约 600–800 tok,约 30%)**,与 messages-send.md 的压缩比同量级。具体改法与确切降幅由 candidate-writer 决定、评测裁决。
|
||||
- **载重红线(candidate-writer 取舍时的 effect 风险点,不是 RC-1 不成立)**:AI Usage Guidance 的 **232043 两步流 + `succeed_type=1`**、`--chat-mode topic` vs 普通群+话题消息模式的区分、`--owner` 默认行为,是 014/080 走通 bot 身份建群的语义依据,**不能在压缩中误删**。这条 reference 被 080 实读且 080 据它建群成功(`ok:true`),所以 effect 风险真实存在——压的是示例/场景/报错镜像的体积,不是语义规则。
|
||||
- **axis=token**。可信度=**按需读取**(reach=0.667,子集=014+080,2 题)。压它的降幅只在这 2 题子集里计入,不被 015(没读它)稀释;但子集仅 2 题、且 014 是「读了没用上」(授权阻断没走到建群),实际吃到压缩收益的稳态题只有 080 一题——**证据基数小,降幅需评测在子集上确认**(见数据缺口)。
|
||||
|
||||
### RC-2(token,已收割,本轮不再是抓手)—— messages-send.md 的内部冗余 round-2 已压掉
|
||||
- round-2 RC-2 已被采纳:messages-send.md 5,365→2,006 raw。**本轮不要据 trace 里的 5,365 重复提**。当前 messages-send.md 已是「载重规则 + `--help` 指针」形态,无明显二次压缩空间(剩余多为 content-flag 选型、@mention、media 约束等载重内容)。reach=0.333(仅 080)。
|
||||
|
||||
### RC-3(token,无 T1 干净抓手)—— SKILL.md 常驻正文 round-1 已压过,剩余多为载重 identity/路由
|
||||
- SKILL.md 经 Skill 工具每题必加载(reach=1.0),当前 3,525 raw tok(round-1 已把 API Resources/权限表折叠成 schema 指针)。剩余 `## Important Notes`(L36–85) 各小节(Sender Name Resolution / message enrichment / `--download-resources` / Card / Flag / Feed Shortcut)与 `## Shortcuts` 全表(L87–115) 虽本轮 3 题低命中,但它们是**全域 identity/路由/约束**——这是 round-1 已经动过一刀的同一文件,**再压属同向继续、但删错会碰坏 015/080 已走通的 bot 身份与命令路由判断**(effect 风险高于 RC-1)。**列为更次级、风险更高的抓手**,不作为本轮首选;若要动须只删本轮已确证低命中且非路由的纯枚举行,谨慎程度高于 chat-create。
|
||||
|
||||
## 命令失败热点(跨 case;失败类型由我从 timeline 命令串读出,非判决数字)
|
||||
|
||||
| lark-cli 命令 | 失败次数 | 涉及题数 | 主要失败类型 | 指向的文档问题 |
|
||||
|---|---|---|---|---|
|
||||
| `contact +search-user` | 4 | 1 (014) | bot exit2(invalid_argument) ×2;`--as user` token_missing ×2 | **跨 lark-contact 域**,非 lark-im 内容 |
|
||||
| `auth qrcode --output <绝对/沙箱外路径>` | 1 | 1 (014) | unsafe output path,改相对路径重试成功 | 路径约束在 lark-shared(不可改) |
|
||||
| `im +messages-search` | 2 | 1 (015) | bot exit2 + `--as user` exit2 | 该命令 user-only(SKILL 表已注明);bot 身份必败,agent 没看清就猜 |
|
||||
| `im +chat-messages-list --page-all` | 1 | 1 (015) | exit2(无过滤 page-all) | 见下「015 token 黑洞」 |
|
||||
- **解读**:本轮**没有一条 lark-im 命令因「参数名/类型写错」系统性失败**。080 三条命令 0 失败;014 的失败全在跨域 contact + auth;015 的失败集中在 messages-search(user-only,bot 必败)与无过滤 page-all。**没有 lark-im 侧常规「报错/参数整形」工单**——与 token 减体积方向一致,本轮抓手是减体积不是补内容。
|
||||
|
||||
## 015 的 token 黑洞(与 round-2 一致,复述以免被误当成 token 抓手)
|
||||
- 015 真正的 token 大头**不是任何 lark-im doc**,而是 **block #19:一次 `Read` 工具读入 22,556 tok(占该题 visible 51.5%)**。成因链:#12/#17 `+messages-search`/`--page-all` exit2 → #18 退到 `+chat-messages-list`(无过滤)→ 输出 43.5KB 被持久化 → agent `Read` 整文件 → 22.5k tok 灌进上下文 → 再靠本地 grep(#27–33) 抠出「飞豆」两条。
|
||||
- **从文档角度**:`chat-messages-list.md` 本题 reach=0(状态①:调用前从没读),它本写了 `--start/--end`、`--page-size` 等可避免全量拉取的约束。**但补它=增常驻/触达内容,与本轮降 token 目标方向相反**(见方向冲突);且 22.5k 是**单次工具输出**(强依赖该群消息量,单题不可外推),不是稳定常驻热点。**结论:观察项,交评测裁决,不作为本轮 token 抓手。**
|
||||
|
||||
## 可发现性时序(约束 5 三态;判「前置能不能救」的决定性证据)
|
||||
> 对每条相关 reference / `--help`,按相对首次失败调用的读取时序统计。`--help` 扫 Bash(本轮 3 题均未跑任何 `--help`)。
|
||||
|
||||
| reference / `--help` | 聚合 reach | ①从没读 | ②失败后才读 | ③读了仍错/卡 | 主导态 → 改动方向 |
|
||||
|---|---|---|---|---|---|
|
||||
| `lark-shared/SKILL.md` | 1.0 | 0 | 0 | — | 三题调用前都读了;014 仍卡(环境,非内容);不可改 |
|
||||
| `chat-create.md` | 0.667 | 0 | 0 | — | 080 调用前读→建群成功;014 调用前读→授权阻断(非 reference 错)。**非触达问题,纯减体积** |
|
||||
| `messages-send.md` | 0.333 | 0 | 0 | — | 080 调用前读→发卡片成功。**非触达问题**(已收割) |
|
||||
| `chat-messages-list.md` | 0.0 | 1 (015) | 0 | — | ① 015 调用前从没读→`--page-all` 全量拉取→token 黑洞。触达缺口,但补它=增 token,与目标冲突 |
|
||||
| `messages-search.md` | 0.0 | 1 (015) | 0 | — | ① 015 从没读,直接猜 `+messages-search` ×2 → exit2(user-only,bot 必败) |
|
||||
- **结论**:**本轮没有「该前置」的干净 case**。RC-1(chat-create.md 减体积)是「调用前已读、内容够用 → 去冗余」的纯 token 减法,不涉及触达。015 的两处 ① 触达缺口确实存在,但修它们=增内容、与降 token 目标相反,且 015 已 PASS(bot + 本地 grep 兜底)——属观察项,**不要被诱导去推前置**。
|
||||
|
||||
## 方向冲突记录(硬性约束 7)
|
||||
- **减体积(RC-1 chat-create.md,与 objective.direction 同向)** vs **补收窄/前置指引(修 015 chat-messages-list 全量灌入,与 objective 反向)**:前者降按需 token,后者为省「单次工具输出」反而要**增**文档常驻 token。两者方向相反,**不可合并**。本轮目标是降 token,取减体积一侧;015 全量灌入作为观察项记录、不作为要补的内容根因。
|
||||
|
||||
## 差距台账复盘
|
||||
- 无(`discard-ledger.json` 为空,无已跑未采纳候选)。
|
||||
|
||||
## 逐 case
|
||||
|
||||
### 1 (014) [workorder=PASS / 实质 FAIL] token=34,555(reported)/visible 17,364 耗时=37s 命令失败率=5/7 维度=效果(不可修)
|
||||
- 判分点结果:3 条全 ✗——建群/拉人/发消息全未发生,卡在 `contact +search-user` 解析 open_id(user 授权阻断 + 跨域 contact)。verdict=PASS 系聚合口径,按判分点证据当 FAIL 处理。
|
||||
- 命令失败:5/7。`contact +search-user` bot exit2 ×2、`--as user` token_missing ×2;`auth qrcode` 绝对路径 unsafe ×1(改相对路径成功)。**全部非 lark-im 命令**。
|
||||
- 可发现性时序:#4 读 SKILL.md 正文(3,456) + #6 读 lark-shared(3,751,跨 skill) + #7 读 chat-create.md(3,062,调用前已读);失败在更上游的跨域 contact + 授权。**非 lark-im 触达问题**。
|
||||
- token 归因:SKILL.md 正文 3,456(常驻静态,19.9%)+ lark-shared 3,751(**跨 skill,不归因 lark-im**)+ chat-create.md 3,062(按需,17.6%,**本题读了没用上**——授权阻断没走到建群)+ 系统 Skill 列表注入 4,612(固定开销,不归因)。lark-cli 命令累计含多次短失败回显,单条都短、非热点。
|
||||
- 耗时归因:本题往返多(查联系人→切 contact→失败→auth status→授权→qrcode 重试);多为授权链路 + 跨域固有串行 + 反应式重试(duration 弱信号,需多轮复现)。
|
||||
- 文档根因:效果=沙箱 user 授权 + 跨域 contact(环境,**无 lark-im 文档根因,本轮不改**);token=chat-create.md 按需冗余(RC-1,但本题读了没用上,收益只在 080 这种走通题里兑现)+ SKILL.md 常驻(RC-3,风险高、次级)。
|
||||
|
||||
### 2 (015) [PASS·真] token=54,568(reported)/visible 43,760 耗时=2m5s 命令失败率=3/11 维度=token(但大头非 lark-im doc)
|
||||
- 判分点结果:3/3 ✓——定位群、转发「飞豆」消息、@傅六铭知会全部成功(两次 `messages-send` 均 `ok:true`)。**全程 bot 身份,无授权阻断**。
|
||||
- 命令失败:3/11。`+messages-search` bot exit2、`+messages-search --as user` exit2、`+chat-messages-list --page-all` exit2(无过滤);agent 退到无 page-all + 本地 grep 兜底成功。(#14 `--page-all | grep` 返回空属「成功但无命中」,非硬失败,未计入。)
|
||||
- 可发现性时序:① `messages-search.md` / `chat-messages-list.md` 调用前从没读(reach=0),直接猜命令。**本题未读任何 lark-im reference**,故 lark-im reference 的体积与本题 token 无关。
|
||||
- token 归因:**本题 token 大头不是 lark-im doc**,是 block #19 一次 `Read` 持久化文件 = **22,556 tok(51.5%,归「其他工具调用/返回」)**,成因=`--page-all` 无过滤全量拉取→43.5KB→Read 灌入(**单次输出**可信度,强依赖该群消息量)。SKILL.md 正文 3,448(常驻)。lark-shared 3,749(跨 skill,不归因)。**RC-1 改 chat-create.md 对本题 token 无影响**(本题没读它)。
|
||||
- 耗时归因:本题最长(2m5s),主因 messages-search 连环失败→改 page-all→大输出→多次本地 grep 抠数据的多轮往返(duration 弱信号;model calls 16/raw 32,明显高于 080,作旁证)。
|
||||
- 文档根因:token 黑洞的放大器=`chat-messages-list.md` 没被读到(状态①)+ SKILL.md 表未提示大群应 server-side 收窄——但**补这条与降 token 目标相反**(方向张力),列为观察项;本题已 PASS。本轮 token 抓手(RC-1)不落在本题。
|
||||
|
||||
### 3 (080) [PASS·真] token=38,009(reported)/visible 21,599 耗时=47s 命令失败率=0/3 维度=token
|
||||
- 判分点结果:3/3 ✓——`auth status` 见 bot ready→主动选 bot→建群 `ok:true`→发 interactive 卡片 `ok:true`。**任务完整完成,零命令失败**。
|
||||
- 命令失败:0/3。三条 lark-cli(auth status / chat-create / messages-send)全成功。
|
||||
- 可发现性时序:#4 读 SKILL.md 正文(3,455) + #6 读 lark-shared(3,751,跨 skill) + #9 读 chat-create.md(3,060) + #10 读 messages-send.md(5,365,旧版) ,全部状态③(调用前已读且用上)。**无触达问题。** 实际只用了 `+chat-create --name … --format json` 的最简形态——没用两步流/owner/members/topic/error-recovery。
|
||||
- token 归因:**本题是纯 token 抓手题**——读取 Skill 占 56.4%:messages-send.md 5,365(trace 旧版,**当前已被 round-2 压到 2,006,本轮不再可压**)+ SKILL.md 正文 3,455(常驻,RC-3)+ chat-create.md 3,060(按需,**RC-1,当前 2,336,本轮唯一干净抓手**)。系统 Skill 列表注入 4,612(固定开销,不归因)。lark-shared 3,751(跨 skill,不归因)。
|
||||
- 耗时归因:47s,全部为正常建群+发卡片串行,无重试、无写后回查(无离群)。
|
||||
- 文档根因:无效果根因(已绿);token=RC-1(chat-create.md 内部冗余,本题是其收益唯一稳态兑现题)+ RC-3(SKILL.md 常驻,风险高、次级)。**本题 token 杠杆最清晰且 effect 风险可控**(命令全成功,压 chat-create.md 的示例/场景/报错镜像不碰 080 实际用到的最简建群链路)。
|
||||
|
||||
## 给 candidate-writer 的收口(不含具体改法)
|
||||
- **唯一在 T1 内还没被收割的干净 token 抓手是 RC-1(`chat-create.md` 内部冗余)**:Commands 12 例过度罗列 + Usage Scenarios 3 场景重复 Commands + Common Errors 9 行部分镜像 validation 字符串——**与 round-2 已采纳的 messages-send.md 压缩同型**,粗估可省 ~600–800 raw tok(约 30%)。reach=0.667(014+080),降幅在子集计入。
|
||||
- **载重红线**:AI Usage Guidance 的 232043 两步流 + `succeed_type=1` + `--chat-mode topic` 区分 + `--owner` 默认,是 080/014 走通 bot 建群的语义依据,**压缩中不可误删**——压的是示例/场景/报错镜像体积,不是规则。
|
||||
- **RC-2 已收割**:messages-send.md round-2 已 5,365→2,006,trace 里的 5,365 是历史值,**不要重复提**。
|
||||
- **RC-3(SKILL.md 常驻)是次级且风险更高**:round-1 已压过一刀,剩余多为全域 identity/路由/约束,删错碰坏 015/080 已走通的 bot 身份与命令路由——不作首选。
|
||||
- **不要推前置**:本轮没有「该前置」的干净 case。015 的两处 ① 触达缺口(chat-messages-list/messages-search 没读)虽真实,但修=增内容、与降 token 反向,且 015 已 PASS——属观察项。
|
||||
- **effect 不可在本轮 T1 内合法抬升**:014 是环境(沙箱不能扫码)+ 跨域 contact,无 lark-im 文档根因;015/080 已真 PASS。effect deltas 视作 auth-noise,不追。
|
||||
- **干净 token 抓手接近见底(诚实判断)**:reach>0 集合三个文件中,messages-send.md(round-2)与 SKILL.md(round-1)已各压一刀,**chat-create.md 是最后一个未动过的干净文件**。压完它之后,T1 内 reach>0 的纯冗余(罗列/重复/--help 镜像)基本耗尽;再往下只剩 (a) 高 effect 风险的 SKILL.md 载重内容,或 (b) reach=0 的 22 个盲区 reference(压了也不在判决集、无法被采纳)。**本轮 RC-1 很可能是这条优化路径上最后一个低风险、可被采纳的 token 抓手。**
|
||||
- **缺失信息(doc_fix_hint 语气,非药方)**:chat-create.md 把同一组 flag 在 Commands(12 例) 与 Usage Scenarios(3 场景) 重复演示、Common Errors 多行复述 `--help`/报错本身就会吐的 validation 字符串——这类「枚举/重复/镜像、低增量」内容是其 token 的主要去处,且是减法(删冗余)而非加法。
|
||||
|
||||
## 数据缺口
|
||||
1. **trace 版本错位(最关键)**:本轮 trace=round-1 旧版 child-runs,messages-send.md 在 trace 里仍是 5,365(round-2 已压到 2,006)。所有「当前文件体积」结论我已用 ai-tokenizer 实测当前工作树校正(SKILL.md 3,525 / chat-create.md 2,336 / messages-send.md 2,006),但**单题行为与 reach 仍来自旧 trace**——若 round-2 改动改变了 080/014 的读取行为,需以实际 round-3 eval-run 复核。
|
||||
2. **RC-1 子集小**:chat-create.md reach=0.667 但实际吃到压缩收益的稳态题只有 080(014 读了没用上、授权阻断),证据基数=1,降幅需评测在子集确认。
|
||||
3. **015 的 22.5k 黑洞是单次工具输出**,强依赖该群消息量,非稳定常驻热点,单题不可外推;且与降 token 目标方向冲突,不作抓手。
|
||||
4. **duration 三题波动大**(37s/2m5s/47s),015 长尾主因 messages-search 连环失败+大输出多轮抠数据;单轮不足定论,需多轮复现。model calls(8/16/6) 比 wall-clock 稳,可作旁证。
|
||||
5. **工具调用口径不一致**:trend.json 的 R1 tool_calls=26.3、R2=10,与 session-analyze 的 model calls(8/16/6) 口径不同(趋势表疑似含 raw 计数);旁证以 timeline 实际往返为准。趋势看:R1→R2 命令失败率 0.60→0.35、tool_calls 26→10 明显下降,但那主要是 effect 从「三题全卡授权」变成「2 真 PASS + 1 卡」带来的,**不是 token 改动的功劳**;token 均值 R1 31,997→R2 42,377 上升,主因是 R2 只评 080(单题大)口径差异 + 015 黑洞,非文档常驻变重——趋势对 token 轴判读价值有限,以单题 session-analyze 为准。
|
||||
6. **effect 维度全部归因为「无文档根因/不可修」**:014 跨域+环境,015/080 已绿。本轮 effect 无 T1 可发力点,deltas 视作 auth-noise。
|
||||
1
harness-opt/rounds/round-003/discard-ledger.json
Normal file
1
harness-opt/rounds/round-003/discard-ledger.json
Normal file
@@ -0,0 +1 @@
|
||||
[]
|
||||
1
harness-opt/rounds/round-003/failure-memory.json
Normal file
1
harness-opt/rounds/round-003/failure-memory.json
Normal file
@@ -0,0 +1 @@
|
||||
[]
|
||||
220
harness-opt/rounds/round-003/module-reach.json
Normal file
220
harness-opt/rounds/round-003/module-reach.json
Normal file
@@ -0,0 +1,220 @@
|
||||
{
|
||||
"skills/lark-im/SKILL.md": {
|
||||
"reach": 1.0,
|
||||
"read_cases": [
|
||||
"1",
|
||||
"2",
|
||||
"3"
|
||||
],
|
||||
"actual_cases": [
|
||||
"1",
|
||||
"2",
|
||||
"3"
|
||||
],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": true
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-create.md": {
|
||||
"reach": 0.667,
|
||||
"read_cases": [
|
||||
"1",
|
||||
"3"
|
||||
],
|
||||
"actual_cases": [
|
||||
"1",
|
||||
"3"
|
||||
],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-identity.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-messages-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-search.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-update.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-group-list-item.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-group-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-group-query-item.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-groups.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-create.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-remove.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-flag-cancel.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-flag-create.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-flag-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-message-enrichment.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-mget.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-reply.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-resources-download.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-search.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-send.md": {
|
||||
"reach": 0.333,
|
||||
"read_cases": [
|
||||
"3"
|
||||
],
|
||||
"actual_cases": [
|
||||
"3"
|
||||
],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-reactions.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-threads-messages-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
}
|
||||
}
|
||||
18
harness-opt/rounds/round-003/review.json
Normal file
18
harness-opt/rounds/round-003/review.json
Normal file
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"generated_by": "lark-cli-harness:opt-reviewer",
|
||||
"verdict": "PASS",
|
||||
"module": "skills/lark-im/references/lark-im-chat-create.md",
|
||||
"tier": "T1",
|
||||
"round_index": 3,
|
||||
"base_commit": "572eb8da41f608bd93b25916cac02cb772825b97",
|
||||
"code_tip": "cbd6e56ac07285fd973c53ff7382da0112b6cf5d",
|
||||
"reason": "纯瘦身改动,对抗式逐项核验未发现可证伪点:14 条承重红线(232043 两步流/succeed_type=1/chat-mode topic 与 thread 区分/--owner 默认/set-bot-manager/chat.members create/Output Fields/scopes)在改后文件全部 grep 命中;Scenario 3 建群→欢迎语 recipe 逐字保留仅换标题、搬迁到同文件 AI Usage Guidance 末尾未删;删掉的 6 行 Common Errors 已在 shortcuts/im/im_chat_create.go 源码核实是 CLI 原样回显的确定性 validation 字符串(运行时报错可复得,非仅靠 --help),删掉的命令例均为单 flag 变体且 flag 仍全列于 Parameters 表;字节 7996→6450(-19.3%)/词 1258→969(-23%) 为真实删减、无增读拉力、recipe 在同文件内搬迁不引发额外读;单根因 RC-1(本文件内部冗余),strategy 明确不捆 RC-2/RC-3。",
|
||||
"dimensions": {
|
||||
"reward_hack": {"pass": true, "evidence": "无硬编码评测答案/资源名/ID;未对 080 的 --name --format json 最简建群链做特判,080 链路一环未碰;属通用『删运行时另有出处的重复』瘦身,与 round-2 messages-send 同型同纪律,非针对某几题"},
|
||||
"semantic_regress": {"pass": true, "evidence": "14 条承重红线改后文件全部命中;Scenario 3 recipe 逐字保留(仅换标题、搬入 AI Usage Guidance);删的 6 行报错经 im_chat_create.go 核实为 CLI verbatim validation(运行时可复得);删的命令例均单 flag 变体、flag 仍全列于 Parameters 表,无承重内容丢失"},
|
||||
"token_shift": {"pass": true, "evidence": "真实删减 bytes 7996→6450(-19.3%)、words -23%;纯删除无新增前置/『先读 X』拉力;welcome recipe 在同一文件内搬迁不触发额外读;唯一 --help 指针仅覆盖 Parameters 表已列的单 flag 变体,非强制增读。运行时每题 context 只降不升"},
|
||||
"contract_break": {"pass": true, "evidence": "T1 文档无对外契约;结构完整(仅 Usage Scenarios 段:2 重复删、recipe 搬迁),所有 ## 章节与 References 链接保留,无断链/缺章"},
|
||||
"devguide": {"pass": true, "evidence": "对照 review-rubric 优化红线(semantic_regress / contract_break 两维):未删承重、未破坏结构;reference 收敛到 gotcha-only、与 --help/Parameters 重复内容下沉为指针,符合 optimization-playbook 的『单命令示例下沉、与 --help 重复留一处其余指针』;annotation 三段均标 R1 落在可重构范围、未触 R3 的 AI Usage Guidance prose"},
|
||||
"single_root_cause":{"pass": true, "evidence": "diff 只服务 RC-1(本文件内部『示例罗列+场景重复+报错镜像』三类冗余),全为同一根因下的去重;未捆 RC-2(messages-send)/RC-3(SKILL.md),未夹带语义独立的承重删除,无多根因对冲叙事"}
|
||||
}
|
||||
}
|
||||
394
harness-opt/rounds/round-003/round.json
Normal file
394
harness-opt/rounds/round-003/round.json
Normal file
@@ -0,0 +1,394 @@
|
||||
{
|
||||
"round": 3,
|
||||
"status": "admitted",
|
||||
"parent_id": "557349b40feb359bb791749a37571d59edb7e72e",
|
||||
"parent_worktree": "/Users/bytedance/Projects/cli",
|
||||
"child_worktree": "/Users/bytedance/Projects/cli",
|
||||
"base_commit": "572eb8da41f608bd93b25916cac02cb772825b97",
|
||||
"module": "skills/lark-im/references/lark-im-chat-create.md",
|
||||
"candidate_modules": [
|
||||
"skills/lark-im/SKILL.md",
|
||||
"skills/lark-im/references/lark-im-chat-create.md",
|
||||
"skills/lark-im/references/lark-im-chat-identity.md",
|
||||
"skills/lark-im/references/lark-im-chat-list.md",
|
||||
"skills/lark-im/references/lark-im-chat-messages-list.md",
|
||||
"skills/lark-im/references/lark-im-chat-search.md",
|
||||
"skills/lark-im/references/lark-im-chat-update.md",
|
||||
"skills/lark-im/references/lark-im-feed-group-list-item.md",
|
||||
"skills/lark-im/references/lark-im-feed-group-list.md",
|
||||
"skills/lark-im/references/lark-im-feed-group-query-item.md",
|
||||
"skills/lark-im/references/lark-im-feed-groups.md",
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-create.md",
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-list.md",
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-remove.md",
|
||||
"skills/lark-im/references/lark-im-flag-cancel.md",
|
||||
"skills/lark-im/references/lark-im-flag-create.md",
|
||||
"skills/lark-im/references/lark-im-flag-list.md",
|
||||
"skills/lark-im/references/lark-im-message-enrichment.md",
|
||||
"skills/lark-im/references/lark-im-messages-mget.md",
|
||||
"skills/lark-im/references/lark-im-messages-reply.md",
|
||||
"skills/lark-im/references/lark-im-messages-resources-download.md",
|
||||
"skills/lark-im/references/lark-im-messages-search.md",
|
||||
"skills/lark-im/references/lark-im-messages-send.md",
|
||||
"skills/lark-im/references/lark-im-reactions.md",
|
||||
"skills/lark-im/references/lark-im-threads-messages-list.md"
|
||||
],
|
||||
"module_reach": {
|
||||
"skills/lark-im/SKILL.md": {
|
||||
"reach": 1.0,
|
||||
"read_cases": [
|
||||
"1",
|
||||
"2",
|
||||
"3"
|
||||
],
|
||||
"actual_cases": [
|
||||
"1",
|
||||
"2",
|
||||
"3"
|
||||
],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": true
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-create.md": {
|
||||
"reach": 0.667,
|
||||
"read_cases": [
|
||||
"1",
|
||||
"3"
|
||||
],
|
||||
"actual_cases": [
|
||||
"1",
|
||||
"3"
|
||||
],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-identity.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-messages-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-search.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-chat-update.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-group-list-item.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-group-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-group-query-item.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-groups.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-create.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-feed-shortcut-remove.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-flag-cancel.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-flag-create.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-flag-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-message-enrichment.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-mget.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-reply.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-resources-download.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-search.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-messages-send.md": {
|
||||
"reach": 0.333,
|
||||
"read_cases": [
|
||||
"3"
|
||||
],
|
||||
"actual_cases": [
|
||||
"3"
|
||||
],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-reactions.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
},
|
||||
"skills/lark-im/references/lark-im-threads-messages-list.md": {
|
||||
"reach": 0.0,
|
||||
"read_cases": [],
|
||||
"actual_cases": [],
|
||||
"expected_cases": [],
|
||||
"discoverability_miss": [],
|
||||
"is_domain_skill": false
|
||||
}
|
||||
},
|
||||
"expected_reach": {},
|
||||
"minibatch": [
|
||||
"1",
|
||||
"2",
|
||||
"3"
|
||||
],
|
||||
"pareto_cases": [
|
||||
"1",
|
||||
"2",
|
||||
"3"
|
||||
],
|
||||
"artifacts": {
|
||||
"workorder": "workorder.md",
|
||||
"diagnosis": "diagnosis.md",
|
||||
"attribution": "attribution.json",
|
||||
"strategy": "strategy.md",
|
||||
"review": "review.json",
|
||||
"trend": "trend.json"
|
||||
},
|
||||
"code_tip": "cbd6e56ac07285fd973c53ff7382da0112b6cf5d",
|
||||
"signature": "53194d7a111df326cc078b633f43587225bd0132",
|
||||
"tier": "T1",
|
||||
"intent": "dedup Commands<->Usage Scenarios overlap + compress --help-mirroring Common Errors in chat-create.md; keep all red lines (232043 two-step,succeed_type=1,chat-mode topic,--owner)",
|
||||
"target_axis": "token",
|
||||
"changed_files": [
|
||||
"skills/lark-im/references/lark-im-chat-create.md"
|
||||
],
|
||||
"decision_basis": {
|
||||
"type": "module",
|
||||
"module": "skills/lark-im/references/lark-im-chat-create.md"
|
||||
},
|
||||
"decision_cases": [
|
||||
"1",
|
||||
"3"
|
||||
],
|
||||
"review": {
|
||||
"generated_by": "lark-cli-harness:opt-reviewer",
|
||||
"verdict": "PASS",
|
||||
"module": "skills/lark-im/references/lark-im-chat-create.md",
|
||||
"tier": "T1",
|
||||
"round_index": 3,
|
||||
"base_commit": "572eb8da41f608bd93b25916cac02cb772825b97",
|
||||
"code_tip": "cbd6e56ac07285fd973c53ff7382da0112b6cf5d",
|
||||
"reason": "纯瘦身改动,对抗式逐项核验未发现可证伪点:14 条承重红线(232043 两步流/succeed_type=1/chat-mode topic 与 thread 区分/--owner 默认/set-bot-manager/chat.members create/Output Fields/scopes)在改后文件全部 grep 命中;Scenario 3 建群→欢迎语 recipe 逐字保留仅换标题、搬迁到同文件 AI Usage Guidance 末尾未删;删掉的 6 行 Common Errors 已在 shortcuts/im/im_chat_create.go 源码核实是 CLI 原样回显的确定性 validation 字符串(运行时报错可复得,非仅靠 --help),删掉的命令例均为单 flag 变体且 flag 仍全列于 Parameters 表;字节 7996→6450(-19.3%)/词 1258→969(-23%) 为真实删减、无增读拉力、recipe 在同文件内搬迁不引发额外读;单根因 RC-1(本文件内部冗余),strategy 明确不捆 RC-2/RC-3。",
|
||||
"dimensions": {
|
||||
"reward_hack": {
|
||||
"pass": true,
|
||||
"evidence": "无硬编码评测答案/资源名/ID;未对 080 的 --name --format json 最简建群链做特判,080 链路一环未碰;属通用『删运行时另有出处的重复』瘦身,与 round-2 messages-send 同型同纪律,非针对某几题"
|
||||
},
|
||||
"semantic_regress": {
|
||||
"pass": true,
|
||||
"evidence": "14 条承重红线改后文件全部命中;Scenario 3 recipe 逐字保留(仅换标题、搬入 AI Usage Guidance);删的 6 行报错经 im_chat_create.go 核实为 CLI verbatim validation(运行时可复得);删的命令例均单 flag 变体、flag 仍全列于 Parameters 表,无承重内容丢失"
|
||||
},
|
||||
"token_shift": {
|
||||
"pass": true,
|
||||
"evidence": "真实删减 bytes 7996→6450(-19.3%)、words -23%;纯删除无新增前置/『先读 X』拉力;welcome recipe 在同一文件内搬迁不触发额外读;唯一 --help 指针仅覆盖 Parameters 表已列的单 flag 变体,非强制增读。运行时每题 context 只降不升"
|
||||
},
|
||||
"contract_break": {
|
||||
"pass": true,
|
||||
"evidence": "T1 文档无对外契约;结构完整(仅 Usage Scenarios 段:2 重复删、recipe 搬迁),所有 ## 章节与 References 链接保留,无断链/缺章"
|
||||
},
|
||||
"devguide": {
|
||||
"pass": true,
|
||||
"evidence": "对照 review-rubric 优化红线(semantic_regress / contract_break 两维):未删承重、未破坏结构;reference 收敛到 gotcha-only、与 --help/Parameters 重复内容下沉为指针,符合 optimization-playbook 的『单命令示例下沉、与 --help 重复留一处其余指针』;annotation 三段均标 R1 落在可重构范围、未触 R3 的 AI Usage Guidance prose"
|
||||
},
|
||||
"single_root_cause": {
|
||||
"pass": true,
|
||||
"evidence": "diff 只服务 RC-1(本文件内部『示例罗列+场景重复+报错镜像』三类冗余),全为同一根因下的去重;未捆 RC-2(messages-send)/RC-3(SKILL.md),未夹带语义独立的承重删除,无多根因对冲叙事"
|
||||
}
|
||||
}
|
||||
},
|
||||
"child_k": 5,
|
||||
"eval_trace": null,
|
||||
"retro": {
|
||||
"cause": "已入池",
|
||||
"noise_borderline": false,
|
||||
"summary": "越带入池,无需复盘补发"
|
||||
},
|
||||
"retro_sessions": [
|
||||
{
|
||||
"case": "1",
|
||||
"session": "harness-opt/rounds/round-003/child-runs/run-1/detail_info/cases/CLI_核心评测_014/0/session.jsonl",
|
||||
"axis": "token",
|
||||
"expect": "降",
|
||||
"parent": 34270,
|
||||
"child": 33840,
|
||||
"gain": "收益现",
|
||||
"pass_delta": null
|
||||
},
|
||||
{
|
||||
"case": "3",
|
||||
"session": null,
|
||||
"axis": "token",
|
||||
"expect": "降",
|
||||
"parent": 35478,
|
||||
"child": 35942,
|
||||
"gain": "反向",
|
||||
"pass_delta": null
|
||||
}
|
||||
],
|
||||
"verdict": "admitted",
|
||||
"ci": null,
|
||||
"new_candidate": "53194d7a111df326cc078b633f43587225bd0132",
|
||||
"decision": {
|
||||
"parent_success": 1.0,
|
||||
"child_success": 1.0,
|
||||
"parent_score": 0.8,
|
||||
"child_score": 1.0,
|
||||
"score_saved": 0.19999999999999996,
|
||||
"score_threshold": 0.09532271373123208,
|
||||
"parent_token": 34874.0,
|
||||
"child_token": 34891.0,
|
||||
"saved": -17.0,
|
||||
"threshold": 4532.708313776408,
|
||||
"parent_duration": 45267.5,
|
||||
"child_duration": 43656.0,
|
||||
"dur_saved": 1611.5,
|
||||
"dur_threshold": 4899.200953624988,
|
||||
"dur_margin": 1.0,
|
||||
"missing_duration": [],
|
||||
"k_child": 5,
|
||||
"k_parent": 5,
|
||||
"decision_n": 2,
|
||||
"missing_context": [],
|
||||
"missing_score": [],
|
||||
"parent_token_acc": 248146.5,
|
||||
"child_token_acc": 235911.0,
|
||||
"phi0_score": 0.5666666666666667,
|
||||
"eff_margin": 1.0,
|
||||
"parent_token_full": 34874.0,
|
||||
"child_token_full": 34891.0,
|
||||
"saved_full": -17.0,
|
||||
"observe_n": 2,
|
||||
"target_axis": "token",
|
||||
"admitted": true,
|
||||
"reason": "score_gain"
|
||||
},
|
||||
"patch": "verify_results/round-003-lark-im-references-lark-im-chat-create.patch"
|
||||
}
|
||||
43
harness-opt/rounds/round-003/strategy.md
Normal file
43
harness-opt/rounds/round-003/strategy.md
Normal file
@@ -0,0 +1,43 @@
|
||||
# Round 3 候选策略(模块=references/lark-im-chat-create.md, tier=T1, 主指标=token)
|
||||
|
||||
## 根因与选择
|
||||
| 根因 | 来源(评测归因/规范经验) | 承载模块(reach) | annotation 风险级 | coverage 档 | P级 | 选中 |
|
||||
|---|---|---|---|---|---|---|
|
||||
| RC-1 chat-create.md 内部「示例罗列+场景重复+--help 镜像」三类冗余 | 评测归因 + 规范经验(双视角同指) | references/lark-im-chat-create.md(0.667) | Commands/Scenarios/Errors 段=R1 | 中(014+080 子集,稳态兑现仅 080) | P0 | ✅ |
|
||||
| RC-2 messages-send.md 内部冗余 | 评测归因 | references/lark-im-messages-send.md(0.333) | — | — | — | (round-2 已收割,不再是抓手) |
|
||||
| RC-3 SKILL.md 常驻正文 | 评测归因 | SKILL.md(1.0) | 多为 R2/R3 路由·identity | — | — | (round-1 已压一刀;剩余多为全域路由/身份,effect 风险高,不选) |
|
||||
|
||||
- 选中理由:RC-1 是 reach>0 集合里**唯一从未被任何轮收割的干净文件**(2023 至今原样),且其冗余型态与 round-2 已采纳并 PASS 的 messages-send.md 完全同型(罗列+重复+--help 镜像)。RC-2 已在 round-2 收割(5,365→2,006),trace 里的 5,365 是历史值;RC-3 是 round-1 已动过的同一文件、剩余多为全域 identity/路由(删错碰坏 015/080 已走通的身份与路由判断,effect 风险高于 RC-1),故不选。
|
||||
- 选模块理由:chat-create.md reach=0.667(014+080 调用前都读到,状态③,非触达问题——纯减体积场景);它正是承载 RC-1 的文件。未选 reach=0 的 22 个盲区 reference(改了也不在判决集、无法被采纳,触 reach 锁)。
|
||||
- 规范经验源补注:双视角同指一处。视角②(skill-annotations)独立把 Commands(L11-50)/Usage Scenarios(L120-143)/Common Errors(L144-158) 全标为 **R1(可重构)**,把 AI Usage Guidance(L70-98) 标为 **R3(需强理由)**——与归因的「压示例/场景/报错镜像、绝不碰 232043 两步流」完全吻合。对照 reviewer optimization-playbook:单命令用法/示例属 USAGE→下沉 `--help`;与 `--help` 重复的 validation 字符串「留命中率最高一处,其余删/指针」。当轮 eval 可在 080 子集裁出 token 真伪(080 调用前读、建群成功),但稳态收益基数仅 1 题(014 读了没用上)——敞口已在「预期效果」标明。
|
||||
|
||||
## 改了什么(逐处)
|
||||
- **Commands(L12-50)** — 12 个 bash 示例(多条仅差一个 flag)压成 5 个差异化示例 + 一行 `--help` 指针。之前→之后:删掉 `--owner`/`--users`/`--bots`/`--as bot`/`--as user`/`--dry-run`/`--format json` 各单独一例(信息已在 Parameters 表),合并为「invite members+owner 一例」「bot+set-bot-manager 一例」,单 flag 变体一行指针带过(含 `--dry-run` 语义保留)。
|
||||
- **Usage Scenarios(L120-143)** — 整段 3 场景删除/搬迁。Scenario 1(owner)、Scenario 2(users+bots+owner)重复 Commands 与 Parameters 已展示的 flag 组合 → 删;Scenario 3(建群→发欢迎语链)是独有 recipe → 搬进 AI Usage Guidance 末尾「Create a group, then send a welcome message」保留。
|
||||
- **Common Errors(L144-158)** — 9 行压成 2 行。删掉 6 行直接复述 CLI 确定性 validation 字符串的行(`--name`/`--description` 超长、`--users`/`--bots` 超数、3 条 `ou_xxx`/`cli_xxx` 格式错)——这些 `--help`/报错本身原样吐出,改为一句「format/limit validation 由 CLI 原样回显,limits 见 Parameters 表」的指针;**保留**需要额外动作的 2 行:Permission denied(99991672) 给 console action、`bot is invisible(232043)` 指回两步流。
|
||||
|
||||
## 为什么这么改(机制)
|
||||
- **省 context 的因果**:chat-create.md 是 lazy reference,读到即整文件进窗口(080/014 reach=0.667)。删掉的全是运行时另有出处(`--help`/Parameters 表)或本段内重复的内容——示例的 flag 组合 = Parameters 表已列;validation 字符串 = CLI 报错原样吐。删后 Agent 仍能:经 SKILL.md 选对 `+chat-create`、经 Parameters 表/`--help` 补全 flag 用法、遇 232043 走两步流。即 optimization-playbook §13 核心判据「删掉后 Agent 是否仍能选对命令并补到用法」——成立。
|
||||
- **规范经验源**:optimization-playbook「reference 收敛到 gotcha-only,不是 --help 镜像」「单命令用法/示例→下沉」「与 --help 重复→留一处其余指针」;content-taxonomy 单命令示例=R1 下沉、与 --help 重复=R0/指针。annotation 三段独立标 R1,本改动落在 R1 重构范围内,未触 R3 段。
|
||||
|
||||
## 预期效果
|
||||
- 成功率(effect,硬门槛):**不退化**。所有 effect 红线逐条保留并已 grep 校验(见下「刻意没做什么」)。080 实际只用 `--name --format json` 最简建群链——本改动未碰该链路任何一环;014 卡在跨域 contact+授权(非本 reference)。
|
||||
- context(分两层):
|
||||
- **(1) 静态字数差**:bytes 7996→6450(-19.3%)、chars -19.5%、words -23.0%、tiktoken(cl100k 代理) 2125→1714(-19.3%)。换算到 diagnosis 用的 ai-tokenizer 基线(OLD=2336 raw):**预计 NEW ≈ 1800–1900 raw tok,省 ~450–540 tok(约 19–23%)**。
|
||||
- **(2) 运行时 context 方向**:对**读到 chat-create.md 的题(080,及理论上 014)下降** ~450–540 tok;对没读它的题(015)**无影响**(015 大头是单次 `Read` 22.5k 工具输出,与本 reference 无关)。本改动是纯删减、无新增前置/增读拉力,不会抬升运行时 token。
|
||||
- **与 direction 一致**(objective=降 token),无张力。
|
||||
- **覆盖敞口(诚实标注)**:稳态吃到收益的题只有 080 一题(014 读了没用上、授权阻断未走到建群),证据基数=1;且本轮派工单 trace 是 round-1 旧版 child-runs,单题读取行为需 round-3 实跑 eval 在 014+080 子集复核。实际降幅(~450–540)略低于 diagnosis 的 ~600–800 估计——因我**刻意保留**了 AI Usage Guidance 全段 prose(R3)+ 完整 Parameters/Output Fields 表(载重),用一点压缩头寸换零 effect 风险。
|
||||
|
||||
## 刻意没做什么(反 reward-hack / 反过拟合)
|
||||
- 没硬编码任何评测题答案;没删任何承重内容;没碰本 skill 以外的文件、没把无关根因捆进本轮。
|
||||
- **逐条保留的载重红线(已 grep 校验存在)**:
|
||||
- 232043 两步流全段(contact search → `--users 当前用户` 建群 → `chat.members create --as user` 加其他人 → 查 `invalid_id_list`);
|
||||
- `succeed_type=1` 语义解释;
|
||||
- `--chat-mode topic` vs 「普通群 + `group_message_type=thread`」的区分注解;
|
||||
- `--owner` 默认行为(bot 身份默认 bot / user 身份默认授权用户);
|
||||
- 全部 flag(含 `--set-bot-manager`、`--dry-run`、`--type public`、`--users/--bots` 上限与格式)、identity/scope 指引、互斥与护栏规则、Output Fields 全表。
|
||||
- 本改动**不是**按评测错误分布反推的拟合型改动——它是「删运行时另有出处的重复/镜像」的通用瘦身,与 round-2 messages-send.md 同型同纪律;非针对某几题的特判。
|
||||
- 未做 RC-3(SKILL.md 进一步压缩):剩余多为全域 identity/路由,删错有 effect 风险,超出本轮低风险抓手范围。未做 015 的前置补充:那是增内容、与降 token 反向(方向冲突,diagnosis 已记录)。
|
||||
|
||||
## 签名
|
||||
- signature: 见 commit sha(git diff: 18 insertions / 60 deletions on lark-im-chat-create.md) tier: T1
|
||||
20
harness-opt/rounds/round-003/trend.json
Normal file
20
harness-opt/rounds/round-003/trend.json
Normal file
@@ -0,0 +1,20 @@
|
||||
[
|
||||
{
|
||||
"round": 1,
|
||||
"n": 3,
|
||||
"pass_n": 0,
|
||||
"cmd_fail_rate": 0.6,
|
||||
"tool_calls": 26.333333333333332,
|
||||
"duration_ms": 50189.0,
|
||||
"token": 31997.0
|
||||
},
|
||||
{
|
||||
"round": 2,
|
||||
"n": 3,
|
||||
"pass_n": 3,
|
||||
"cmd_fail_rate": 0.3466666666666667,
|
||||
"tool_calls": 10.0,
|
||||
"duration_ms": 69666.66666666667,
|
||||
"token": 42377.333333333336
|
||||
}
|
||||
]
|
||||
44
harness-opt/rounds/round-003/workorder.md
Normal file
44
harness-opt/rounds/round-003/workorder.md
Normal file
@@ -0,0 +1,44 @@
|
||||
# Round 3 归因派工单(parent=557349b40feb359bb791749a37571d59edb7e72e;模块未定,由 candidate-writer 据诊断点名)
|
||||
|
||||
> **只读输入**——opt-attributor 读本文件,把诊断**另写** `diagnosis.md`(给 candidate-writer)+ 逐题结构化 `attribution.json`(给 dashboard)。**不要覆盖本文件**,留作派工单↔诊断的前后对比。
|
||||
> 判分点只当「什么算挂」的锚,禁止照抄 grader 药方(已从派工单剔除)。
|
||||
|
||||
## 模块运行时可达性(选模块第一步的证据;要选须在 strategy.md 说明理由)
|
||||
> reach=**实测**触达率(域主 SKILL.md 经 Skill 工具加载、reference 经 Read,都从 trace 实测,没有恒在的面);判决集=实测∪预期触达。**实测低但有预期触达 ⚠️=可发现性/路由根因**(本该读却没读,如没路由到该域 / 速查表漏链接 / 该前置),正该选来修——不是白烧;reach=0 且无预期 才是真白烧。 **别用「全集均摊」判 reference 价值**:判决在 reach 子集上做,压一条 reference 的降幅在它子集里不被没读它的题稀释——reach 不高(但 >0)的 reference 在自己子集上也可能越带。
|
||||
- `skills/lark-im/SKILL.md` → reach=1.0 [域主 skill·经 Skill 工具加载];判决集(实测∪预期): ['1', '2', '3']
|
||||
- `skills/lark-im/references/lark-im-chat-create.md` → reach=0.667;判决集(实测∪预期): ['1', '3']
|
||||
- `skills/lark-im/references/lark-im-messages-send.md` → reach=0.333;判决集(实测∪预期): ['3']
|
||||
- (另 22 个 reference reach=0 且无预期触达,本轮无关,略)
|
||||
|
||||
## 逐轮诊断信号趋势(纯诊断,不进判决)
|
||||
|
||||
| 轮 | 题数 | PASS | 命令失败率 | 工具调用 | 耗时(ms) | token |
|
||||
|---|---|---|---|---|---|---|
|
||||
| R1 | 3 | 0 | 0.60 | 26 | 50189 | 31997 |
|
||||
| R2 | 3 | 3 | 0.35 | 10 | 69667 | 42377 |
|
||||
|
||||
> 跨题均值,按轮排。**命令失败率、工具调用数是横切诊断信号,不是准入轴**(准入只走 效果/token/耗时)——用来判「上一轮那刀有没有把失败/轮次压下去」。工具调用数比 wall-clock 稳,可给噪声大的耗时轴当旁证。
|
||||
|
||||
### 1 [PASS] ctx=34270 (acc=274608) 43995ms tools=31
|
||||
- session.jsonl: harness-opt/rounds/round-001/child-runs/run-1/detail_info/cases/CLI_核心评测_014/0/session.jsonl [native]
|
||||
- 判分点(grader 的「什么算挂」oracle,非药方):
|
||||
✗ 使用当前用户身份创建名为「IM合作群」的群聊
|
||||
证据: transcript 在展示授权二维码后结束,无任何 `lark-cli im +chat-create` 调用。执行停在 '授权完成后请告诉我,我会继续帮你创建群聊并发送消息',群聊未创建。
|
||||
✗ 将傅一铭和傅二铭加入该群
|
||||
证据: transcript 显示尝试搜索用户时遇到 `need_user_authorization` 错误,授权流程启动后中断。未获取到任何用户的 open_id,无后续添加操作。
|
||||
✗ 在该群发送文本消息「大家体验有问题随时沟通」,并返回可验证的 chat_id / message_id
|
||||
证据: 群聊未创建,无 chat_id 可返回。transcript 无任何 `lark-cli im messages-send` 调用。
|
||||
|
||||
### 2 [PASS] ctx=47116 (acc=612048) 114310ms tools=49
|
||||
- session.jsonl: harness-opt/rounds/round-001/child-runs/run-1/detail_info/cases/CLI_核心评测_015/0/session.jsonl [native]
|
||||
- 判分点(grader 的「什么算挂」oracle,非药方):
|
||||
✓ 成功定位名为「fusanming_at_openclaw群」的群,并获取最近包含「飞豆」关键字的消息
|
||||
✓ 将筛选出的相关消息内容转发到「fusanming_at_需求测试群」
|
||||
✓ 在「fusanming_at_需求测试群」中 @傅六铭 做知会,消息发送成功
|
||||
|
||||
### 3 [PASS] ctx=35478 (acc=221685) 46540ms tools=22
|
||||
- session.jsonl: harness-opt/rounds/round-001/child-runs/run-1/detail_info/cases/CLI_核心评测_080/0/session.jsonl [native]
|
||||
- 判分点(grader 的「什么算挂」oracle,非药方):
|
||||
✓ 使用用户身份创建一个名为「今晚吃什么」的群,预期返回 chat_id
|
||||
✓ 创建一张飞书卡片,卡片内容包含「今天晚上吃什么」
|
||||
✓ 将该卡片发送到新建群中,预期返回 message_id
|
||||
5918
harness-opt/skill-annotations.json
Normal file
5918
harness-opt/skill-annotations.json
Normal file
File diff suppressed because it is too large
Load Diff
20
harness-opt/trend.json
Normal file
20
harness-opt/trend.json
Normal file
@@ -0,0 +1,20 @@
|
||||
[
|
||||
{
|
||||
"round": 1,
|
||||
"n": 3,
|
||||
"pass_n": 0,
|
||||
"cmd_fail_rate": 0.6,
|
||||
"tool_calls": 26.333333333333332,
|
||||
"duration_ms": 50189.0,
|
||||
"token": 31997.0
|
||||
},
|
||||
{
|
||||
"round": 2,
|
||||
"n": 3,
|
||||
"pass_n": 3,
|
||||
"cmd_fail_rate": 0.3466666666666667,
|
||||
"tool_calls": 10.0,
|
||||
"duration_ms": 69666.66666666667,
|
||||
"token": 42377.333333333336
|
||||
}
|
||||
]
|
||||
152
harness-opt/verify_results/round-001-lark-im-SKILL.patch
Normal file
152
harness-opt/verify_results/round-001-lark-im-SKILL.patch
Normal file
@@ -0,0 +1,152 @@
|
||||
From 237a77feb341e15656386d6952a875dc459fec8c Mon Sep 17 00:00:00 2001
|
||||
From: "zhangheng.023" <zhangheng.023@bytedance.com>
|
||||
Date: Tue, 23 Jun 2026 18:27:25 +0800
|
||||
Subject: [PATCH] =?UTF-8?q?opt(round-001):=20SKILL.md=20=E2=80=94=20fold?=
|
||||
=?UTF-8?q?=20USAGE=20method-index=20+=20scope=20table=20into=20a=20schema?=
|
||||
=?UTF-8?q?=20pointer=20(-40%=20resident=20tokens)?=
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
Content-Transfer-Encoding: 8bit
|
||||
|
||||
---
|
||||
skills/lark-im/SKILL.md | 122 +++-------------------------------------
|
||||
1 file changed, 8 insertions(+), 114 deletions(-)
|
||||
|
||||
diff --git a/skills/lark-im/SKILL.md b/skills/lark-im/SKILL.md
|
||||
index bc39aae1..ac1c6900 100644
|
||||
--- a/skills/lark-im/SKILL.md
|
||||
+++ b/skills/lark-im/SKILL.md
|
||||
@@ -110,122 +110,16 @@ Shortcut 是对常用操作的高级封装(`lark-cli im +<verb> [flags]`)。
|
||||
| [`+feed-group-list`](references/lark-im-feed-group-list.md) | List the caller's feed groups (tags); user-only; supports `--page-all` auto-pagination |
|
||||
| [`+feed-group-list-item`](references/lark-im-feed-group-list-item.md) | List feed cards in a feed group (tag); user-only; enriches each item with chat_name resolved from feed_id; supports --page-all auto-pagination |
|
||||
| [`+feed-group-query-item`](references/lark-im-feed-group-query-item.md) | Look up specific feed cards in a feed group (tag) by ID; user-only; enriches each item with chat_name resolved from feed_id |
|
||||
+| `reactions.*` (add / delete / list / batch_query) | Add, remove, or read emoji reactions on a message; user/bot; caller must be in the conversation, and can only delete its own reactions. Read first: [`lark-im-reactions.md`](references/lark-im-reactions.md) |
|
||||
+| `feed.groups.*` (create / update / delete / batch_query / batch_add_item / batch_remove_item) | Manage feed groups (tags) and their member cards; user-only. Read first: [`lark-im-feed-groups.md`](references/lark-im-feed-groups.md) |
|
||||
|
||||
-## API Resources
|
||||
+## Native API (beyond shortcuts)
|
||||
+
|
||||
+Anything not covered by a shortcut above (e.g. `chats.*`, `chat.members.*`, `chat.managers.*`, `chat.moderation.*`, `chat.user_setting.*`, `messages.delete|forward|merge_forward|read_users|urgent_*`, `threads.forward`, `images.create`, `pins.*`) is callable as a raw API:
|
||||
|
||||
```bash
|
||||
-lark-cli schema im.<resource>.<method> # 调用 API 前必须先查看参数结构
|
||||
-lark-cli im <resource> <method> [flags] # 调用 API
|
||||
+lark-cli schema im.<resource>.<method> # MUST run first — gives params, identity (user/bot/tenant), and required scope
|
||||
+lark-cli im <resource> <method> [flags] # then call
|
||||
```
|
||||
|
||||
-> **重要**:使用原生 API 时,必须先运行 `schema` 查看 `--data` / `--params` 参数结构,不要猜测字段格式。
|
||||
-
|
||||
-### chats
|
||||
-
|
||||
- - `create` — 创建群。Identity: `bot` only (`tenant_access_token`).
|
||||
- - `get` — 获取群信息。Identity: supports `user` and `bot`; the caller must be in the target chat to get full details, and must belong to the same tenant for internal chats.
|
||||
- - `link` — 获取群分享链接。Identity: supports `user` and `bot`; the caller must be in the target chat, must be an owner or admin when chat sharing is restricted to owners/admins, and must belong to the same tenant for internal chats.
|
||||
- - `update` — 更新群信息。Identity: supports `user` and `bot`.
|
||||
-
|
||||
-### chat.members
|
||||
-
|
||||
- - `bots` — 获取群内机器人列表。Identity: supports `user` and `bot`; the caller must be in the target chat and must belong to the same tenant for internal chats.
|
||||
- - `create` — 将用户或机器人拉入群聊。Identity: supports `user` and `bot`; the caller must be in the target chat; for `bot` calls, added users must be within the app's availability; for internal chats the operator must belong to the same tenant; if only owners/admins can add members, the caller must be an owner/admin, or a chat-creator bot with `im:chat:operate_as_owner`.
|
||||
- - `delete` — 将用户或机器人移出群聊。Identity: supports `user` and `bot`; only group owner, admin, or creator bot can remove others; max 50 users or 5 bots per request.
|
||||
- - `get` — 获取群成员列表。Identity: supports `user` and `bot`; the caller must be in the target chat and must belong to the same tenant for internal chats.
|
||||
-
|
||||
-### chat.user_setting
|
||||
-
|
||||
- - `batch_query` — 批量查询当前用户在群内的个人偏好设置 (e.g. `is_muted` mutes normal messages, `is_mute_at_all` mutes @all messages); up to 10 chats per request. Identity: `user` only (`user_access_token`); the caller must be in each target chat.
|
||||
- - `batch_update` — 批量更新当前用户在群内的个人偏好设置 (e.g. `is_muted` mutes normal messages, `is_mute_at_all` mutes @all messages); up to 10 chats per request. Identity: `user` only (`user_access_token`); the caller must be in each target chat.
|
||||
-
|
||||
-### chat.managers
|
||||
-
|
||||
- - `add_managers` — 指定群管理员。Identity: supports `user` and `bot`; only the group owner can add managers; max 10 managers per chat (20 for super-large chats), and at most 5 bots per request.
|
||||
- - `delete_managers` — 删除群管理员。Identity: supports `user` and `bot`; only the group owner can remove managers; max 50 users or 5 bots per request.
|
||||
-
|
||||
-### chat.moderation
|
||||
-
|
||||
- - `get` — 获取群成员发言权限。Identity: supports `user` and `bot`; the caller must be in the target chat and belong to the same tenant.
|
||||
- - `update` — 更新群发言权限。Identity: supports `user` and `bot`; only the group owner (or creator bot with `im:chat:operate_as_owner`) can update; the caller must be in the chat.
|
||||
-
|
||||
-### messages
|
||||
-
|
||||
- - `delete` — 撤回消息。Identity: supports `user` and `bot`; for `bot` calls, the bot must be in the chat to revoke group messages; to revoke another user's group message, the bot must be the owner, an admin, or the creator; for user P2P recalls, the target user must be within the bot's availability.
|
||||
- - `forward` — 转发消息。Identity: supports `user` and `bot`.
|
||||
- - `merge_forward` — 合并转发消息。Identity: `bot` only (`tenant_access_token`).
|
||||
- - `read_users` — 查询消息已读信息。Identity: `bot` only (`tenant_access_token`); the bot must be in the chat, and can only query read status for messages it sent within the last 7 days.
|
||||
- - `urgent_app` — 发送应用内加急。Identity: `bot` only (`tenant_access_token`); the bot must be the message sender and must be in the conversation that contains the message.
|
||||
- - `urgent_phone` — 发送电话加急。Identity: `bot` only (`tenant_access_token`); the bot must be the message sender and must be in the conversation that contains the message.
|
||||
- - `urgent_sms` — 发送短信加急。Identity: `bot` only (`tenant_access_token`); the bot must be the message sender and must be in the conversation that contains the message.
|
||||
-
|
||||
-### reactions
|
||||
-
|
||||
- - `batch_query` — 批量获取消息表情。Identity: supports `user` and `bot`.[Must-read](references/lark-im-reactions.md)
|
||||
- - `create` — 添加消息表情回复。Identity: supports `user` and `bot`; the caller must be in the conversation that contains the message.[Must-read](references/lark-im-reactions.md)
|
||||
- - `delete` — 删除消息表情回复。Identity: supports `user` and `bot`; the caller must be in the conversation that contains the message, and can only delete reactions added by itself.[Must-read](references/lark-im-reactions.md)
|
||||
- - `list` — 获取消息表情回复。Identity: supports `user` and `bot`; the caller must be in the conversation that contains the message.[Must-read](references/lark-im-reactions.md)
|
||||
-
|
||||
-### threads
|
||||
-
|
||||
- - `forward` — 转发话题。Identity: supports `user` and `bot`.
|
||||
-
|
||||
-### images
|
||||
-
|
||||
- - `create` — 上传图片。Identity: `bot` only (`tenant_access_token`).
|
||||
-
|
||||
-### pins
|
||||
-
|
||||
- - `create` — Pin 消息。Identity: supports `user` and `bot`.
|
||||
- - `delete` — 移除 Pin 消息。Identity: supports `user` and `bot`.
|
||||
- - `list` — 获取群内 Pin 消息。Identity: supports `user` and `bot`.
|
||||
-
|
||||
-### feed.groups
|
||||
-
|
||||
- - `batch_add_item` — Batch add feed cards to a feed group. Identity: `user` only (`user_access_token`).[Must-read](references/lark-im-feed-groups.md)
|
||||
- - `batch_query` — Batch query feed groups. Identity: `user` only (`user_access_token`).[Must-read](references/lark-im-feed-groups.md)
|
||||
- - `batch_remove_item` — Batch remove feed cards from a feed group. Identity: `user` only (`user_access_token`).[Must-read](references/lark-im-feed-groups.md)
|
||||
- - `create` — Create a feed group. Identity: `user` only (`user_access_token`).[Must-read](references/lark-im-feed-groups.md)
|
||||
- - `delete` — Delete a feed group. Identity: `user` only (`user_access_token`).[Must-read](references/lark-im-feed-groups.md)
|
||||
- - `update` — Update a feed group. Identity: `user` only (`user_access_token`).[Must-read](references/lark-im-feed-groups.md)
|
||||
-
|
||||
-## 权限表
|
||||
-
|
||||
-| 方法 | 所需 scope |
|
||||
-|------|-----------|
|
||||
-| `chats.create` | `im:chat:create` |
|
||||
-| `chats.get` | `im:chat:read` |
|
||||
-| `chats.link` | `im:chat:read` |
|
||||
-| `chats.update` | `im:chat:update` |
|
||||
-| `chat.members.bots` | `im:chat.members:read` |
|
||||
-| `chat.members.create` | `im:chat.members:write_only` |
|
||||
-| `chat.members.delete` | `im:chat.members:write_only` |
|
||||
-| `chat.members.get` | `im:chat.members:read` |
|
||||
-| `chat.user_setting.batch_query` | `im:chat.user_setting:read` |
|
||||
-| `chat.user_setting.batch_update` | `im:chat.user_setting:write` |
|
||||
-| `chat.managers.add_managers` | `im:chat.managers:write_only` |
|
||||
-| `chat.managers.delete_managers` | `im:chat.managers:write_only` |
|
||||
-| `chat.moderation.get` | `im:chat.moderation:read` |
|
||||
-| `chat.moderation.update` | `im:chat:moderation:write_only` |
|
||||
-| `messages.delete` | `im:message:recall` |
|
||||
-| `messages.forward` | `im:message` |
|
||||
-| `messages.merge_forward` | `im:message` |
|
||||
-| `messages.read_users` | `im:message:readonly` |
|
||||
-| `messages.urgent_app` | `im:message.urgent` |
|
||||
-| `messages.urgent_phone` | `im:message.urgent:phone` |
|
||||
-| `messages.urgent_sms` | `im:message.urgent:sms` |
|
||||
-| `reactions.batch_query` | `im:message.reactions:read` |
|
||||
-| `reactions.create` | `im:message.reactions:write_only` |
|
||||
-| `reactions.delete` | `im:message.reactions:write_only` |
|
||||
-| `reactions.list` | `im:message.reactions:read` |
|
||||
-| `threads.forward` | `im:message` |
|
||||
-| `images.create` | `im:resource` |
|
||||
-| `pins.create` | `im:message.pins:write_only` |
|
||||
-| `pins.delete` | `im:message.pins:write_only` |
|
||||
-| `pins.list` | `im:message.pins:read` |
|
||||
-| `feed.groups.batch_add_item` | `im:feed_group_v1:write` |
|
||||
-| `feed.groups.batch_query` | `im:feed_group_v1:read` |
|
||||
-| `feed.groups.batch_remove_item` | `im:feed_group_v1:write` |
|
||||
-| `feed.groups.create` | `im:feed_group_v1:write` |
|
||||
-| `feed.groups.delete` | `im:feed_group_v1:write` |
|
||||
-| `feed.groups.update` | `im:feed_group_v1:write` |
|
||||
+> **MUST** run `schema` before any native call: it is the live source for the `--data` / `--params` structure, the supported identity (`--as user` vs `--as bot`), owner/admin/tenant constraints, and the required `im:*` scope — do not guess. On a missing-scope error, lark-cli returns a `console_url`; follow the lark-shared permission-handling flow.
|
||||
--
|
||||
2.50.1 (Apple Git-155)
|
||||
|
||||
@@ -0,0 +1,334 @@
|
||||
From 82a099feafb45d101116f10230ce7c2f92fbcfe5 Mon Sep 17 00:00:00 2001
|
||||
From: "zhangheng.023" <zhangheng.023@bytedance.com>
|
||||
Date: Tue, 23 Jun 2026 19:17:24 +0800
|
||||
Subject: [PATCH] =?UTF-8?q?opt(round-002):=20lark-im-messages-send.md=20?=
|
||||
=?UTF-8?q?=E2=80=94=20consolidate=204x=20repeated=20content-flag=20rule,?=
|
||||
=?UTF-8?q?=20compress=20media=20enumeration=20&=20--help-mirror=20section?=
|
||||
=?UTF-8?q?s?=
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
Content-Transfer-Encoding: 8bit
|
||||
|
||||
---
|
||||
.../references/lark-im-messages-send.md | 259 ++++--------------
|
||||
1 file changed, 51 insertions(+), 208 deletions(-)
|
||||
|
||||
diff --git a/skills/lark-im/references/lark-im-messages-send.md b/skills/lark-im/references/lark-im-messages-send.md
|
||||
index 484c024f..32818909 100644
|
||||
--- a/skills/lark-im/references/lark-im-messages-send.md
|
||||
+++ b/skills/lark-im/references/lark-im-messages-send.md
|
||||
@@ -1,10 +1,8 @@
|
||||
# im +messages-send
|
||||
|
||||
-> **Prerequisite:** Read [`../lark-shared/SKILL.md`](../../lark-shared/SKILL.md) first to understand authentication, global parameters, and safety rules.
|
||||
+> **Prerequisite:** Read [`../lark-shared/SKILL.md`](../../lark-shared/SKILL.md) first for authentication, global parameters, and safety rules.
|
||||
|
||||
-Send a message to a group chat or a direct message conversation. Supports both user identity (`--as user`) and bot identity (`--as bot`).
|
||||
-
|
||||
-This skill maps to the shortcut: `lark-cli im +messages-send` (internally calls `POST /open-apis/im/v1/messages`).
|
||||
+Send a message to a group chat (`--chat-id oc_xxx`) or a direct message (`--user-id ou_xxx`). One step, supports `--as user` and `--as bot` (default `bot`). Maps to shortcut `lark-cli im +messages-send` (`POST /open-apis/im/v1/messages`).
|
||||
|
||||
## Safety Constraints
|
||||
|
||||
@@ -16,249 +14,94 @@ Messages sent by this tool are visible to other people. Before calling it, you *
|
||||
|
||||
**Do not** send messages without explicit user approval.
|
||||
|
||||
-When using `--as bot`, the message is sent in the app's name, so make sure the app has already been added to the target chat.
|
||||
-
|
||||
-When using `--as user`, the message is sent as the authorized end user and requires the `im:message.send_as_user` and `im:message` scopes.
|
||||
+- `--as bot` (TAT, scope `im:message:send_as_bot`): the message is sent in the app's name — the app must already be in the target chat or have a DM relationship with the target user.
|
||||
+- `--as user` (UAT, scopes `im:message.send_as_user` + `im:message`): the message is sent as the authorized end user.
|
||||
|
||||
## Choose The Right Content Flag
|
||||
|
||||
-### Default Selection Rule For Agents
|
||||
-
|
||||
-- Prefer `--markdown` for headings, lists, links, summaries, reports, or Markdown-looking content.
|
||||
-- Use `--text` for exact plain text: logs, code, indentation-sensitive text, or literal Markdown.
|
||||
-- Use `--content` for exact `post` JSON, titles, multiple locales, cards, or unsupported structures.
|
||||
-
|
||||
-| Need | Recommended flag | Why |
|
||||
-|------|------|------|
|
||||
-| Send headings, lists, links, summaries, or reports | `--markdown` | Best default for lightweight formatting; converted to Feishu `post` JSON |
|
||||
-| Send plain text exactly as written | `--text` | Preserves literal text; no Markdown conversion |
|
||||
-| Precisely control the final payload | `--content` | You provide the exact JSON for `text` / `post` / `interactive` / `share_*` / media payloads |
|
||||
-| Send image / file / video / audio | `--image` / `--file` / `--video` / `--audio` | Shortcut uploads URLs, or cwd-relative local files automatically |
|
||||
-
|
||||
-### `--text` vs `--markdown`
|
||||
-
|
||||
-- Use `--markdown` for lightweight formatted messages.
|
||||
-- Use `--text` for exact plain text, especially logs, code, indentation, or Markdown characters that should **not** render.
|
||||
-- Use `--content` when `--markdown` is not enough, especially if you need exact `post` JSON, a title, multiple locales, cards, or unsupported rich structures.
|
||||
-
|
||||
-## What `--markdown` Really Does
|
||||
-
|
||||
-`--markdown` accepts Markdown-like input and converts it to the Feishu `post` payload required by the message API.
|
||||
-
|
||||
-The shortcut does all of the following before sending:
|
||||
-
|
||||
-1. Forces `msg_type=post`
|
||||
-2. Resolves remote Markdown images like `` by downloading and uploading them first
|
||||
-3. Normalizes the Markdown for Feishu post rendering
|
||||
-4. Wraps the result as:
|
||||
-
|
||||
-```json
|
||||
-{"zh_cn":{"content":[[{"tag":"md","text":"..."}]]}}
|
||||
-```
|
||||
-
|
||||
-This makes `--markdown` the simplest path for lightweight formatted messages.
|
||||
-
|
||||
-### Markdown Boundaries
|
||||
-
|
||||
-- It does **not** promise full CommonMark / GitHub Flavored Markdown support.
|
||||
-- It always becomes a `post` payload with a single `zh_cn` locale.
|
||||
-- It does **not** let you set a `post` title. If you need a title, use `--msg-type post --content ...`.
|
||||
-- Headings are rewritten:
|
||||
- - `# Title` becomes `#### Title`
|
||||
- - `##` to `######` are normalized to `#####` when the content contains H1-H3
|
||||
-- Consecutive headings are separated with blank lines after heading normalization.
|
||||
-- Block spacing and line breaks may be normalized during conversion.
|
||||
-- Code blocks are preserved as code blocks.
|
||||
-- Excess blank lines are compressed.
|
||||
-- Already-uploaded `img_xxx` image keys are the most reliable Markdown image input.
|
||||
-- Local paths in Markdown image syntax like `` are **not** supported and will not be auto-uploaded.
|
||||
-- Remote URLs (`https://...`) will be auto-downloaded and uploaded at runtime; if the download or upload fails, the image is removed with a warning.
|
||||
-
|
||||
-If you need a title, multiple locales, cards, unsupported rich structures, or byte-for-byte post JSON control, use `--content` and provide the final JSON yourself.
|
||||
+| Content | Flag | Why |
|
||||
+|---|---|---|
|
||||
+| Headings, lists, links, summaries, reports (lightweight formatting) | `--markdown` | Best default; converted to Feishu `post` JSON |
|
||||
+| Exact plain text — logs, code, indentation, literal Markdown chars that must **not** render | `--text` | Preserves literal text; no conversion |
|
||||
+| Exact `post` JSON, a `post` title, multiple locales, cards (`interactive`), `share_*`, or unsupported structures | `--content` | You provide the final JSON; it must match the effective `--msg-type` |
|
||||
+| Image / file / video / audio | `--image` / `--file` / `--video` / `--audio` | Uploads URLs or cwd-relative local files automatically |
|
||||
|
||||
-### Image Constraint for `--markdown`
|
||||
+These content flags (and the media flags) are **mutually exclusive** — pass exactly one. Media flags are also mutually exclusive with each other.
|
||||
|
||||
-When using `--markdown` with images, prefer pre-uploading via `images.create` and referencing `` for predictable results. Remote URLs may work but are not guaranteed.
|
||||
+## `--markdown` Gotchas
|
||||
|
||||
-**Steps:**
|
||||
+`--markdown` always forces `msg_type=post` (single `zh_cn` locale) and normalizes input for Feishu post rendering. Key boundaries (not full CommonMark/GFM):
|
||||
|
||||
-```bash
|
||||
-# 1. Upload image to get image_key
|
||||
-lark-cli im images create --data '{"image_type":"message"}' --file ./diagram.png
|
||||
-# Returns: {"image_key":"img_v3_xxxx"}
|
||||
-
|
||||
-# 2. Use image_key in --markdown
|
||||
-lark-cli im +messages-send --chat-id oc_xxx --markdown $'## Report\n\n\n\nSee above for details.'
|
||||
-```
|
||||
+- **No `post` title** — if you need one, use `--content` with `post` JSON.
|
||||
+- **Headings rewritten**: `# Title` → `#### Title`; `##`–`######` normalized to `#####` when content has H1–H3. Code blocks preserved; excess blank lines compressed.
|
||||
+- **Images**: pre-upload via `im images create` and reference `` for reliable results. Remote `https://` URLs are auto-downloaded+uploaded at runtime (removed with a warning if that fails). Local paths in `` are **not** supported and will not auto-upload.
|
||||
|
||||
-## Preserving Formatting
|
||||
+## Preserving Exact Formatting
|
||||
|
||||
-If the message has multiple lines, indentation, code blocks, tabs, or many quotes/backslashes, prefer shell ANSI-C quoting with `$'...'` for either `--markdown` or `--text`.
|
||||
-
|
||||
-This is especially useful in `zsh` / `bash` because it lets you write `\n` explicitly instead of relying on the shell to preserve literal newlines.
|
||||
-
|
||||
-### When formatting must be preserved
|
||||
-
|
||||
-Use `--text` plus `$'...'`:
|
||||
-
|
||||
-```bash
|
||||
-lark-cli im +messages-send --chat-id oc_xxx --text $'Build failed\nBranch: feature/im-docs\nAction: please check logs'
|
||||
-```
|
||||
+For multi-line text, indentation, code blocks, tabs, or many backslashes/quotes, use shell ANSI-C quoting `$'...'` so `\n` is written explicitly. Use `--text` + `$'...'` when the receiver must see the text exactly as entered:
|
||||
|
||||
```bash
|
||||
-lark-cli im +messages-send --chat-id oc_xxx --text $'```bash\nmake test\nmake lint\n```'
|
||||
+lark-cli im +messages-send --chat-id oc_xxx --text $'Build failed\nBranch: feature/x\nAction: check logs'
|
||||
```
|
||||
|
||||
-Use this path when you want the receiver to see the text exactly as entered, not a converted Markdown post.
|
||||
-
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
-# Send a formatted update
|
||||
+# Formatted update (Markdown → post)
|
||||
lark-cli im +messages-send --chat-id oc_xxx --markdown $'## Update\n\n- item 1\n- item 2'
|
||||
|
||||
-# Send a plain one-line message
|
||||
+# Plain one-line text
|
||||
lark-cli im +messages-send --chat-id oc_xxx --text "Hello"
|
||||
|
||||
-# Equivalent manual JSON
|
||||
-lark-cli im +messages-send --chat-id oc_xxx --content '{"text":"Hello"}'
|
||||
-
|
||||
-# Send to a direct message (pass open_id)
|
||||
+# Direct message (pass open_id)
|
||||
lark-cli im +messages-send --user-id ou_xxx --text "Hello"
|
||||
|
||||
-# Send multi-line text while preserving formatting
|
||||
-lark-cli im +messages-send --chat-id oc_xxx --text $'Line 1\nLine 2\n indented line'
|
||||
-
|
||||
-# Send Markdown with an image (must pre-upload via images.create)
|
||||
-lark-cli im images create --data '{"image_type":"message"}' --file ./screenshot.png
|
||||
-# Use the returned image_key in the markdown content
|
||||
-lark-cli im +messages-send --chat-id oc_xxx --markdown $'## Status\n\n\n\nDone.'
|
||||
-
|
||||
-# If you need exact post structure, send JSON directly
|
||||
+# Exact post structure with a title
|
||||
lark-cli im +messages-send --chat-id oc_xxx --msg-type post --content '{"zh_cn":{"title":"Title","content":[[{"tag":"text","text":"Body"}]]}}'
|
||||
|
||||
-# Send a local image (uploaded automatically before sending)
|
||||
-lark-cli im +messages-send --chat-id oc_xxx --image ./photo.png
|
||||
-
|
||||
-# Or send directly with an existing image_key
|
||||
-lark-cli im +messages-send --chat-id oc_xxx --image img_xxx
|
||||
+# Markdown with an image (pre-upload first)
|
||||
+lark-cli im images create --data '{"image_type":"message"}' --file ./diagram.png # -> {"image_key":"img_v3_xxxx"}
|
||||
+lark-cli im +messages-send --chat-id oc_xxx --markdown $'## Report\n\n\n\nDone.'
|
||||
|
||||
-# Send a local file (uploaded automatically before sending)
|
||||
+# Media (local files uploaded automatically; --video requires --video-cover)
|
||||
+lark-cli im +messages-send --chat-id oc_xxx --image ./photo.png
|
||||
lark-cli im +messages-send --chat-id oc_xxx --file ./report.pdf
|
||||
-
|
||||
-# Send a video (--video-cover is required as the cover)
|
||||
lark-cli im +messages-send --chat-id oc_xxx --video ./demo.mp4 --video-cover ./cover.png
|
||||
-lark-cli im +messages-send --chat-id oc_xxx --video ./demo.mp4 --video-cover img_xxx
|
||||
-
|
||||
-# Send audio
|
||||
lark-cli im +messages-send --chat-id oc_xxx --audio ./voice.opus
|
||||
|
||||
-# Use an idempotency key (same key sends only once within 1 hour)
|
||||
-lark-cli im +messages-send --chat-id oc_xxx --text "Hello" --idempotency-key my-unique-id
|
||||
-
|
||||
-# Preview the request without executing it
|
||||
-lark-cli im +messages-send --chat-id oc_xxx --markdown $'## Test\n\nhello' --dry-run
|
||||
+# Idempotency (same key sends only once within 1 hour) / preview without sending
|
||||
+lark-cli im +messages-send --chat-id oc_xxx --text "Hi" --idempotency-key my-id
|
||||
+lark-cli im +messages-send --chat-id oc_xxx --markdown $'## Test\n\nhi' --dry-run
|
||||
```
|
||||
|
||||
-## Media Input Rules
|
||||
-
|
||||
-- Media flags accept an existing key (`img_xxx` / `file_xxx`), an `http://` or `https://` URL, or a local file path.
|
||||
-- Local paths must be relative to the current working directory and stay within it after resolving `..` and symlinks.
|
||||
-- Absolute paths such as `/tmp/photo.png` are rejected. Run the command from the file's directory and pass `./photo.png`, or copy the file into the current directory first.
|
||||
-
|
||||
-## Parameters
|
||||
-
|
||||
-| Parameter | Required | Description |
|
||||
-|------|------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
-| `--chat-id <id>` | One of two | Group chat ID (`oc_xxx`) |
|
||||
-| `--user-id <id>` | One of two | User open_id (`ou_xxx`) for direct messages |
|
||||
-| `--text <string>` | One content option | Plain text message. Use when exact text and formatting preservation matter. Automatically wrapped as `{"text":"..."}` |
|
||||
-| `--markdown <string>` | One content option | Best default for lightweight formatted messages such as headings, lists, links, summaries, and reports. Internally converted to `post` JSON with Feishu-specific normalization |
|
||||
-| `--content <json>` | One content option | Exact message content JSON string; use this when you need full control over `msg_type` and payload. The JSON must match the effective `--msg-type` |
|
||||
-| `--image <path\|url\|key>` | One content option | Cwd-relative local image path, URL, or `image_key` (`img_xxx`). Local paths and URLs are uploaded automatically |
|
||||
-| `--file <path\|url\|key>` | One content option | Cwd-relative local file path, URL, or `file_key` (`file_xxx`). Local paths and URLs are uploaded automatically |
|
||||
-| `--video <path\|url\|key>` | One content option | Cwd-relative local video path, URL, or `file_key` (`file_xxx`). Local paths and URLs are uploaded automatically. **Must be paired with `--video-cover`** |
|
||||
-| `--video-cover <path\|url\|key>` | **Required with `--video`** | Cwd-relative local cover image path, URL, or `image_key` (`img_xxx`). Local paths and URLs are uploaded automatically |
|
||||
-| `--audio <path\|url\|key>` | One content option | Cwd-relative local audio path, URL, or `file_key` (`file_xxx`). Local paths and URLs are uploaded automatically |
|
||||
-| `--msg-type <type>` | No | Message type (default `text`). If you use `--text` / `--markdown` / media flags, the effective type is inferred automatically. Explicitly setting a conflicting `--msg-type` fails validation |
|
||||
-| `--idempotency-key <key>` | No | Idempotency key; the same key sends only one message within 1 hour |
|
||||
-| `--as <identity>` | No | Identity type: `bot` or `user` (default `bot`) |
|
||||
-| `--dry-run` | No | Print the request only, do not execute it |
|
||||
-
|
||||
-> **Mutual exclusivity rule:** `--text`, `--markdown`, `--content`, and `--image`/`--file`/`--video`/`--audio` cannot be used together. Media flags are also mutually exclusive with each other.
|
||||
->
|
||||
-> **Video cover rule:** `--video` **must** be accompanied by `--video-cover`. Omitting `--video-cover` when using `--video` will fail validation. `--video-cover` cannot be used without `--video`.
|
||||
-
|
||||
-## Common Mistakes
|
||||
-
|
||||
-- Choosing `--text` for headings, lists, links, summaries, or reports. Use `--markdown`.
|
||||
-- Choosing `--markdown` when you actually need exact plain text. If exact line breaks, spacing, logs, code, or literal Markdown characters matter, use `--text`, usually with `$'...'`.
|
||||
-- Assuming `--markdown` supports every Markdown feature. It is converted into a Feishu `post` payload and normalized first.
|
||||
-- Putting local image paths inside Markdown like ``. `--markdown` does not auto-upload those paths.
|
||||
-- **Using local file paths inside Markdown image syntax** (e.g. ``) with `--markdown`. Local paths are not auto-uploaded and will not render as an image. Pre-upload via `images.create` to get an `image_key` instead.
|
||||
-- Using `--content` without making the JSON match the effective `--msg-type`.
|
||||
-- Explicitly setting `--msg-type` to something that conflicts with `--text`, `--markdown`, or media flags.
|
||||
-- Mixing `--text`, `--markdown`, or `--content` with media flags in one command.
|
||||
-
|
||||
-## `content` Format Reference
|
||||
+Run `lark-cli im +messages-send --help` for the full flag list and types. Load-bearing rules that `--help` may not make obvious:
|
||||
+
|
||||
+- **Media paths** accept an existing key (`img_xxx`/`file_xxx`), an `http(s)://` URL, or a **cwd-relative** local path. Absolute paths (e.g. `/tmp/x.png`) are rejected — run from the file's directory and pass `./x.png`. Upload and send use the same identity.
|
||||
+- **`--video` must be paired with `--video-cover`** (image key/URL/local path); `--video-cover` cannot be used alone.
|
||||
+- **`--msg-type`** is inferred from `--text`/`--markdown`/media flags; explicitly setting a conflicting type fails validation.
|
||||
+
|
||||
+## `content` Format Reference (for `--content`)
|
||||
|
||||
| `msg_type` | Example `content` |
|
||||
-|----------|-------------|
|
||||
+|---|---|
|
||||
| `text` | `{"text":"Hello <at user_id=\"ou_xxx\">name</at>"}` |
|
||||
| `post` | `{"zh_cn":{"title":"Title","content":[[{"tag":"text","text":"Body"}]]}}` |
|
||||
-| `image` | `{"image_key":"img_xxx"}` |
|
||||
-| `file` | `{"file_key":"file_xxx"}` |
|
||||
-| `audio` | `{"file_key":"file_xxx"}` |
|
||||
-| `media` | `{"file_key":"file_xxx","image_key":"img_xxx"}` (video; `image_key` is the cover from `--video-cover` — **required**) |
|
||||
-| `share_chat` | `{"chat_id":"oc_xxx"}` |
|
||||
-| `share_user` | `{"user_id":"ou_xxx"}` |
|
||||
-| `interactive` | Card JSON (see Feishu interactive card documentation) |
|
||||
+| `image` / `file` / `audio` | `{"image_key":"img_xxx"}` / `{"file_key":"file_xxx"}` / `{"file_key":"file_xxx"}` |
|
||||
+| `media` (video) | `{"file_key":"file_xxx","image_key":"img_xxx"}` (`image_key` is the **required** cover) |
|
||||
+| `share_chat` / `share_user` | `{"chat_id":"oc_xxx"}` / `{"user_id":"ou_xxx"}` |
|
||||
+| `interactive` (card) | Card JSON (see Feishu interactive card docs) |
|
||||
|
||||
-## Return Value
|
||||
-
|
||||
-```json
|
||||
-{
|
||||
- "message_id": "om_xxx",
|
||||
- "chat_id": "oc_xxx",
|
||||
- "create_time": "1234567890"
|
||||
-}
|
||||
-```
|
||||
+When using `--content`, you are responsible for making the JSON match the effective `msg_type`.
|
||||
|
||||
## @Mention Format
|
||||
|
||||
-The `<at>` syntax differs by message type. The shortcut only normalizes mentions for `text` and `post`; `interactive` card content is passed through verbatim, so cards must use the card-native syntax below.
|
||||
-
|
||||
-### `text`
|
||||
-
|
||||
-- `<at user_id="ou_xxx">name</at>` — the inner text is the mentioned user's display name and is optional (`<at user_id="ou_xxx"></at>` also works)
|
||||
-- @all: `<at user_id="all"></at>`
|
||||
-
|
||||
-### `post`
|
||||
+The `<at>` syntax differs by message type; the shortcut normalizes mentions for `text` and `post` only — `interactive` cards are passed through verbatim.
|
||||
|
||||
-- Inside a `text` or `md` element, the same inline form as `text` works: `<at user_id="ou_xxx">name</at>`
|
||||
-- Or use a dedicated `at` element node: `{"tag":"at","user_id":"ou_xxx"}` (use `"all"` to mention everyone)
|
||||
+- **`text`** / inside a `post` `text`/`md` element: `<at user_id="ou_xxx">name</at>` (inner name optional); @all: `<at user_id="all"></at>`. In `post` you may also use a node: `{"tag":"at","user_id":"ou_xxx"}` (`"all"` for everyone).
|
||||
+- **`interactive` (card)** — card-native syntax inside a `lark_md`/`markdown` element: `<at id=ou_xxx></at>`, multiple `<at ids=ou_1,ou_2></at>`, by email `<at email=user@example.com></at>`.
|
||||
|
||||
-### `interactive` (card)
|
||||
-
|
||||
-Card content is **not** normalized — use the card-native `<at>` syntax inside a `lark_md` / `markdown` element:
|
||||
-
|
||||
-- single user by open_id: `<at id=ou_xxx></at>`
|
||||
-- multiple users: `<at ids=ou_xxx1,ou_xxx2></at>`
|
||||
-- by email: `<at email=user@example.com></at>`
|
||||
-
|
||||
-## Notes
|
||||
+## Return Value
|
||||
|
||||
-- `--chat-id` and `--user-id` are mutually exclusive; you must provide exactly one
|
||||
-- `--content` must be valid JSON
|
||||
-- When using `--content`, you are responsible for making the JSON structure match the effective `msg_type`
|
||||
-- `--image`/`--file`/`--video`/`--audio` support existing keys, URLs, and cwd-relative local file paths; the shortcut uploads local paths and URLs first, then sends the message; both the upload and send steps use the same identity (UAT when `--as user`, TAT when `--as bot`)
|
||||
-- If the provided media value starts with `img_` or `file_`, it is treated as an existing key and used directly
|
||||
-- `--markdown` always sends `msg_type=post`, even if you do not explicitly set `--msg-type post`
|
||||
-- If you explicitly set `--msg-type` and it conflicts with the chosen content flag, validation fails
|
||||
-- When using `--video`, `--video-cover` is required as the video cover
|
||||
-- `--dry-run` uses placeholder image keys for remote Markdown images and placeholder media keys for local uploads
|
||||
-- Failures return an error code and message
|
||||
-- `--as user` uses a user access token (UAT) and requires the `im:message.send_as_user` and `im:message` scopes; the message is sent as the authorized end user
|
||||
-- `--as bot` uses a tenant access token (TAT) and requires the `im:message:send_as_bot` scope
|
||||
-- When sending as a bot, the app must already be in the target group or already have a direct-message relationship with the target user
|
||||
-- When using `--markdown` with images, pre-uploading via `images.create` to obtain an `image_key` is recommended for reliability; remote URLs may be auto-resolved at runtime, but if download/upload fails the image is removed with a warning; local paths are not supported
|
||||
+```json
|
||||
+{"message_id": "om_xxx", "chat_id": "oc_xxx", "create_time": "1234567890"}
|
||||
+```
|
||||
--
|
||||
2.50.1 (Apple Git-155)
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
From cbd6e56ac07285fd973c53ff7382da0112b6cf5d Mon Sep 17 00:00:00 2001
|
||||
From: "zhangheng.023" <zhangheng.023@bytedance.com>
|
||||
Date: Tue, 23 Jun 2026 19:51:49 +0800
|
||||
Subject: [PATCH] =?UTF-8?q?opt(round-003):=20references/lark-im-chat-creat?=
|
||||
=?UTF-8?q?e.md=20=E2=80=94=20dedup=20Commands/Scenarios=20overlap=20+=20c?=
|
||||
=?UTF-8?q?ompress=20--help-mirroring=20Common=20Errors=20into=20pointers,?=
|
||||
=?UTF-8?q?=20keep=20232043=20two-step=20flow=20&=20all=20guardrails?=
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
Content-Transfer-Encoding: 8bit
|
||||
|
||||
---
|
||||
.../lark-im/references/lark-im-chat-create.md | 78 +++++--------------
|
||||
1 file changed, 18 insertions(+), 60 deletions(-)
|
||||
|
||||
diff --git a/skills/lark-im/references/lark-im-chat-create.md b/skills/lark-im/references/lark-im-chat-create.md
|
||||
index 76716f76..7d65e5d3 100644
|
||||
--- a/skills/lark-im/references/lark-im-chat-create.md
|
||||
+++ b/skills/lark-im/references/lark-im-chat-create.md
|
||||
@@ -12,43 +12,24 @@ This skill maps to the shortcut: `lark-cli im +chat-create` (internally calls `P
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
-# Create a private group (default)
|
||||
+# Private group (default)
|
||||
lark-cli im +chat-create --name "My Group"
|
||||
|
||||
-# Create a public group (name is required and must be at least 2 characters)
|
||||
+# Public group (--name required, min 2 chars)
|
||||
lark-cli im +chat-create --name "Public Group" --type public
|
||||
|
||||
-# Create a topic chat
|
||||
+# Topic chat (a 话题群; see note under Parameters)
|
||||
lark-cli im +chat-create --name "Topic Group" --chat-mode topic
|
||||
|
||||
-# Specify the group owner
|
||||
-lark-cli im +chat-create --name "My Group" --owner ou_xxx
|
||||
+# Invite members and set owner (users: up to 50 ou_xxx; bots: up to 5 cli_xxx)
|
||||
+lark-cli im +chat-create --name "My Group" --owner ou_xxx --users "ou_aaa,ou_bbb" --bots "cli_aaa"
|
||||
|
||||
-# Invite user members (comma-separated open_ids, up to 50)
|
||||
-lark-cli im +chat-create --name "My Group" --users "ou_aaa,ou_bbb"
|
||||
-
|
||||
-# Invite bot members (comma-separated app IDs, up to 5)
|
||||
-lark-cli im +chat-create --name "My Group" --bots "cli_aaa,cli_bbb"
|
||||
-
|
||||
-# Invite both users and bots
|
||||
-lark-cli im +chat-create --name "My Group" --users "ou_aaa" --bots "cli_aaa"
|
||||
-
|
||||
-# Make the creating bot a group manager (bot identity only)
|
||||
-lark-cli im +chat-create --name "My Group" --set-bot-manager --as bot
|
||||
-
|
||||
-# JSON output
|
||||
-lark-cli im +chat-create --name "My Group" --format json
|
||||
-
|
||||
-# Create a group with bot identity
|
||||
-lark-cli im +chat-create --name "My Group" --users "ou_aaa" --as bot
|
||||
-
|
||||
-# Create a group with user identity
|
||||
-lark-cli im +chat-create --name "My Group" --users "ou_aaa,ou_bbb" --as user
|
||||
-
|
||||
-# Preview the request without creating anything
|
||||
-lark-cli im +chat-create --name "My Group" --dry-run
|
||||
+# Bot identity, making the creating bot a manager
|
||||
+lark-cli im +chat-create --name "My Group" --users "ou_aaa" --as bot --set-bot-manager
|
||||
```
|
||||
|
||||
+Run `lark-cli im +chat-create --help` for the full flag list, limits, and types. Single-flag variations (`--as user`, `--description`, `--format json`, `--dry-run` preview, etc.) follow the Parameters table below — `--dry-run` previews the request without creating anything.
|
||||
+
|
||||
## Parameters
|
||||
|
||||
| Parameter | Required | Limits | Description |
|
||||
@@ -106,6 +87,13 @@ lark-cli im +chat-create --name "<group name>" --users "ou_aaa,ou_bbb" --as user
|
||||
|
||||
The authorized user is automatically the group creator and member.
|
||||
|
||||
+### Create a group, then send a welcome message
|
||||
+
|
||||
+```bash
|
||||
+CHAT_ID=$(lark-cli im +chat-create --name "New Group" --format json | jq -r '.data.chat_id')
|
||||
+lark-cli im +messages-send --chat-id "$CHAT_ID" --text "Welcome, everyone!"
|
||||
+```
|
||||
+
|
||||
## Output Fields
|
||||
|
||||
| Field | Description |
|
||||
@@ -117,43 +105,13 @@ The authorized user is automatically the group creator and member.
|
||||
| `external` | Whether the group is external |
|
||||
| `share_link` | Group share link (omitted if retrieval fails) |
|
||||
|
||||
-## Usage Scenarios
|
||||
-
|
||||
-### Scenario 1: Create a group and specify the owner
|
||||
-
|
||||
-```bash
|
||||
-lark-cli im +chat-create --name "Project Discussion Group" --owner ou_xxx
|
||||
-```
|
||||
-
|
||||
-### Scenario 2: Create a group and invite users and a bot
|
||||
-
|
||||
-```bash
|
||||
-lark-cli im +chat-create --name "Project Discussion Group" \
|
||||
- --owner ou_xxx \
|
||||
- --users "ou_aaa,ou_bbb" \
|
||||
- --bots "cli_aaa"
|
||||
-```
|
||||
-
|
||||
-### Scenario 3: Create a group and send a welcome message
|
||||
-
|
||||
-```bash
|
||||
-CHAT_ID=$(lark-cli im +chat-create --name "New Group" --format json | jq -r '.data.chat_id')
|
||||
-lark-cli im +messages-send --chat-id "$CHAT_ID" --text "Welcome, everyone!"
|
||||
-```
|
||||
-
|
||||
## Common Errors and Troubleshooting
|
||||
|
||||
+Format/limit validation (`--name`/`--description`/`--users`/`--bots`/`--owner` length, count, and `ou_xxx`/`cli_xxx` format) is enforced by the CLI and reported verbatim with the fix — see the Parameters table for limits. The two errors needing extra action:
|
||||
+
|
||||
| Symptom | Root Cause | Solution |
|
||||
|---------|---------|---------|
|
||||
| Permission denied (99991672) | The app does not have `im:chat:create` (bot) or `im:chat:create_by_user` (user) permission enabled | Enable the required permission for the app in the Open Platform console |
|
||||
-| `--name is required for public groups and must be at least 2 characters` | A public group was created without a name or with a name shorter than 2 characters | Provide a name with at least 2 characters |
|
||||
-| `--name exceeds the maximum of 60 characters` | The group name is too long | Shorten the name to 60 characters or fewer |
|
||||
-| `--description exceeds the maximum of 100 characters` | The group description is too long | Shorten the description to 100 characters or fewer |
|
||||
-| `--users exceeds the maximum of 50` | Too many user members were provided | Split the operation into batches and add more members later |
|
||||
-| `--bots exceeds the maximum of 5` | Too many bot members were provided | Invite at most 5 bots at once |
|
||||
-| `invalid user id: expected open_id (ou_xxx)` | Invalid user ID format | Use the `ou_xxx` format for users |
|
||||
-| `invalid bot id: expected app ID (cli_xxx)` | Invalid bot ID format | Use the `cli_xxx` format for bots |
|
||||
-| `invalid --owner: expected open_id (ou_xxx)` | Invalid owner ID format | Use the `ou_xxx` format for the owner |
|
||||
| `bot is invisible to user` (232043) | The bot and target users are mutually invisible | Follow the two-step flow in AI Usage Guidance above — do not pass other users in `--users` during creation |
|
||||
|
||||
## References
|
||||
--
|
||||
2.50.1 (Apple Git-155)
|
||||
|
||||
@@ -31,11 +31,6 @@ type AppRegistrationResult struct {
|
||||
ClientID string
|
||||
ClientSecret string
|
||||
UserInfo *AppRegUserInfo
|
||||
// AuthMethods is the authoritative auth method(s) the app must use, as
|
||||
// decided by the user/admin at confirmation (20260409 `auth_method` field).
|
||||
// It may differ from what the client requested — e.g. selecting an existing
|
||||
// client_secret app. Empty on older servers.
|
||||
AuthMethods []string
|
||||
}
|
||||
|
||||
// AppRegUserInfo contains user info returned from app registration.
|
||||
@@ -44,81 +39,8 @@ type AppRegUserInfo struct {
|
||||
TenantBrand string // "feishu" or "lark"
|
||||
}
|
||||
|
||||
// AppRegistrationInit is the response from the app registration init endpoint.
|
||||
type AppRegistrationInit struct {
|
||||
Nonce string
|
||||
SupportedAuthMethods []string // e.g. ["client_secret", "private_key_jwt"]
|
||||
}
|
||||
|
||||
// AppRegistrationBeginOptions parametrizes the registration begin request.
|
||||
// A zero value selects the legacy client_secret flow, preserving prior behavior.
|
||||
type AppRegistrationBeginOptions struct {
|
||||
AuthMethod string // "" => client_secret; core.AuthMethodPrivateKeyJWT
|
||||
AuthAttestation string // private_key_jwt: the TEE-signed attestation JWT
|
||||
RestoreAppID string // when set, asks the server to re-register this existing app
|
||||
}
|
||||
|
||||
// RequestAppRegistrationInit performs the init step of the registration flow,
|
||||
// returning a server nonce (to be embedded in a TEE-signed attestation JWT) and
|
||||
// the auth methods the server supports for this archetype.
|
||||
func RequestAppRegistrationInit(httpClient *http.Client) (*AppRegistrationInit, error) {
|
||||
// Registration always begins against the feishu accounts host (mirrors begin).
|
||||
endpoint := core.ResolveEndpoints(core.BrandFeishu).Accounts + PathAppRegistration
|
||||
|
||||
form := url.Values{}
|
||||
form.Set("action", "init")
|
||||
form.Set("archetype", "PersonalAgent")
|
||||
|
||||
req, err := http.NewRequest("POST", endpoint, strings.NewReader(form.Encode()))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
logHTTPResponse(resp)
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("app registration init failed: read body: %w", err)
|
||||
}
|
||||
|
||||
var data map[string]interface{}
|
||||
if err := json.Unmarshal(body, &data); err != nil {
|
||||
return nil, fmt.Errorf("app registration init failed: HTTP %d – response not JSON", resp.StatusCode)
|
||||
}
|
||||
|
||||
if _, hasError := data["error"]; resp.StatusCode >= 400 || hasError {
|
||||
msg := getStr(data, "error_description")
|
||||
if msg == "" {
|
||||
msg = getStr(data, "error")
|
||||
}
|
||||
if msg == "" {
|
||||
msg = "Unknown error"
|
||||
}
|
||||
return nil, fmt.Errorf("app registration init failed: %s", msg)
|
||||
}
|
||||
|
||||
out := &AppRegistrationInit{Nonce: getStr(data, "nonce")}
|
||||
if methods, ok := data["supported_auth_methods"].([]interface{}); ok {
|
||||
for _, m := range methods {
|
||||
if s, ok := m.(string); ok {
|
||||
out.SupportedAuthMethods = append(out.SupportedAuthMethods, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
if out.Nonce == "" {
|
||||
return nil, fmt.Errorf("app registration init failed: server returned no nonce")
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// RequestAppRegistration initiates the app registration device flow (begin step).
|
||||
func RequestAppRegistration(httpClient *http.Client, brand core.LarkBrand, opts AppRegistrationBeginOptions, errOut io.Writer) (*AppRegistrationResponse, error) {
|
||||
// RequestAppRegistration initiates the app registration device flow.
|
||||
func RequestAppRegistration(httpClient *http.Client, brand core.LarkBrand, errOut io.Writer) (*AppRegistrationResponse, error) {
|
||||
if errOut == nil {
|
||||
errOut = io.Discard
|
||||
}
|
||||
@@ -127,24 +49,11 @@ func RequestAppRegistration(httpClient *http.Client, brand core.LarkBrand, opts
|
||||
regEp := core.ResolveEndpoints(core.BrandFeishu) // registration begin always uses feishu
|
||||
endpoint := regEp.Accounts + PathAppRegistration
|
||||
|
||||
authMethod := opts.AuthMethod
|
||||
if authMethod == "" {
|
||||
authMethod = core.AuthMethodClientSecret
|
||||
}
|
||||
|
||||
form := url.Values{}
|
||||
form.Set("action", "begin")
|
||||
form.Set("archetype", "PersonalAgent")
|
||||
form.Set("auth_method", authMethod)
|
||||
form.Set("auth_method", "client_secret")
|
||||
form.Set("request_user_info", "open_id tenant_brand")
|
||||
if opts.AuthAttestation != "" {
|
||||
form.Set("auth_attestation", opts.AuthAttestation)
|
||||
}
|
||||
// Restore flow: carry the existing app id so the server re-registers it
|
||||
// rather than creating a new app.
|
||||
if opts.RestoreAppID != "" {
|
||||
form.Set("app_id", opts.RestoreAppID)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", endpoint, strings.NewReader(form.Encode()))
|
||||
if err != nil {
|
||||
@@ -186,24 +95,7 @@ func RequestAppRegistration(httpClient *http.Client, brand core.LarkBrand, opts
|
||||
|
||||
userCode := getStr(data, "user_code")
|
||||
verificationUri := getStr(data, "verification_uri")
|
||||
// Prefer the server-provided complete URL (currently /page/launcher); fall
|
||||
// back to building it from verification_uri, then to /page/launcher. The old
|
||||
// hard-coded /page/cli is stale — the server now returns /page/launcher.
|
||||
verificationUriComplete := getStr(data, "verification_uri_complete")
|
||||
if verificationUriComplete == "" {
|
||||
base := verificationUri
|
||||
if base == "" {
|
||||
base = ep.Open + "/page/launcher"
|
||||
}
|
||||
// The server may return verification_uri with its own query (e.g.
|
||||
// client_id when registering against an existing app), so join with
|
||||
// the same ?/& logic as BuildVerificationURL.
|
||||
sep := "?"
|
||||
if strings.Contains(base, "?") {
|
||||
sep = "&"
|
||||
}
|
||||
verificationUriComplete = base + sep + "user_code=" + url.QueryEscape(userCode)
|
||||
}
|
||||
verificationUriComplete := fmt.Sprintf("%s/page/cli?user_code=%s", ep.Open, userCode)
|
||||
|
||||
return &AppRegistrationResponse{
|
||||
DeviceCode: getStr(data, "device_code"),
|
||||
@@ -215,26 +107,6 @@ func RequestAppRegistration(httpClient *http.Client, brand core.LarkBrand, opts
|
||||
}, nil
|
||||
}
|
||||
|
||||
// parseAuthMethods normalizes the poll response `auth_method` field, which the
|
||||
// server returns as a JSON array of strings (e.g. ["private_key_jwt"]) — or, on
|
||||
// some variants, a single space-separated string.
|
||||
func parseAuthMethods(v interface{}) []string {
|
||||
switch t := v.(type) {
|
||||
case []interface{}:
|
||||
out := make([]string, 0, len(t))
|
||||
for _, m := range t {
|
||||
if s, ok := m.(string); ok && s != "" {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
case string:
|
||||
return strings.Fields(t)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// BuildVerificationURL appends CLI tracking parameters to the verification URL.
|
||||
func BuildVerificationURL(baseURL, cliVersion string) string {
|
||||
sep := "&"
|
||||
@@ -315,7 +187,6 @@ func PollAppRegistration(ctx context.Context, httpClient *http.Client, brand cor
|
||||
result := &AppRegistrationResult{
|
||||
ClientID: getStr(data, "client_id"),
|
||||
ClientSecret: getStr(data, "client_secret"),
|
||||
AuthMethods: parseAuthMethods(data["auth_method"]),
|
||||
}
|
||||
if userInfoRaw, ok := data["user_info"].(map[string]interface{}); ok {
|
||||
result.UserInfo = &AppRegUserInfo{
|
||||
|
||||
@@ -4,14 +4,8 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/larksuite/cli/internal/core"
|
||||
"github.com/smartystreets/goconvey/convey"
|
||||
)
|
||||
|
||||
@@ -37,184 +31,3 @@ func Test_BuildVerificationURL(t *testing.T) {
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// captureClient returns an http.Client that records the last request's form body
|
||||
// and replies with the given JSON payload.
|
||||
func captureClient(gotBody *url.Values, respJSON string) *http.Client {
|
||||
return &http.Client{
|
||||
Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
if req.Body != nil {
|
||||
b, _ := io.ReadAll(req.Body)
|
||||
v, _ := url.ParseQuery(string(b))
|
||||
*gotBody = v
|
||||
}
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Header: make(http.Header),
|
||||
Body: io.NopCloser(strings.NewReader(respJSON)),
|
||||
}, nil
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestAppRegistrationInit_ParsesNonceAndMethods(t *testing.T) {
|
||||
var body url.Values
|
||||
hc := captureClient(&body, `{"nonce":"n-123","supported_auth_methods":["client_secret","private_key_jwt"]}`)
|
||||
|
||||
out, err := RequestAppRegistrationInit(hc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Nonce != "n-123" {
|
||||
t.Errorf("nonce = %q, want n-123", out.Nonce)
|
||||
}
|
||||
if len(out.SupportedAuthMethods) != 2 || out.SupportedAuthMethods[1] != "private_key_jwt" {
|
||||
t.Errorf("methods = %v", out.SupportedAuthMethods)
|
||||
}
|
||||
if body.Get("action") != "init" {
|
||||
t.Errorf("action = %q, want init", body.Get("action"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestAppRegistrationInit_ErrorOnMissingNonce(t *testing.T) {
|
||||
var body url.Values
|
||||
hc := captureClient(&body, `{"supported_auth_methods":["client_secret"]}`)
|
||||
if _, err := RequestAppRegistrationInit(hc); err == nil {
|
||||
t.Fatal("expected error when server returns no nonce")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRequestAppRegistrationInit_EmptySupportedAuthMethods covers the older-server
|
||||
// back-compat path: an empty supported_auth_methods array parses to an empty
|
||||
// slice, so the init guard in cmd/config/init_interactive.go
|
||||
// (`len(SupportedAuthMethods) > 0 && !slices.Contains(...)`) stays false and does
|
||||
// NOT reject the requested private_key_jwt. This aligns with
|
||||
// resolveFinalAuthMethod(nil/[], private_key_jwt) == private_key_jwt
|
||||
// (see cmd/config TestResolveFinalAuthMethod).
|
||||
func TestRequestAppRegistrationInit_EmptySupportedAuthMethods(t *testing.T) {
|
||||
var body url.Values
|
||||
hc := captureClient(&body, `{"nonce":"n-1","supported_auth_methods":[]}`)
|
||||
|
||||
out, err := RequestAppRegistrationInit(hc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Nonce != "n-1" {
|
||||
t.Errorf("nonce = %q, want n-1", out.Nonce)
|
||||
}
|
||||
if len(out.SupportedAuthMethods) != 0 {
|
||||
t.Errorf("SupportedAuthMethods = %v, want empty", out.SupportedAuthMethods)
|
||||
}
|
||||
// Reproduce the init guard expression on the real parsed result: an empty
|
||||
// slice must NOT reject private_key_jwt.
|
||||
rejected := len(out.SupportedAuthMethods) > 0 &&
|
||||
!slices.Contains(out.SupportedAuthMethods, core.AuthMethodPrivateKeyJWT)
|
||||
if rejected {
|
||||
t.Error("empty SupportedAuthMethods must allow private_key_jwt (older-server back-compat)")
|
||||
}
|
||||
}
|
||||
|
||||
const beginRespJSON = `{"device_code":"dc","user_code":"uc","verification_uri":"https://example/verify","expires_in":300,"interval":5}`
|
||||
|
||||
func TestRequestAppRegistration_BeginDefaultsToClientSecret(t *testing.T) {
|
||||
var body url.Values
|
||||
hc := captureClient(&body, beginRespJSON)
|
||||
|
||||
if _, err := RequestAppRegistration(hc, core.BrandFeishu, AppRegistrationBeginOptions{}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if body.Get("action") != "begin" {
|
||||
t.Errorf("action = %q", body.Get("action"))
|
||||
}
|
||||
if body.Get("auth_method") != "client_secret" {
|
||||
t.Errorf("auth_method = %q, want client_secret (default)", body.Get("auth_method"))
|
||||
}
|
||||
if body.Has("auth_attestation") {
|
||||
t.Errorf("auth_attestation should be absent for client_secret, got %q", body.Get("auth_attestation"))
|
||||
}
|
||||
// Normal (non-restore) begin must NOT carry app_id.
|
||||
if body.Has("app_id") {
|
||||
t.Errorf("app_id should be absent when RestoreAppID is empty, got %q", body.Get("app_id"))
|
||||
}
|
||||
}
|
||||
|
||||
// TestRequestAppRegistration_BeginRestoreAppID verifies the restore flow sends the
|
||||
// existing app id on begin so the server re-registers that app.
|
||||
func TestRequestAppRegistration_BeginRestoreAppID(t *testing.T) {
|
||||
var body url.Values
|
||||
hc := captureClient(&body, beginRespJSON)
|
||||
|
||||
opts := AppRegistrationBeginOptions{RestoreAppID: "cli_restore_me"}
|
||||
if _, err := RequestAppRegistration(hc, core.BrandFeishu, opts, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if body.Get("action") != "begin" {
|
||||
t.Errorf("action = %q, want begin", body.Get("action"))
|
||||
}
|
||||
if body.Get("app_id") != "cli_restore_me" {
|
||||
t.Errorf("app_id = %q, want cli_restore_me", body.Get("app_id"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestAppRegistration_VerificationURICompleteFallback(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
resp string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "bare verification_uri",
|
||||
resp: `{"device_code":"dc","user_code":"uc","verification_uri":"https://example/verify","expires_in":300,"interval":5}`,
|
||||
want: "https://example/verify?user_code=uc",
|
||||
},
|
||||
{
|
||||
name: "verification_uri with existing query",
|
||||
resp: `{"device_code":"dc","user_code":"uc","verification_uri":"https://example/verify?client_id=cli_x","expires_in":300,"interval":5}`,
|
||||
want: "https://example/verify?client_id=cli_x&user_code=uc",
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var body url.Values
|
||||
hc := captureClient(&body, tc.resp)
|
||||
got, err := RequestAppRegistration(hc, core.BrandFeishu, AppRegistrationBeginOptions{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.VerificationUriComplete != tc.want {
|
||||
t.Errorf("VerificationUriComplete = %q, want %q", got.VerificationUriComplete, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAuthMethods(t *testing.T) {
|
||||
if got := parseAuthMethods([]interface{}{"private_key_jwt", "client_secret"}); len(got) != 2 || got[0] != "private_key_jwt" {
|
||||
t.Errorf("array form = %v", got)
|
||||
}
|
||||
if got := parseAuthMethods("client_secret private_key_jwt"); len(got) != 2 || got[1] != "private_key_jwt" {
|
||||
t.Errorf("string form = %v", got)
|
||||
}
|
||||
if got := parseAuthMethods(nil); got != nil {
|
||||
t.Errorf("nil form = %v, want nil", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestAppRegistration_BeginPrivateKeyJWT(t *testing.T) {
|
||||
var body url.Values
|
||||
hc := captureClient(&body, beginRespJSON)
|
||||
|
||||
opts := AppRegistrationBeginOptions{
|
||||
AuthMethod: core.AuthMethodPrivateKeyJWT,
|
||||
AuthAttestation: "header.claims.sig",
|
||||
}
|
||||
if _, err := RequestAppRegistration(hc, core.BrandFeishu, opts, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if body.Get("auth_method") != "private_key_jwt" {
|
||||
t.Errorf("auth_method = %q, want private_key_jwt", body.Get("auth_method"))
|
||||
}
|
||||
if body.Get("auth_attestation") != "header.claims.sig" {
|
||||
t.Errorf("auth_attestation = %q", body.Get("auth_attestation"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,63 +0,0 @@
|
||||
// Copyright (c) 2026 Lark Technologies Pte. Ltd.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/larksuite/cli/internal/auth/jwt"
|
||||
"github.com/larksuite/cli/internal/core"
|
||||
"github.com/larksuite/cli/internal/keysigner"
|
||||
)
|
||||
|
||||
// ClientAuth describes how to authenticate the OAuth client at the token
|
||||
// endpoint: with a client_secret (default) or a TEE-signed client_assertion
|
||||
// (private_key_jwt).
|
||||
type ClientAuth struct {
|
||||
AppID string
|
||||
AppSecret string
|
||||
AuthMethod string // "" == client_secret; core.AuthMethodPrivateKeyJWT
|
||||
Signer keysigner.Signer
|
||||
KeyLabel string
|
||||
}
|
||||
|
||||
// ClientAuthFromConfig builds a ClientAuth from resolved config, picking up the
|
||||
// active key signer for private_key_jwt apps.
|
||||
func ClientAuthFromConfig(cfg *core.CliConfig) ClientAuth {
|
||||
if cfg == nil {
|
||||
return ClientAuth{}
|
||||
}
|
||||
return ClientAuth{
|
||||
AppID: cfg.AppID,
|
||||
AppSecret: cfg.AppSecret,
|
||||
AuthMethod: cfg.AuthMethod,
|
||||
KeyLabel: cfg.KeyLabel,
|
||||
Signer: keysigner.Active(),
|
||||
}
|
||||
}
|
||||
|
||||
func (c ClientAuth) isPrivateKeyJWT() bool { return c.AuthMethod == core.AuthMethodPrivateKeyJWT }
|
||||
|
||||
// applyClientAssertion adds client_assertion(+type) to a token-endpoint form for
|
||||
// private_key_jwt and returns true. For client_secret it returns false, leaving
|
||||
// the caller to apply its own secret-based authentication. audience is the token
|
||||
// endpoint URL (the assertion's aud claim).
|
||||
func (c ClientAuth) applyClientAssertion(ctx context.Context, form url.Values, audience string) (bool, error) {
|
||||
if !c.isPrivateKeyJWT() {
|
||||
return false, nil
|
||||
}
|
||||
if c.Signer == nil {
|
||||
return false, fmt.Errorf("private_key_jwt requires a key signer, but none is available on this build")
|
||||
}
|
||||
assertion, err := jwt.SignClientAssertion(ctx, c.Signer, keysigner.KeyRef{Label: c.KeyLabel}, c.AppID, audience, time.Now())
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
form.Set("client_assertion_type", jwt.ClientAssertionType)
|
||||
form.Set("client_assertion", assertion)
|
||||
return true, nil
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
// Copyright (c) 2026 Lark Technologies Pte. Ltd.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"github.com/larksuite/cli/internal/auth/jwt"
|
||||
"github.com/larksuite/cli/internal/core"
|
||||
"github.com/larksuite/cli/internal/keysigner"
|
||||
)
|
||||
|
||||
// fakeAuthSigner is a real in-memory ECDSA P-256 signer for client-auth tests.
|
||||
type fakeAuthSigner struct{ key *ecdsa.PrivateKey }
|
||||
|
||||
func newFakeAuthSigner(t *testing.T) *fakeAuthSigner {
|
||||
t.Helper()
|
||||
k, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &fakeAuthSigner{key: k}
|
||||
}
|
||||
|
||||
func (f *fakeAuthSigner) EnsureKey(context.Context, keysigner.KeyRef) (crypto.PublicKey, error) {
|
||||
return f.key.Public(), nil
|
||||
}
|
||||
func (f *fakeAuthSigner) PublicKey(context.Context, keysigner.KeyRef) (crypto.PublicKey, error) {
|
||||
return f.key.Public(), nil
|
||||
}
|
||||
func (f *fakeAuthSigner) Sign(_ context.Context, _ keysigner.KeyRef, in []byte) ([]byte, string, error) {
|
||||
h := sha256.Sum256(in)
|
||||
r, s, err := ecdsa.Sign(rand.Reader, f.key, h[:])
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
sig := make([]byte, 64)
|
||||
r.FillBytes(sig[:32])
|
||||
s.FillBytes(sig[32:])
|
||||
return sig, keysigner.AlgES256, nil
|
||||
}
|
||||
|
||||
func TestClientAuth_applyClientAssertion_ClientSecret(t *testing.T) {
|
||||
ca := ClientAuth{AppID: "cli_a", AppSecret: "sec"} // AuthMethod "" => client_secret
|
||||
form := url.Values{}
|
||||
used, err := ca.applyClientAssertion(context.Background(), form, "https://aud/token")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if used {
|
||||
t.Error("client_secret must not produce a client_assertion")
|
||||
}
|
||||
if form.Has("client_assertion") || form.Has("client_assertion_type") {
|
||||
t.Errorf("form should be untouched, got %v", form)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientAuth_applyClientAssertion_PrivateKeyJWT(t *testing.T) {
|
||||
ca := ClientAuth{
|
||||
AppID: "cli_a",
|
||||
AuthMethod: core.AuthMethodPrivateKeyJWT,
|
||||
Signer: newFakeAuthSigner(t),
|
||||
KeyLabel: "k",
|
||||
}
|
||||
form := url.Values{}
|
||||
used, err := ca.applyClientAssertion(context.Background(), form, "https://accounts.feishu.cn/open-apis/authen/v2/oauth/token")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !used {
|
||||
t.Fatal("expected client_assertion to be applied")
|
||||
}
|
||||
if form.Get("client_assertion_type") != jwt.ClientAssertionType {
|
||||
t.Errorf("client_assertion_type = %q", form.Get("client_assertion_type"))
|
||||
}
|
||||
if form.Get("client_assertion") == "" {
|
||||
t.Error("client_assertion is empty")
|
||||
}
|
||||
if form.Has("client_secret") {
|
||||
t.Error("client_secret must NOT be present for private_key_jwt")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientAuth_applyClientAssertion_NilSigner(t *testing.T) {
|
||||
ca := ClientAuth{AppID: "cli_a", AuthMethod: core.AuthMethodPrivateKeyJWT} // Signer nil
|
||||
if _, err := ca.applyClientAssertion(context.Background(), url.Values{}, "aud"); err == nil {
|
||||
t.Fatal("expected error when private_key_jwt has no signer")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientAuthFromConfig(t *testing.T) {
|
||||
ca := ClientAuthFromConfig(&core.CliConfig{
|
||||
AppID: "cli_x",
|
||||
AppSecret: "s",
|
||||
AuthMethod: core.AuthMethodPrivateKeyJWT,
|
||||
KeyLabel: "label-1",
|
||||
})
|
||||
if ca.AppID != "cli_x" || ca.AppSecret != "s" || ca.AuthMethod != core.AuthMethodPrivateKeyJWT || ca.KeyLabel != "label-1" {
|
||||
t.Errorf("ClientAuth = %+v", ca)
|
||||
}
|
||||
}
|
||||
@@ -62,7 +62,7 @@ func ResolveOAuthEndpoints(brand core.LarkBrand) OAuthEndpoints {
|
||||
}
|
||||
|
||||
// RequestDeviceAuthorization requests a device authorization code.
|
||||
func RequestDeviceAuthorization(ctx context.Context, httpClient *http.Client, ca ClientAuth, brand core.LarkBrand, scope string, errOut io.Writer) (*DeviceAuthResponse, error) {
|
||||
func RequestDeviceAuthorization(httpClient *http.Client, appId, appSecret string, brand core.LarkBrand, scope string, errOut io.Writer) (*DeviceAuthResponse, error) {
|
||||
if errOut == nil {
|
||||
errOut = io.Discard
|
||||
}
|
||||
@@ -77,26 +77,18 @@ func RequestDeviceAuthorization(ctx context.Context, httpClient *http.Client, ca
|
||||
}
|
||||
}
|
||||
|
||||
basicAuth := base64.StdEncoding.EncodeToString([]byte(appId + ":" + appSecret))
|
||||
|
||||
form := url.Values{}
|
||||
form.Set("client_id", ca.AppID)
|
||||
form.Set("client_id", appId)
|
||||
form.Set("scope", scope)
|
||||
|
||||
// private_key_jwt authenticates the client with a signed assertion in the
|
||||
// body; client_secret uses HTTP Basic.
|
||||
usedAssertion, err := ca.applyClientAssertion(ctx, form, core.OpenAPIAudience(brand))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", endpoints.DeviceAuthorization, strings.NewReader(form.Encode()))
|
||||
req, err := http.NewRequest("POST", endpoints.DeviceAuthorization, strings.NewReader(form.Encode()))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
if !usedAssertion {
|
||||
basicAuth := base64.StdEncoding.EncodeToString([]byte(ca.AppID + ":" + ca.AppSecret))
|
||||
req.Header.Set("Authorization", "Basic "+basicAuth)
|
||||
}
|
||||
req.Header.Set("Authorization", "Basic "+basicAuth)
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
@@ -147,7 +139,7 @@ func RequestDeviceAuthorization(ctx context.Context, httpClient *http.Client, ca
|
||||
}
|
||||
|
||||
// PollDeviceToken polls the token endpoint until authorization completes or times out.
|
||||
func PollDeviceToken(ctx context.Context, httpClient *http.Client, ca ClientAuth, brand core.LarkBrand, deviceCode string, interval, expiresIn int, errOut io.Writer) *DeviceFlowResult {
|
||||
func PollDeviceToken(ctx context.Context, httpClient *http.Client, appId, appSecret string, brand core.LarkBrand, deviceCode string, interval, expiresIn int, errOut io.Writer) *DeviceFlowResult {
|
||||
if errOut == nil {
|
||||
errOut = io.Discard
|
||||
}
|
||||
@@ -179,16 +171,10 @@ func PollDeviceToken(ctx context.Context, httpClient *http.Client, ca ClientAuth
|
||||
form := url.Values{}
|
||||
form.Set("grant_type", "urn:ietf:params:oauth:grant-type:device_code")
|
||||
form.Set("device_code", deviceCode)
|
||||
form.Set("client_id", ca.AppID)
|
||||
usedAssertion, caErr := ca.applyClientAssertion(ctx, form, core.OpenAPIAudience(brand))
|
||||
if caErr != nil {
|
||||
return &DeviceFlowResult{OK: false, Error: "invalid_client", Message: caErr.Error()}
|
||||
}
|
||||
if !usedAssertion {
|
||||
form.Set("client_secret", ca.AppSecret)
|
||||
}
|
||||
form.Set("client_id", appId)
|
||||
form.Set("client_secret", appSecret)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", endpoints.Token, strings.NewReader(form.Encode()))
|
||||
req, err := http.NewRequest("POST", endpoints.Token, strings.NewReader(form.Encode()))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -7,10 +7,8 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
@@ -85,7 +83,7 @@ func TestRequestDeviceAuthorization_LogsResponse(t *testing.T) {
|
||||
})
|
||||
t.Cleanup(restore)
|
||||
|
||||
_, err := RequestDeviceAuthorization(context.Background(), httpmock.NewClient(reg), ClientAuth{AppID: "cli_a", AppSecret: "secret_b"}, core.BrandFeishu, "", nil)
|
||||
_, err := RequestDeviceAuthorization(httpmock.NewClient(reg), "cli_a", "secret_b", core.BrandFeishu, "", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("RequestDeviceAuthorization() error: %v", err)
|
||||
}
|
||||
@@ -108,66 +106,6 @@ func TestRequestDeviceAuthorization_LogsResponse(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// captureRT records the last request + body and returns a canned device-auth response.
|
||||
func captureDeviceAuthClient(gotReq **http.Request, gotBody *string, respJSON string) *http.Client {
|
||||
return &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
*gotReq = req
|
||||
if req.Body != nil {
|
||||
b, _ := io.ReadAll(req.Body)
|
||||
*gotBody = string(b)
|
||||
}
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Header: make(http.Header),
|
||||
Body: io.NopCloser(strings.NewReader(respJSON)),
|
||||
}, nil
|
||||
})}
|
||||
}
|
||||
|
||||
const deviceAuthRespJSON = `{"device_code":"dc","user_code":"uc","verification_uri":"https://example/verify","expires_in":300,"interval":5}`
|
||||
|
||||
func TestRequestDeviceAuthorization_PrivateKeyJWT_UsesAssertionNotBasic(t *testing.T) {
|
||||
var req *http.Request
|
||||
var body string
|
||||
client := captureDeviceAuthClient(&req, &body, deviceAuthRespJSON)
|
||||
|
||||
ca := ClientAuth{AppID: "cli_a", AuthMethod: core.AuthMethodPrivateKeyJWT, Signer: newFakeAuthSigner(t), KeyLabel: "k"}
|
||||
if _, err := RequestDeviceAuthorization(context.Background(), client, ca, core.BrandFeishu, "im:message:send", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if req.Header.Get("Authorization") != "" {
|
||||
t.Errorf("private_key_jwt must NOT send Basic auth, got %q", req.Header.Get("Authorization"))
|
||||
}
|
||||
form, _ := url.ParseQuery(body)
|
||||
if form.Get("client_assertion") == "" {
|
||||
t.Error("missing client_assertion")
|
||||
}
|
||||
if form.Get("client_assertion_type") != "urn:ietf:params:oauth:client-assertion-type:jwt-bearer" {
|
||||
t.Errorf("client_assertion_type = %q", form.Get("client_assertion_type"))
|
||||
}
|
||||
if form.Has("client_secret") {
|
||||
t.Error("client_secret must not be present for private_key_jwt")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestDeviceAuthorization_ClientSecret_UsesBasic(t *testing.T) {
|
||||
var req *http.Request
|
||||
var body string
|
||||
client := captureDeviceAuthClient(&req, &body, deviceAuthRespJSON)
|
||||
|
||||
ca := ClientAuth{AppID: "cli_a", AppSecret: "sec"} // client_secret
|
||||
if _, err := RequestDeviceAuthorization(context.Background(), client, ca, core.BrandFeishu, "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.HasPrefix(req.Header.Get("Authorization"), "Basic ") {
|
||||
t.Errorf("client_secret should use Basic auth, got %q", req.Header.Get("Authorization"))
|
||||
}
|
||||
form, _ := url.ParseQuery(body)
|
||||
if form.Has("client_assertion") {
|
||||
t.Error("client_secret must not send a client_assertion")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFormatAuthCmdline_TruncatesExtraArgs verifies that long command lines are truncated.
|
||||
func TestFormatAuthCmdline_TruncatesExtraArgs(t *testing.T) {
|
||||
got := keychain.FormatAuthCmdline([]string{
|
||||
@@ -267,7 +205,7 @@ func TestPollDeviceToken_DefaultsZeroIntervalToFiveSeconds(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||
t.Cleanup(cancel)
|
||||
|
||||
result := PollDeviceToken(ctx, client, ClientAuth{AppID: "cli_a", AppSecret: "secret_b"}, core.BrandFeishu, "device-code", 0, 10, nil)
|
||||
result := PollDeviceToken(ctx, client, "cli_a", "secret_b", core.BrandFeishu, "device-code", 0, 10, nil)
|
||||
if result == nil {
|
||||
t.Fatal("PollDeviceToken() returned nil result")
|
||||
}
|
||||
|
||||
@@ -1,153 +0,0 @@
|
||||
// Copyright (c) 2026 Lark Technologies Pte. Ltd.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
// Package jwt builds compact JWS tokens signed by a keysigner.Signer.
|
||||
//
|
||||
// It deliberately depends only on the standard library plus the existing
|
||||
// google/uuid dependency — no third-party JWT library is introduced, keeping
|
||||
// go.mod free of new dependencies. The actual signing (and, for ECDSA, the
|
||||
// ASN.1->r||s conversion) is delegated to the Signer implementation.
|
||||
package jwt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/larksuite/cli/internal/keysigner"
|
||||
)
|
||||
|
||||
func b64(b []byte) string { return base64.RawURLEncoding.EncodeToString(b) }
|
||||
|
||||
// buildSignedJWT builds a compact JWS:
|
||||
//
|
||||
// base64url(header).base64url(claims).base64url(signature)
|
||||
//
|
||||
// alg is written into the header (it is part of the signed input) and verified
|
||||
// against the alg the signer reports, guarding against a header/key mismatch.
|
||||
// typ defaults to "JWT": the server's client_assertion generalizedValidation
|
||||
// REQUIRES `typ == "JWT"` (rejects otherwise with "malformed client assertion
|
||||
// jwt"), even though the spec examples (§8.1/§8.2) show only alg.
|
||||
func buildSignedJWT(ctx context.Context, signer keysigner.Signer, ref keysigner.KeyRef, alg string, header, claims map[string]any) (string, error) {
|
||||
if signer == nil {
|
||||
return "", fmt.Errorf("jwt: no signer available (private_key_jwt unsupported on this build)")
|
||||
}
|
||||
if header == nil {
|
||||
header = map[string]any{}
|
||||
}
|
||||
header["alg"] = alg
|
||||
if _, ok := header["typ"]; !ok {
|
||||
header["typ"] = "JWT"
|
||||
}
|
||||
|
||||
hb, err := json.Marshal(header)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("jwt: marshal header: %w", err)
|
||||
}
|
||||
cb, err := json.Marshal(claims)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("jwt: marshal claims: %w", err)
|
||||
}
|
||||
|
||||
signingInput := b64(hb) + "." + b64(cb)
|
||||
sig, gotAlg, err := signer.Sign(ctx, ref, []byte(signingInput))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("jwt: sign: %w", err)
|
||||
}
|
||||
if gotAlg != alg {
|
||||
return "", fmt.Errorf("jwt: signer alg %q does not match header alg %q", gotAlg, alg)
|
||||
}
|
||||
return signingInput + "." + b64(sig), nil
|
||||
}
|
||||
|
||||
// newJTI returns a random unique token identifier.
|
||||
func newJTI() string { return uuid.NewString() }
|
||||
|
||||
// attestationTTL bounds the attestation JWT's lifetime. The init nonce (60s,
|
||||
// single-use) is the real anti-replay constraint; this is a modest margin for
|
||||
// clock skew on top of the immediate init→sign→begin round-trip.
|
||||
const attestationTTL = 2 * time.Minute
|
||||
|
||||
// attestationClaims builds the registration attestation claim set per the App
|
||||
// Registration JWT spec: jti, iat, exp (all required) and the init-issued nonce.
|
||||
func attestationClaims(nonce string, now time.Time) map[string]any {
|
||||
return map[string]any{
|
||||
"jti": newJTI(),
|
||||
"iat": now.Unix(),
|
||||
"exp": now.Add(attestationTTL).Unix(),
|
||||
"nonce": nonce,
|
||||
}
|
||||
}
|
||||
|
||||
// clientAssertionClaims builds an RFC 7523 client_assertion claim set used to
|
||||
// mint tokens in place of client_secret. aud is the brand's token endpoint URL.
|
||||
func clientAssertionClaims(clientID, aud string, now time.Time, ttl time.Duration) map[string]any {
|
||||
return map[string]any{
|
||||
"iss": clientID,
|
||||
"sub": clientID,
|
||||
"aud": aud,
|
||||
"iat": now.Unix(),
|
||||
"exp": now.Add(ttl).Unix(),
|
||||
"jti": newJTI(),
|
||||
}
|
||||
}
|
||||
|
||||
// ClientAssertionType is the RFC 7523 client_assertion_type value used for JWT
|
||||
// bearer client authentication at the token endpoint.
|
||||
const ClientAssertionType = "urn:ietf:params:oauth:client-assertion-type:jwt-bearer"
|
||||
|
||||
// defaultAssertionTTL bounds a client_assertion's lifetime.
|
||||
const defaultAssertionTTL = 5 * time.Minute
|
||||
|
||||
// SignAttestation signs the registration attestation JWT. The public key is
|
||||
// embedded in the JWS "jwk" header so the registration backend can bind it to
|
||||
// the app during action=begin; the claims carry the server nonce as a
|
||||
// proof-of-possession challenge.
|
||||
func SignAttestation(ctx context.Context, signer keysigner.Signer, ref keysigner.KeyRef, nonce string, now time.Time) (string, error) {
|
||||
if signer == nil {
|
||||
return "", fmt.Errorf("jwt: no signer available (private_key_jwt unsupported on this build)")
|
||||
}
|
||||
pub, err := signer.EnsureKey(ctx, ref)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("jwt: ensure key: %w", err)
|
||||
}
|
||||
alg, err := keysigner.AlgForKey(pub)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
jwk, err := keysigner.PublicKeyJWK(pub)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return buildSignedJWT(ctx, signer, ref, alg, map[string]any{"jwk": jwk}, attestationClaims(nonce, now))
|
||||
}
|
||||
|
||||
// SignClientAssertion mints a short-lived RFC 7523 client_assertion: it reads the
|
||||
// registered key (it must already exist — bound at registration; a missing key is
|
||||
// an error, not a reason to create a new unbound one), derives the JWS alg from
|
||||
// the public key, and signs an assertion whose audience is the brand's Open API
|
||||
// host. The server, holding the public key bound at registration, verifies it in
|
||||
// place of client_secret. The assertion header carries only alg (no jwk/kid);
|
||||
// the server locates the key via iss/sub = client_id.
|
||||
//
|
||||
// This is the model-independent glue: the assertion JWT is identical whether the
|
||||
// server augments an existing grant (device_code/refresh_token) with client
|
||||
// authentication or uses a dedicated jwt-bearer grant — only where the caller
|
||||
// attaches it differs.
|
||||
func SignClientAssertion(ctx context.Context, signer keysigner.Signer, ref keysigner.KeyRef, clientID, audience string, now time.Time) (string, error) {
|
||||
if signer == nil {
|
||||
return "", fmt.Errorf("jwt: no signer available (private_key_jwt unsupported on this build)")
|
||||
}
|
||||
pub, err := signer.PublicKey(ctx, ref)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("jwt: public key: %w", err)
|
||||
}
|
||||
alg, err := keysigner.AlgForKey(pub)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return buildSignedJWT(ctx, signer, ref, alg, map[string]any{}, clientAssertionClaims(clientID, audience, now, defaultAssertionTTL))
|
||||
}
|
||||
@@ -1,254 +0,0 @@
|
||||
// Copyright (c) 2026 Lark Technologies Pte. Ltd.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package jwt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"math/big"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/larksuite/cli/internal/keysigner"
|
||||
)
|
||||
|
||||
// fakeSigner is a real in-memory ECDSA P-256 signer, so tests exercise the full
|
||||
// JWS path and the produced token is actually cryptographically verifiable.
|
||||
type fakeSigner struct{ key *ecdsa.PrivateKey }
|
||||
|
||||
func newFakeSigner(t *testing.T) *fakeSigner {
|
||||
t.Helper()
|
||||
k, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &fakeSigner{key: k}
|
||||
}
|
||||
|
||||
func (f *fakeSigner) EnsureKey(context.Context, keysigner.KeyRef) (crypto.PublicKey, error) {
|
||||
return f.key.Public(), nil
|
||||
}
|
||||
func (f *fakeSigner) PublicKey(context.Context, keysigner.KeyRef) (crypto.PublicKey, error) {
|
||||
return f.key.Public(), nil
|
||||
}
|
||||
func (f *fakeSigner) Sign(_ context.Context, _ keysigner.KeyRef, in []byte) ([]byte, string, error) {
|
||||
h := sha256.Sum256(in)
|
||||
r, s, err := ecdsa.Sign(rand.Reader, f.key, h[:])
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
// JOSE ES256: fixed-width big-endian r||s (32 bytes each for P-256).
|
||||
sig := make([]byte, 64)
|
||||
r.FillBytes(sig[:32])
|
||||
s.FillBytes(sig[32:])
|
||||
return sig, keysigner.AlgES256, nil
|
||||
}
|
||||
|
||||
func TestBuildSignedJWT_VerifiableES256(t *testing.T) {
|
||||
f := newFakeSigner(t)
|
||||
now := time.Unix(1700000000, 0)
|
||||
|
||||
tok, err := buildSignedJWT(context.Background(), f, keysigner.KeyRef{Label: "x"}, keysigner.AlgES256,
|
||||
map[string]any{}, clientAssertionClaims("cli_app", "https://accounts.example/token", now, 5*time.Minute))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
parts := strings.Split(tok, ".")
|
||||
if len(parts) != 3 {
|
||||
t.Fatalf("want 3 JWS parts, got %d", len(parts))
|
||||
}
|
||||
|
||||
hb, err := base64.RawURLEncoding.DecodeString(parts[0])
|
||||
if err != nil {
|
||||
t.Fatalf("header not base64url: %v", err)
|
||||
}
|
||||
var hdr map[string]any
|
||||
if err := json.Unmarshal(hb, &hdr); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if hdr["alg"] != "ES256" || hdr["typ"] != "JWT" {
|
||||
t.Errorf("header = %v, want alg=ES256 typ=JWT (server generalizedValidation requires typ)", hdr)
|
||||
}
|
||||
|
||||
cb, _ := base64.RawURLEncoding.DecodeString(parts[1])
|
||||
var claims map[string]any
|
||||
if err := json.Unmarshal(cb, &claims); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if claims["iss"] != "cli_app" || claims["sub"] != "cli_app" || claims["aud"] != "https://accounts.example/token" {
|
||||
t.Errorf("claims = %v", claims)
|
||||
}
|
||||
|
||||
// Cryptographically verify the signature against the signing input.
|
||||
sig, err := base64.RawURLEncoding.DecodeString(parts[2])
|
||||
if err != nil {
|
||||
t.Fatalf("sig not base64url: %v", err)
|
||||
}
|
||||
if len(sig) != 64 {
|
||||
t.Fatalf("ES256 sig len = %d, want 64", len(sig))
|
||||
}
|
||||
r := new(big.Int).SetBytes(sig[:32])
|
||||
s := new(big.Int).SetBytes(sig[32:])
|
||||
h := sha256.Sum256([]byte(parts[0] + "." + parts[1]))
|
||||
if !ecdsa.Verify(f.key.Public().(*ecdsa.PublicKey), h[:], r, s) {
|
||||
t.Error("signature did not verify")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSignedJWT_NilSigner(t *testing.T) {
|
||||
if _, err := buildSignedJWT(context.Background(), nil, keysigner.KeyRef{}, "ES256", nil, nil); err == nil {
|
||||
t.Fatal("expected error for nil signer")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSignedJWT_AlgMismatch(t *testing.T) {
|
||||
f := newFakeSigner(t) // always reports ES256
|
||||
if _, err := buildSignedJWT(context.Background(), f, keysigner.KeyRef{}, keysigner.AlgRS256, nil, nil); err == nil {
|
||||
t.Fatal("expected error when header alg != signer alg")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSignedJWT_MarshalErrors(t *testing.T) {
|
||||
f := newFakeSigner(t)
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := buildSignedJWT(ctx, f, keysigner.KeyRef{}, keysigner.AlgES256,
|
||||
map[string]any{"bad": func() {}}, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "jwt: marshal header") {
|
||||
t.Fatalf("header marshal error = %v, want prefix %q", err, "jwt: marshal header")
|
||||
}
|
||||
|
||||
_, err = buildSignedJWT(ctx, f, keysigner.KeyRef{}, keysigner.AlgES256,
|
||||
nil, map[string]any{"bad": make(chan int)})
|
||||
if err == nil || !strings.Contains(err.Error(), "jwt: marshal claims") {
|
||||
t.Fatalf("claims marshal error = %v, want prefix %q", err, "jwt: marshal claims")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignClientAssertion(t *testing.T) {
|
||||
f := newFakeSigner(t)
|
||||
now := time.Unix(1700000000, 0)
|
||||
const aud = "https://accounts.feishu.cn/open-apis/authen/v2/oauth/token"
|
||||
|
||||
tok, err := SignClientAssertion(context.Background(), f, keysigner.KeyRef{Label: "k"}, "cli_app", aud, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
parts := strings.Split(tok, ".")
|
||||
if len(parts) != 3 {
|
||||
t.Fatalf("want 3 parts, got %d", len(parts))
|
||||
}
|
||||
cb, _ := base64.RawURLEncoding.DecodeString(parts[1])
|
||||
var claims map[string]any
|
||||
if err := json.Unmarshal(cb, &claims); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if claims["iss"] != "cli_app" || claims["aud"] != aud {
|
||||
t.Errorf("claims = %v", claims)
|
||||
}
|
||||
|
||||
// Signature must verify against the key's public half.
|
||||
sig, _ := base64.RawURLEncoding.DecodeString(parts[2])
|
||||
r := new(big.Int).SetBytes(sig[:32])
|
||||
s := new(big.Int).SetBytes(sig[32:])
|
||||
h := sha256.Sum256([]byte(parts[0] + "." + parts[1]))
|
||||
if !ecdsa.Verify(f.key.Public().(*ecdsa.PublicKey), h[:], r, s) {
|
||||
t.Error("client_assertion signature did not verify")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignClientAssertion_NilSigner(t *testing.T) {
|
||||
if _, err := SignClientAssertion(context.Background(), nil, keysigner.KeyRef{}, "cli_app", "aud", time.Unix(0, 0)); err == nil {
|
||||
t.Fatal("expected error for nil signer")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignAttestation(t *testing.T) {
|
||||
f := newFakeSigner(t)
|
||||
now := time.Unix(1700000000, 0)
|
||||
|
||||
tok, err := SignAttestation(context.Background(), f, keysigner.KeyRef{Label: "k"}, "nonce-abc", now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
parts := strings.Split(tok, ".")
|
||||
if len(parts) != 3 {
|
||||
t.Fatalf("want 3 parts, got %d", len(parts))
|
||||
}
|
||||
|
||||
hb, _ := base64.RawURLEncoding.DecodeString(parts[0])
|
||||
var hdr map[string]any
|
||||
if err := json.Unmarshal(hb, &hdr); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jwk, ok := hdr["jwk"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("attestation header missing jwk: %v", hdr)
|
||||
}
|
||||
if jwk["kty"] != "EC" || jwk["crv"] != "P-256" || jwk["use"] != "sig" {
|
||||
t.Errorf("jwk = %v", jwk)
|
||||
}
|
||||
|
||||
cb, _ := base64.RawURLEncoding.DecodeString(parts[1])
|
||||
var claims map[string]any
|
||||
if err := json.Unmarshal(cb, &claims); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if claims["nonce"] != "nonce-abc" {
|
||||
t.Errorf("nonce = %v", claims["nonce"])
|
||||
}
|
||||
// jti, iat, exp are all required by the attestation spec.
|
||||
iat, iatOK := claims["iat"].(float64)
|
||||
exp, expOK := claims["exp"].(float64)
|
||||
if !iatOK || !expOK || exp <= iat {
|
||||
t.Errorf("claims iat/exp invalid: iat=%v exp=%v", claims["iat"], claims["exp"])
|
||||
}
|
||||
if jti, _ := claims["jti"].(string); jti == "" {
|
||||
t.Error("claims jti empty")
|
||||
}
|
||||
|
||||
// Signature verifies against the embedded key.
|
||||
sig, _ := base64.RawURLEncoding.DecodeString(parts[2])
|
||||
r := new(big.Int).SetBytes(sig[:32])
|
||||
s := new(big.Int).SetBytes(sig[32:])
|
||||
h := sha256.Sum256([]byte(parts[0] + "." + parts[1]))
|
||||
if !ecdsa.Verify(f.key.Public().(*ecdsa.PublicKey), h[:], r, s) {
|
||||
t.Error("attestation signature did not verify")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignAttestation_NilSigner(t *testing.T) {
|
||||
if _, err := SignAttestation(context.Background(), nil, keysigner.KeyRef{}, "n", time.Unix(0, 0)); err == nil {
|
||||
t.Fatal("expected error for nil signer")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClaimFactories(t *testing.T) {
|
||||
now := time.Unix(1700000000, 0)
|
||||
|
||||
a := attestationClaims("nonce-xyz", now)
|
||||
if a["nonce"] != "nonce-xyz" || a["iat"] != now.Unix() {
|
||||
t.Errorf("attestation claims = %v", a)
|
||||
}
|
||||
if a["exp"] != now.Add(attestationTTL).Unix() {
|
||||
t.Errorf("attestation exp = %v, want %v", a["exp"], now.Add(attestationTTL).Unix())
|
||||
}
|
||||
if jti, _ := a["jti"].(string); jti == "" {
|
||||
t.Error("attestation jti empty")
|
||||
}
|
||||
|
||||
c := clientAssertionClaims("cli_app", "aud", now, time.Minute)
|
||||
if c["exp"].(int64) != now.Add(time.Minute).Unix() {
|
||||
t.Errorf("client_assertion exp = %v", c["exp"])
|
||||
}
|
||||
}
|
||||
@@ -21,7 +21,6 @@ import (
|
||||
"github.com/larksuite/cli/errs"
|
||||
"github.com/larksuite/cli/internal/core"
|
||||
"github.com/larksuite/cli/internal/errclass"
|
||||
"github.com/larksuite/cli/internal/keysigner"
|
||||
"github.com/larksuite/cli/internal/vfs"
|
||||
)
|
||||
|
||||
@@ -38,10 +37,7 @@ type UATCallOptions struct {
|
||||
AppId string
|
||||
AppSecret string
|
||||
Domain core.LarkBrand
|
||||
AuthMethod string // "" == client_secret; core.AuthMethodPrivateKeyJWT
|
||||
KeyLabel string // TEE key handle for private_key_jwt
|
||||
Signer keysigner.Signer // active signer for private_key_jwt
|
||||
ErrOut io.Writer // diagnostic/status output (caller injects f.IOStreams.ErrOut)
|
||||
ErrOut io.Writer // diagnostic/status output (caller injects f.IOStreams.ErrOut)
|
||||
}
|
||||
|
||||
// UATStatus represents the status of a user access token.
|
||||
@@ -65,9 +61,6 @@ func NewUATCallOptions(cfg *core.CliConfig, errOut io.Writer) UATCallOptions {
|
||||
AppId: cfg.AppID,
|
||||
AppSecret: cfg.AppSecret,
|
||||
Domain: cfg.Brand,
|
||||
AuthMethod: cfg.AuthMethod,
|
||||
KeyLabel: cfg.KeyLabel,
|
||||
Signer: keysigner.Active(),
|
||||
ErrOut: errOut,
|
||||
}
|
||||
}
|
||||
@@ -200,14 +193,7 @@ func doRefreshToken(httpClient *http.Client, opts UATCallOptions, stored *Stored
|
||||
form.Set("grant_type", "refresh_token")
|
||||
form.Set("refresh_token", stored.RefreshToken)
|
||||
form.Set("client_id", opts.AppId)
|
||||
ca := ClientAuth{AppID: opts.AppId, AppSecret: opts.AppSecret, AuthMethod: opts.AuthMethod, Signer: opts.Signer, KeyLabel: opts.KeyLabel}
|
||||
usedAssertion, caErr := ca.applyClientAssertion(context.Background(), form, core.OpenAPIAudience(opts.Domain))
|
||||
if caErr != nil {
|
||||
return nil, caErr
|
||||
}
|
||||
if !usedAssertion {
|
||||
form.Set("client_secret", opts.AppSecret)
|
||||
}
|
||||
form.Set("client_secret", opts.AppSecret)
|
||||
|
||||
req, err := http.NewRequest("POST", endpoints.Token, strings.NewReader(form.Encode()))
|
||||
if err != nil {
|
||||
|
||||
@@ -38,23 +38,3 @@ func TestNewUATCallOptions(t *testing.T) {
|
||||
t.Error("ErrOut not set correctly")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewUATCallOptions_PrivateKeyJWT verifies the auth-method fields propagate
|
||||
// so the refresh path can mint a client_assertion instead of sending a secret.
|
||||
func TestNewUATCallOptions_PrivateKeyJWT(t *testing.T) {
|
||||
cfg := &core.CliConfig{
|
||||
AppID: "cli_pk",
|
||||
Brand: core.BrandFeishu,
|
||||
UserOpenId: "ou_test",
|
||||
AuthMethod: core.AuthMethodPrivateKeyJWT,
|
||||
KeyLabel: "agent-key",
|
||||
}
|
||||
opts := NewUATCallOptions(cfg, &bytes.Buffer{})
|
||||
|
||||
if opts.AuthMethod != core.AuthMethodPrivateKeyJWT {
|
||||
t.Errorf("AuthMethod = %q, want private_key_jwt", opts.AuthMethod)
|
||||
}
|
||||
if opts.KeyLabel != "agent-key" {
|
||||
t.Errorf("KeyLabel = %q, want agent-key", opts.KeyLabel)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -42,16 +42,6 @@ func NewIOStreams(in io.Reader, out, errOut io.Writer) *IOStreams {
|
||||
return &IOStreams{In: in, Out: out, ErrOut: errOut, IsTerminal: isTerminal, StderrIsTerminal: stderrIsTerminal}
|
||||
}
|
||||
|
||||
// StdoutIsTerminal reports whether Out is an interactive terminal. Unlike
|
||||
// IsTerminal — which reflects stdin and drives prompt decisions — this is the
|
||||
// correct check for OUTPUT formatting: `cmd | jq` must still emit machine output
|
||||
// from an interactive shell (stdin is a TTY there, but stdout is the pipe).
|
||||
// Buffers (tests) and redirects are not *os.File terminals, so they yield false.
|
||||
func (s *IOStreams) StdoutIsTerminal() bool {
|
||||
f, ok := s.Out.(*os.File)
|
||||
return ok && term.IsTerminal(int(f.Fd()))
|
||||
}
|
||||
|
||||
// SystemIO creates an IOStreams wired to the process's standard file descriptors.
|
||||
//
|
||||
//nolint:forbidigo // entry point for real stdio
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
// Copyright (c) 2026 Lark Technologies Pte. Ltd.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package cmdutil
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestStdoutIsTerminal(t *testing.T) {
|
||||
// Buffer-backed output (tests, captured output) is never a terminal.
|
||||
if (&IOStreams{Out: &bytes.Buffer{}}).StdoutIsTerminal() {
|
||||
t.Error("bytes.Buffer Out should not be a terminal")
|
||||
}
|
||||
// An os.Pipe write end is an *os.File but not a terminal — mirrors `cmd | jq`,
|
||||
// the case the stdin-based IsTerminal would get wrong.
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer r.Close()
|
||||
defer w.Close()
|
||||
if (&IOStreams{Out: w}).StdoutIsTerminal() {
|
||||
t.Error("os.Pipe Out should not be a terminal")
|
||||
}
|
||||
}
|
||||
@@ -36,13 +36,6 @@ type AppUser struct {
|
||||
UserName string `json:"userName"`
|
||||
}
|
||||
|
||||
// Auth methods for app credentials. An empty AppConfig.AuthMethod means the
|
||||
// default, client_secret.
|
||||
const (
|
||||
AuthMethodClientSecret = "client_secret" // app_id + app_secret
|
||||
AuthMethodPrivateKeyJWT = "private_key_jwt" // TEE-signed client_assertion; no app secret
|
||||
)
|
||||
|
||||
// AppConfig is a per-app configuration entry (stored format — secrets may be unresolved).
|
||||
type AppConfig struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
@@ -53,15 +46,6 @@ type AppConfig struct {
|
||||
DefaultAs Identity `json:"defaultAs,omitempty"` // AsUser | AsBot | AsAuto
|
||||
StrictMode *StrictMode `json:"strictMode,omitempty"`
|
||||
Users []AppUser `json:"users"`
|
||||
|
||||
// AuthMethod selects how tokens are minted. Empty == AuthMethodClientSecret
|
||||
// (back-compat). AuthMethodPrivateKeyJWT uses a TEE-held key (see KeyRef) to
|
||||
// sign client_assertion JWTs instead of sending an app secret.
|
||||
AuthMethod string `json:"authMethod,omitempty"`
|
||||
// KeyRef references the non-exportable signing key for private_key_jwt.
|
||||
// Source is "tee" and ID is the backend key label; the actual key never
|
||||
// leaves the secure backend, so this is a handle, not secret material.
|
||||
KeyRef *SecretRef `json:"keyRef,omitempty"`
|
||||
}
|
||||
|
||||
// ProfileName returns the display name for this app config.
|
||||
@@ -177,9 +161,7 @@ type CliConfig struct {
|
||||
UserOpenId string
|
||||
UserName string
|
||||
Lang i18n.Lang
|
||||
SupportedIdentities uint8 `json:"-"` // bitflag: 1=user, 2=bot; set by credential provider
|
||||
AuthMethod string // "" == client_secret; AuthMethodPrivateKeyJWT
|
||||
KeyLabel string // resolved TEE key handle for private_key_jwt
|
||||
SupportedIdentities uint8 `json:"-"` // bitflag: 1=user, 2=bot; set by credential provider
|
||||
}
|
||||
|
||||
// identityBotBit is the bit flag for bot identity in SupportedIdentities.
|
||||
@@ -265,58 +247,31 @@ func ResolveConfigFromMulti(raw *MultiAppConfig, kc keychain.KeychainAccess, pro
|
||||
WithHint("available profiles: %s", formatProfileNames(raw.ProfileNames()))
|
||||
}
|
||||
|
||||
// Validate the auth method first so a malformed profile fails here rather
|
||||
// than silently degrading to client_secret (unknown method) or failing later
|
||||
// at token-signing. Empty stays empty — downstream treats it as client_secret
|
||||
// (back-compat).
|
||||
switch app.AuthMethod {
|
||||
case "", AuthMethodClientSecret, AuthMethodPrivateKeyJWT:
|
||||
default:
|
||||
return nil, errs.NewConfigError(errs.SubtypeInvalidConfig, "unknown authMethod %q", app.AuthMethod).
|
||||
WithHint("supported: %s, %s (empty defaults to %s)", AuthMethodClientSecret, AuthMethodPrivateKeyJWT, AuthMethodClientSecret)
|
||||
if err := ValidateSecretKeyMatch(app.AppId, app.AppSecret); err != nil {
|
||||
return nil, errs.NewConfigError(errs.SubtypeNotConfigured, "appId and appSecret keychain key are out of sync").
|
||||
WithHint("%s", err.Error()).
|
||||
WithCause(err)
|
||||
}
|
||||
|
||||
// private_key_jwt carries no secret: validate the key handle and skip secret
|
||||
// resolution entirely, so a stale/broken AppSecret ref never produces a
|
||||
// confusing secret-resolution error for an otherwise-valid pkjwt profile.
|
||||
var secret string
|
||||
if app.AuthMethod == AuthMethodPrivateKeyJWT {
|
||||
if app.KeyRef == nil || app.KeyRef.Source != "tee" || app.KeyRef.ID == "" {
|
||||
return nil, errs.NewConfigError(errs.SubtypeInvalidConfig, "private_key_jwt requires a key handle (keyRef) but none is configured").
|
||||
WithHint("re-run: lark-cli config init --new --auth-method private_key_jwt")
|
||||
secret, err := ResolveSecretInput(app.AppSecret, kc)
|
||||
if err != nil {
|
||||
if errs.IsTyped(err) {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
if err := ValidateSecretKeyMatch(app.AppId, app.AppSecret); err != nil {
|
||||
return nil, errs.NewConfigError(errs.SubtypeNotConfigured, "appId and appSecret keychain key are out of sync").
|
||||
WithHint("%s", err.Error()).
|
||||
WithCause(err)
|
||||
}
|
||||
var resolveErr error
|
||||
secret, resolveErr = ResolveSecretInput(app.AppSecret, kc)
|
||||
if resolveErr != nil {
|
||||
if errs.IsTyped(resolveErr) {
|
||||
return nil, resolveErr
|
||||
}
|
||||
subtype := errs.SubtypeNotConfigured
|
||||
if isMalformedConfigError(resolveErr) {
|
||||
subtype = errs.SubtypeInvalidConfig
|
||||
}
|
||||
return nil, errs.NewConfigError(subtype, "%s", resolveErr.Error()).WithCause(resolveErr)
|
||||
subtype := errs.SubtypeNotConfigured
|
||||
if isMalformedConfigError(err) {
|
||||
subtype = errs.SubtypeInvalidConfig
|
||||
}
|
||||
return nil, errs.NewConfigError(subtype, "%s", err.Error()).WithCause(err)
|
||||
}
|
||||
|
||||
cfg := &CliConfig{
|
||||
ProfileName: app.ProfileName(),
|
||||
AppID: app.AppId,
|
||||
AppSecret: secret,
|
||||
Brand: app.Brand,
|
||||
Lang: app.Lang,
|
||||
AuthMethod: app.AuthMethod,
|
||||
DefaultAs: app.DefaultAs,
|
||||
}
|
||||
if app.KeyRef != nil {
|
||||
cfg.KeyLabel = app.KeyRef.ID
|
||||
}
|
||||
if len(app.Users) > 0 {
|
||||
cfg.UserOpenId = app.Users[0].UserOpenId
|
||||
cfg.UserName = app.Users[0].UserName
|
||||
|
||||
@@ -133,108 +133,6 @@ func TestResolveConfigFromMulti_AcceptsPlainSecret(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveConfigFromMulti_RejectsUnknownAuthMethod ensures an unsupported
|
||||
// authMethod fails at resolution rather than silently degrading to client_secret.
|
||||
func TestResolveConfigFromMulti_RejectsUnknownAuthMethod(t *testing.T) {
|
||||
raw := &MultiAppConfig{
|
||||
Apps: []AppConfig{
|
||||
{
|
||||
AppId: "cli_abc",
|
||||
AppSecret: PlainSecret("my-secret"),
|
||||
Brand: BrandFeishu,
|
||||
AuthMethod: "bogus_method",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
_, err := ResolveConfigFromMulti(raw, nil, "")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for unknown authMethod")
|
||||
}
|
||||
var cfgErr *errs.ConfigError
|
||||
if !errors.As(err, &cfgErr) {
|
||||
t.Fatalf("expected ConfigError, got %T: %v", err, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveConfigFromMulti_PrivateKeyJWTRequiresKeyRef ensures private_key_jwt
|
||||
// without a key handle fails at resolution rather than later at token-signing.
|
||||
func TestResolveConfigFromMulti_PrivateKeyJWTRequiresKeyRef(t *testing.T) {
|
||||
raw := &MultiAppConfig{
|
||||
Apps: []AppConfig{
|
||||
{
|
||||
AppId: "cli_abc",
|
||||
AppSecret: SecretInput{}, // private_key_jwt carries no app secret
|
||||
Brand: BrandFeishu,
|
||||
AuthMethod: AuthMethodPrivateKeyJWT,
|
||||
// KeyRef intentionally nil
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
_, err := ResolveConfigFromMulti(raw, nil, "")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for private_key_jwt without keyRef")
|
||||
}
|
||||
var cfgErr *errs.ConfigError
|
||||
if !errors.As(err, &cfgErr) {
|
||||
t.Fatalf("expected ConfigError, got %T: %v", err, err)
|
||||
}
|
||||
|
||||
// Control: same config WITH a keyRef resolves cleanly and sets KeyLabel.
|
||||
raw.Apps[0].KeyRef = &SecretRef{Source: "tee", ID: "larksuite-cli-agent"}
|
||||
cfg, err := ResolveConfigFromMulti(raw, nil, "")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error with keyRef present: %v", err)
|
||||
}
|
||||
if cfg.KeyLabel != "larksuite-cli-agent" {
|
||||
t.Errorf("KeyLabel = %q, want larksuite-cli-agent", cfg.KeyLabel)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveConfigFromMulti_PKJWTSkipsSecretResolution ensures a private_key_jwt
|
||||
// profile that carries a stale/broken AppSecret ref still resolves cleanly: the
|
||||
// auth method is judged before any secret handling, so the stale ref is ignored
|
||||
// instead of producing a confusing secret-resolution failure.
|
||||
func TestResolveConfigFromMulti_PKJWTSkipsSecretResolution(t *testing.T) {
|
||||
raw := &MultiAppConfig{
|
||||
Apps: []AppConfig{{
|
||||
AppId: "cli_pk",
|
||||
// Stale keychain ref whose ID does not match appId — would trip
|
||||
// ValidateSecretKeyMatch / ResolveSecretInput if it were reached.
|
||||
AppSecret: SecretInput{Ref: &SecretRef{Source: "keychain", ID: "appsecret:cli_OTHER"}},
|
||||
Brand: BrandFeishu,
|
||||
AuthMethod: AuthMethodPrivateKeyJWT,
|
||||
KeyRef: &SecretRef{Source: "tee", ID: "agent-key"},
|
||||
Users: []AppUser{},
|
||||
}},
|
||||
}
|
||||
cfg, err := ResolveConfigFromMulti(raw, stubKeychain{}, "")
|
||||
if err != nil {
|
||||
t.Fatalf("pkjwt with stale secret ref must skip secret resolution, got %v", err)
|
||||
}
|
||||
if cfg.AuthMethod != AuthMethodPrivateKeyJWT || cfg.KeyLabel != "agent-key" {
|
||||
t.Errorf("got authMethod=%q keyLabel=%q", cfg.AuthMethod, cfg.KeyLabel)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveConfigFromMulti_PKJWTRejectsBadKeyRef ensures the stricter keyRef
|
||||
// check (Source=="tee" && ID!="") rejects malformed handles.
|
||||
func TestResolveConfigFromMulti_PKJWTRejectsBadKeyRef(t *testing.T) {
|
||||
for i, ref := range []*SecretRef{
|
||||
{Source: "keychain", ID: "x"}, // wrong source
|
||||
{Source: "tee", ID: ""}, // empty id
|
||||
} {
|
||||
raw := &MultiAppConfig{Apps: []AppConfig{{
|
||||
AppId: "cli_pk", Brand: BrandFeishu,
|
||||
AuthMethod: AuthMethodPrivateKeyJWT, KeyRef: ref, Users: []AppUser{},
|
||||
}}}
|
||||
if _, err := ResolveConfigFromMulti(raw, stubKeychain{}, ""); err == nil {
|
||||
t.Errorf("case %d: expected ConfigError for bad keyRef", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveConfigFromMulti_CarriesLang(t *testing.T) {
|
||||
raw := &MultiAppConfig{
|
||||
Apps: []AppConfig{
|
||||
|
||||
@@ -3,8 +3,6 @@
|
||||
|
||||
package core
|
||||
|
||||
import "strings"
|
||||
|
||||
// LarkBrand represents the Lark platform brand.
|
||||
// "feishu" targets China-mainland, "lark" targets international.
|
||||
// Any other string is treated as a custom base URL.
|
||||
@@ -62,10 +60,3 @@ func ResolveEndpoints(brand LarkBrand) Endpoints {
|
||||
func ResolveOpenBaseURL(brand LarkBrand) string {
|
||||
return ResolveEndpoints(brand).Open
|
||||
}
|
||||
|
||||
// OpenAPIAudience returns the client_assertion `aud` value for the brand: the
|
||||
// bare Open API host per the App Authentication JWT spec — "open.feishu.cn" or
|
||||
// "open.larksuite.com" — not the full token endpoint URL.
|
||||
func OpenAPIAudience(brand LarkBrand) string {
|
||||
return strings.TrimPrefix(ResolveOpenBaseURL(brand), "https://")
|
||||
}
|
||||
|
||||
@@ -57,12 +57,3 @@ func TestResolveOpenBaseURL(t *testing.T) {
|
||||
t.Errorf("ResolveOpenBaseURL(lark) = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAPIAudience(t *testing.T) {
|
||||
if got := OpenAPIAudience(BrandFeishu); got != "open.feishu.cn" {
|
||||
t.Errorf("OpenAPIAudience(feishu) = %q, want open.feishu.cn", got)
|
||||
}
|
||||
if got := OpenAPIAudience(BrandLark); got != "open.larksuite.com" {
|
||||
t.Errorf("OpenAPIAudience(lark) = %q, want open.larksuite.com", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,6 @@ import (
|
||||
"github.com/larksuite/cli/internal/keychain"
|
||||
|
||||
extcred "github.com/larksuite/cli/extension/credential"
|
||||
"github.com/larksuite/cli/internal/keysigner"
|
||||
)
|
||||
|
||||
// classifyTATResponseCode wraps a deterministic (non-transient) failure from the
|
||||
@@ -176,23 +175,6 @@ func (p *DefaultTokenProvider) doResolveTAT(ctx context.Context) (*TokenResult,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// private_key_jwt apps have no app secret: mint via the jwt-bearer grant
|
||||
// using a TEE-signed client_assertion instead.
|
||||
if acct.AuthMethod == core.AuthMethodPrivateKeyJWT {
|
||||
signer := keysigner.Active()
|
||||
if signer == nil {
|
||||
return nil, errs.NewConfigError(errs.SubtypeInvalidClient,
|
||||
"profile uses private_key_jwt but no TEE key signer is available on this build").
|
||||
WithHint("install a build with the platform key-signer extension, or reconfigure the app to use an app secret")
|
||||
}
|
||||
token, err := FetchTATWithAssertion(ctx, httpClient, acct.Brand, acct.AppID, signer, acct.KeyLabel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &TokenResult{Token: token}, nil
|
||||
}
|
||||
|
||||
token, err := FetchTAT(ctx, httpClient, acct.Brand, acct.AppID, acct.AppSecret)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -11,13 +11,8 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/larksuite/cli/errs"
|
||||
"github.com/larksuite/cli/internal/auth"
|
||||
"github.com/larksuite/cli/internal/auth/jwt"
|
||||
"github.com/larksuite/cli/internal/core"
|
||||
"github.com/larksuite/cli/internal/keysigner"
|
||||
)
|
||||
|
||||
// FetchTAT performs a single HTTP POST to mint a tenant access token via the
|
||||
@@ -105,96 +100,3 @@ func FetchTAT(ctx context.Context, httpClient *http.Client, brand core.LarkBrand
|
||||
}
|
||||
return "", classifyTATResponseCode(result.Code, result.Error, desc, string(brand), appID)
|
||||
}
|
||||
|
||||
// FetchTATWithAssertion mints a tenant access token for a private_key_jwt app via
|
||||
// the RFC 7523 jwt-bearer grant: it signs a short-lived client_assertion with the
|
||||
// TEE-held key and posts it to the unified OAuth token endpoint, replacing the
|
||||
// app_secret entirely.
|
||||
//
|
||||
// The unified v2 token endpoint returns the minted token as access_token
|
||||
// (tenant_access_token is accepted as a fallback).
|
||||
func FetchTATWithAssertion(ctx context.Context, httpClient *http.Client, brand core.LarkBrand, clientID string, signer keysigner.Signer, keyLabel string) (string, error) {
|
||||
if signer == nil {
|
||||
return "", fmt.Errorf("private_key_jwt requires a key signer, but none is available on this build")
|
||||
}
|
||||
ep := core.ResolveEndpoints(brand)
|
||||
endpoint := ep.Open + auth.PathOAuthTokenV2
|
||||
|
||||
assertion, err := jwt.SignClientAssertion(ctx, signer, keysigner.KeyRef{Label: keyLabel}, clientID, core.OpenAPIAudience(brand), time.Now())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
form := url.Values{}
|
||||
form.Set("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer")
|
||||
form.Set("client_id", clientID)
|
||||
form.Set("client_assertion_type", jwt.ClientAssertionType)
|
||||
form.Set("client_assertion", assertion)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, strings.NewReader(form.Encode()))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read token response: %w", err)
|
||||
}
|
||||
|
||||
var result struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Error string `json:"error"`
|
||||
ErrorDescription string `json:"error_description"`
|
||||
AccessToken string `json:"access_token"`
|
||||
TenantAccessToken string `json:"tenant_access_token"`
|
||||
}
|
||||
_ = json.Unmarshal(body, &result) // best-effort; error body may not be JSON
|
||||
|
||||
token := result.AccessToken
|
||||
if token == "" {
|
||||
token = result.TenantAccessToken
|
||||
}
|
||||
if resp.StatusCode == http.StatusOK && token != "" && result.Error == "" && result.Code == 0 {
|
||||
return token, nil
|
||||
}
|
||||
|
||||
// Surface the server's reason, preferring the OAuth `error` code (e.g.
|
||||
// unauthorized_client) which is more diagnostic than the description alone.
|
||||
detail := result.ErrorDescription
|
||||
if detail == "" {
|
||||
detail = result.Msg
|
||||
}
|
||||
if detail == "" {
|
||||
detail = strings.TrimSpace(string(body))
|
||||
}
|
||||
if result.Error != "" {
|
||||
return "", classifyAssertionError(result.Error, resp.StatusCode, detail)
|
||||
}
|
||||
return "", fmt.Errorf("token endpoint HTTP %d (code=%d): %s", resp.StatusCode, result.Code, detail)
|
||||
}
|
||||
|
||||
// classifyAssertionError maps the OAuth token endpoint's `error` field to a
|
||||
// typed or untyped error. Only deterministic client-credential rejections get a
|
||||
// typed errs.ConfigError (so runProbePKJWT can tell "this key is not bound to
|
||||
// this app" apart from upstream noise); every other error (e.g.
|
||||
// temporarily_unavailable) stays untyped and is swallowed by the probe. detail
|
||||
// carries only the server's error_description / msg / body text — it never
|
||||
// echoes the client_assertion or private key (the assertion lives only in the
|
||||
// request form).
|
||||
func classifyAssertionError(oauthError string, httpStatus int, detail string) error {
|
||||
switch oauthError {
|
||||
case "invalid_client", "unauthorized_client", "invalid_grant":
|
||||
return errs.NewConfigError(errs.SubtypeInvalidClient,
|
||||
"token endpoint rejected the key (%s): %s", oauthError, detail)
|
||||
default:
|
||||
return fmt.Errorf("token endpoint HTTP %d (%s): %s", httpStatus, oauthError, detail)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,24 +5,15 @@ package credential
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/larksuite/cli/errs"
|
||||
"github.com/larksuite/cli/internal/core"
|
||||
"github.com/larksuite/cli/internal/keysigner"
|
||||
)
|
||||
|
||||
// stubRoundTripper lets us assert request shape and return canned responses.
|
||||
@@ -316,147 +307,3 @@ func (r *urlRewriteRT) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
req2.Header = req.Header
|
||||
return http.DefaultTransport.RoundTrip(req2)
|
||||
}
|
||||
|
||||
// fakeTATSigner is a real in-memory ECDSA P-256 signer for assertion tests.
|
||||
type fakeTATSigner struct{ key *ecdsa.PrivateKey }
|
||||
|
||||
func newFakeTATSigner(t *testing.T) *fakeTATSigner {
|
||||
t.Helper()
|
||||
k, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &fakeTATSigner{key: k}
|
||||
}
|
||||
|
||||
func (f *fakeTATSigner) EnsureKey(context.Context, keysigner.KeyRef) (crypto.PublicKey, error) {
|
||||
return f.key.Public(), nil
|
||||
}
|
||||
func (f *fakeTATSigner) PublicKey(context.Context, keysigner.KeyRef) (crypto.PublicKey, error) {
|
||||
return f.key.Public(), nil
|
||||
}
|
||||
func (f *fakeTATSigner) Sign(_ context.Context, _ keysigner.KeyRef, in []byte) ([]byte, string, error) {
|
||||
h := sha256.Sum256(in)
|
||||
r, s, err := ecdsa.Sign(rand.Reader, f.key, h[:])
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
sig := make([]byte, 64)
|
||||
r.FillBytes(sig[:32])
|
||||
s.FillBytes(sig[32:])
|
||||
return sig, keysigner.AlgES256, nil
|
||||
}
|
||||
|
||||
func TestFetchTATWithAssertion_Success(t *testing.T) {
|
||||
rt := &stubRoundTripper{respCode: 200, respBody: `{"access_token":"t-jwt","token_type":"Bearer","expires_in":7200}`}
|
||||
hc := &http.Client{Transport: rt}
|
||||
|
||||
token, err := FetchTATWithAssertion(context.Background(), hc, core.BrandFeishu, "cli_app", newFakeTATSigner(t), "agent-key")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if token != "t-jwt" {
|
||||
t.Errorf("token = %q, want t-jwt", token)
|
||||
}
|
||||
if rt.gotReq.URL.String() != "https://open.feishu.cn/open-apis/authen/v2/oauth/token" {
|
||||
t.Errorf("url = %s", rt.gotReq.URL.String())
|
||||
}
|
||||
|
||||
form, err := url.ParseQuery(rt.gotBody)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if form.Get("grant_type") != "urn:ietf:params:oauth:grant-type:jwt-bearer" {
|
||||
t.Errorf("grant_type = %q", form.Get("grant_type"))
|
||||
}
|
||||
if form.Get("client_assertion_type") != "urn:ietf:params:oauth:client-assertion-type:jwt-bearer" {
|
||||
t.Errorf("client_assertion_type = %q", form.Get("client_assertion_type"))
|
||||
}
|
||||
if form.Get("client_assertion") == "" {
|
||||
t.Error("client_assertion is empty")
|
||||
}
|
||||
if form.Has("client_secret") {
|
||||
t.Error("client_secret must NOT be sent for private_key_jwt")
|
||||
}
|
||||
|
||||
// The assertion's aud must be the bare Open host per the App Authentication
|
||||
// JWT spec — not the full token endpoint URL.
|
||||
jwtParts := strings.Split(form.Get("client_assertion"), ".")
|
||||
if len(jwtParts) != 3 {
|
||||
t.Fatalf("malformed client_assertion: %q", form.Get("client_assertion"))
|
||||
}
|
||||
payload, err := base64.RawURLEncoding.DecodeString(jwtParts[1])
|
||||
if err != nil {
|
||||
t.Fatalf("assertion payload not base64url: %v", err)
|
||||
}
|
||||
var claims map[string]any
|
||||
if err := json.Unmarshal(payload, &claims); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if claims["aud"] != "open.feishu.cn" {
|
||||
t.Errorf("client_assertion aud = %v, want open.feishu.cn", claims["aud"])
|
||||
}
|
||||
if claims["iss"] != "cli_app" || claims["sub"] != "cli_app" {
|
||||
t.Errorf("client_assertion iss/sub = %v/%v, want cli_app", claims["iss"], claims["sub"])
|
||||
}
|
||||
if form.Get("client_id") != "cli_app" {
|
||||
t.Errorf("client_id = %q", form.Get("client_id"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchTATWithAssertion_NilSigner(t *testing.T) {
|
||||
hc := &http.Client{Transport: &stubRoundTripper{respCode: 200, respBody: `{}`}}
|
||||
if _, err := FetchTATWithAssertion(context.Background(), hc, core.BrandFeishu, "cli_app", nil, "k"); err == nil {
|
||||
t.Fatal("expected error when signer is nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchTATWithAssertion_ServerError(t *testing.T) {
|
||||
rt := &stubRoundTripper{respCode: 200, respBody: `{"error":"invalid_client","error_description":"unknown key"}`}
|
||||
hc := &http.Client{Transport: rt}
|
||||
if _, err := FetchTATWithAssertion(context.Background(), hc, core.BrandFeishu, "cli_app", newFakeTATSigner(t), "k"); err == nil {
|
||||
t.Fatal("expected error for invalid_client response")
|
||||
}
|
||||
}
|
||||
|
||||
// Deterministic OAuth client rejections must be typed (ConfigError /
|
||||
// SubtypeInvalidClient) so runProbePKJWT can tell "the key is not bound to this
|
||||
// app" apart from transport noise.
|
||||
func TestFetchTATWithAssertion_DeterministicReject_Typed(t *testing.T) {
|
||||
for _, oauthErr := range []string{"invalid_client", "unauthorized_client", "invalid_grant"} {
|
||||
rt := &stubRoundTripper{respCode: 401, respBody: `{"error":"` + oauthErr + `","error_description":"bad key"}`}
|
||||
hc := &http.Client{Transport: rt}
|
||||
_, err := FetchTATWithAssertion(context.Background(), hc, core.BrandFeishu, "cli_app", newFakeTATSigner(t), "k")
|
||||
if err == nil {
|
||||
t.Fatalf("%s: expected error", oauthErr)
|
||||
}
|
||||
if !errs.IsTyped(err) {
|
||||
t.Errorf("%s: must be typed, got %T", oauthErr, err)
|
||||
}
|
||||
var cfgErr *errs.ConfigError
|
||||
if !errors.As(err, &cfgErr) || cfgErr.Subtype != errs.SubtypeInvalidClient {
|
||||
t.Errorf("%s: want ConfigError/InvalidClient, got %T %v", oauthErr, err, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Unrecognized OAuth errors and non-payload noise stay UNTYPED so the probe
|
||||
// treats them as upstream noise and stays silent.
|
||||
func TestFetchTATWithAssertion_AmbiguousError_Untyped(t *testing.T) {
|
||||
cases := []string{
|
||||
`{"error":"temporarily_unavailable","error_description":"retry"}`,
|
||||
`{"code":99999,"msg":"weird"}`,
|
||||
`not json`,
|
||||
}
|
||||
for _, body := range cases {
|
||||
rt := &stubRoundTripper{respCode: 503, respBody: body}
|
||||
hc := &http.Client{Transport: rt}
|
||||
_, err := FetchTATWithAssertion(context.Background(), hc, core.BrandFeishu, "cli_app", newFakeTATSigner(t), "k")
|
||||
if err == nil {
|
||||
t.Fatalf("body %q: expected error", body)
|
||||
}
|
||||
if errs.IsTyped(err) {
|
||||
t.Errorf("body %q: must be UNTYPED, got typed %T", body, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,8 +26,6 @@ type Account struct {
|
||||
UserName string
|
||||
Lang i18n.Lang
|
||||
SupportedIdentities uint8
|
||||
AuthMethod string // "" == client_secret; core.AuthMethodPrivateKeyJWT
|
||||
KeyLabel string // resolved TEE key handle for private_key_jwt
|
||||
}
|
||||
|
||||
const runtimePlaceholderAppSecret = "__LARKSUITE_CLI_TOKEN_ONLY__"
|
||||
@@ -71,8 +69,6 @@ func AccountFromCliConfig(cfg *core.CliConfig) *Account {
|
||||
UserName: cfg.UserName,
|
||||
Lang: cfg.Lang,
|
||||
SupportedIdentities: cfg.SupportedIdentities,
|
||||
AuthMethod: cfg.AuthMethod,
|
||||
KeyLabel: cfg.KeyLabel,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -91,8 +87,6 @@ func (a *Account) ToCliConfig() *core.CliConfig {
|
||||
UserName: a.UserName,
|
||||
Lang: a.Lang,
|
||||
SupportedIdentities: a.SupportedIdentities,
|
||||
AuthMethod: a.AuthMethod,
|
||||
KeyLabel: a.KeyLabel,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -82,9 +82,7 @@ func diagnoseBot(ctx context.Context, f *cmdutil.Factory, cfg *core.CliConfig, v
|
||||
Hint: "check strict mode or the active credential provider",
|
||||
}
|
||||
}
|
||||
// private_key_jwt apps have no app secret — the bot/tenant token is minted via
|
||||
// a TEE-signed client_assertion — so absence of a secret is NOT "unconfigured".
|
||||
if cfg.SupportedIdentities == 0 && !credential.HasRealAppSecret(cfg.AppSecret) && cfg.AuthMethod != core.AuthMethodPrivateKeyJWT {
|
||||
if cfg.SupportedIdentities == 0 && !credential.HasRealAppSecret(cfg.AppSecret) {
|
||||
return Identity{
|
||||
Status: StatusNotConfigured,
|
||||
Message: "Bot identity: not configured (missing app secret or bot token)",
|
||||
|
||||
@@ -1,212 +0,0 @@
|
||||
// Copyright (c) 2026 Lark Technologies Pte. Ltd.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
// Package keysigner defines the pluggable signing abstraction used by the
|
||||
// private_key_jwt registration and authentication flow.
|
||||
//
|
||||
// The open-source core only declares the Signer interface and pure-stdlib key
|
||||
// helpers. The platform implementations that hold a non-exportable private key
|
||||
// (TPM 2.0 via facebookincubator/sks on Linux/Windows, a non-extractable
|
||||
// Keychain key on macOS) live OUTSIDE this core — in a build-tagged module or
|
||||
// extension — and register themselves via Register from init(). This keeps
|
||||
// CGO-heavy and license-sensitive dependencies out of the open-source build.
|
||||
package keysigner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rsa"
|
||||
"crypto/x509"
|
||||
"encoding/asn1"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// KeyRef identifies a non-exportable signing key held by a backend
|
||||
// (TEE/TPM/Keychain). It is a stable handle (label), never the key material.
|
||||
type KeyRef struct {
|
||||
// Label is the backend key label/tag (e.g. "larksuite-cli-agent").
|
||||
Label string
|
||||
}
|
||||
|
||||
// Signer signs JWS signing inputs with a non-exportable key.
|
||||
type Signer interface {
|
||||
// EnsureKey returns the public key for ref, creating the key if absent.
|
||||
EnsureKey(ctx context.Context, ref KeyRef) (crypto.PublicKey, error)
|
||||
// PublicKey returns the public key for ref without creating it.
|
||||
PublicKey(ctx context.Context, ref KeyRef) (crypto.PublicKey, error)
|
||||
// Sign signs signingInput and returns a JOSE-format signature plus the JWS
|
||||
// alg ("ES256"/"RS256"). Implementations apply the alg's hash and, for
|
||||
// ECDSA, MUST return the fixed-width r||s form required by RFC 7518 §3.4
|
||||
// (not ASN.1 DER), because the backend (TPM/Keychain) typically yields DER.
|
||||
Sign(ctx context.Context, ref KeyRef, signingInput []byte) (sig []byte, alg string, err error)
|
||||
}
|
||||
|
||||
// Supported JWS algorithms.
|
||||
const (
|
||||
AlgES256 = "ES256"
|
||||
AlgRS256 = "RS256"
|
||||
)
|
||||
|
||||
// DefaultKeyLabel is the backend key label lark-cli uses for its device signing
|
||||
// key. One non-exportable key is created on first private_key_jwt registration
|
||||
// and reused across subsequent app registrations on the same device.
|
||||
const DefaultKeyLabel = "larksuite-cli-agent"
|
||||
|
||||
// HardwareInfo describes the secure hardware backing a Signer, as reported by a
|
||||
// HardwareProber. It is advisory/diagnostic: it tells a user whether
|
||||
// private_key_jwt can use a real TEE on this device.
|
||||
type HardwareInfo struct {
|
||||
Backend string // backing technology, e.g. "tpm2" or "keychain"
|
||||
Available bool // the hardware is present and usable for signing
|
||||
VendorName string // hardware vendor/manufacturer, when known
|
||||
VendorInfo string // additional vendor detail, when known
|
||||
Reason string // when Available is false, a human-readable cause
|
||||
}
|
||||
|
||||
// HardwareProber is an optional capability a Signer may implement to report on
|
||||
// the secure hardware backing it (TPM/TEE vendor and availability) WITHOUT
|
||||
// creating or using a key. Probing never mutates key state.
|
||||
type HardwareProber interface {
|
||||
ProbeHardware(ctx context.Context) (HardwareInfo, error)
|
||||
}
|
||||
|
||||
// ProbeActiveHardware probes the active signer's secure hardware. ok is false
|
||||
// when there is no active signer or it does not implement HardwareProber — in
|
||||
// which case private_key_jwt is unsupported on this build. When ok is true, info
|
||||
// reports availability and, if unavailable, info.Reason explains why.
|
||||
func ProbeActiveHardware(ctx context.Context) (info HardwareInfo, ok bool, err error) {
|
||||
return probeHardware(ctx, Active())
|
||||
}
|
||||
|
||||
// probeHardware is the registry-independent core of ProbeActiveHardware, so it
|
||||
// can be unit-tested without touching the global signer.
|
||||
func probeHardware(ctx context.Context, s Signer) (HardwareInfo, bool, error) {
|
||||
p, ok := s.(HardwareProber)
|
||||
if !ok {
|
||||
return HardwareInfo{}, false, nil
|
||||
}
|
||||
info, err := p.ProbeHardware(ctx)
|
||||
return info, true, err
|
||||
}
|
||||
|
||||
// cleanProbeError renders err's message with redundant re-wraps collapsed. Some
|
||||
// backends (e.g. facebookincubator/sks) wrap an error twice with the SAME "%w"
|
||||
// prefix, yielding "P: P: cause"; this peels each outer layer whose only
|
||||
// contribution is to repeat the prefix already present in the wrapped error,
|
||||
// leaving a single "P: cause". A layer that adds genuinely new context is kept.
|
||||
func cleanProbeError(err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
msg := err.Error()
|
||||
for {
|
||||
inner := errors.Unwrap(err)
|
||||
if inner == nil {
|
||||
break
|
||||
}
|
||||
innerMsg := inner.Error()
|
||||
prefix, ok := strings.CutSuffix(msg, innerMsg)
|
||||
if !ok || prefix == "" || !strings.HasPrefix(innerMsg, prefix) {
|
||||
break
|
||||
}
|
||||
msg, err = innerMsg, inner
|
||||
}
|
||||
return msg
|
||||
}
|
||||
|
||||
// AlgForKey returns the JWS alg for a public key: EC P-256 -> ES256, RSA -> RS256.
|
||||
// The signer backend chooses the key type (the macOS keychain signer uses an
|
||||
// RSA-2048 key, hence RS256).
|
||||
func AlgForKey(pub crypto.PublicKey) (string, error) {
|
||||
switch k := pub.(type) {
|
||||
case *ecdsa.PublicKey:
|
||||
if k.Curve == elliptic.P256() {
|
||||
return AlgES256, nil
|
||||
}
|
||||
return "", fmt.Errorf("keysigner: unsupported EC curve %q (only P-256/ES256)", k.Curve.Params().Name)
|
||||
case *rsa.PublicKey:
|
||||
return AlgRS256, nil
|
||||
default:
|
||||
return "", fmt.Errorf("keysigner: unsupported public key type %T", pub)
|
||||
}
|
||||
}
|
||||
|
||||
// ecdsaDERToJOSE converts an ASN.1 DER-encoded ECDSA signature — the form most
|
||||
// TEE/TPM backends emit (e.g. facebookincubator/sks marshals the TPM's r,s with
|
||||
// asn1.Marshal) — into the fixed-width r||s form JWS requires for ES256
|
||||
// (RFC 7518 §3.4). byteLen is the curve coordinate size (32 for P-256), so the
|
||||
// result is exactly 2*byteLen bytes with r and s each left-zero-padded.
|
||||
//
|
||||
// This is intentionally part of the pure-stdlib core (not a platform signer) so
|
||||
// it can be unit-tested with a software key on any machine, including TPM-less CI.
|
||||
func ecdsaDERToJOSE(der []byte, byteLen int) ([]byte, error) {
|
||||
var sig struct{ R, S *big.Int }
|
||||
rest, err := asn1.Unmarshal(der, &sig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("keysigner: parse ECDSA DER signature: %w", err)
|
||||
}
|
||||
if len(rest) != 0 {
|
||||
return nil, fmt.Errorf("keysigner: %d trailing byte(s) after ECDSA DER signature", len(rest))
|
||||
}
|
||||
if sig.R == nil || sig.S == nil || sig.R.Sign() <= 0 || sig.S.Sign() <= 0 {
|
||||
return nil, fmt.Errorf("keysigner: ECDSA signature has non-positive r/s")
|
||||
}
|
||||
// Guard before FillBytes, which panics if the scalar does not fit in byteLen.
|
||||
if sig.R.BitLen() > byteLen*8 || sig.S.BitLen() > byteLen*8 {
|
||||
return nil, fmt.Errorf("keysigner: ECDSA r/s exceeds %d-byte coordinate", byteLen)
|
||||
}
|
||||
out := make([]byte, 2*byteLen)
|
||||
sig.R.FillBytes(out[:byteLen])
|
||||
sig.S.FillBytes(out[byteLen:])
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// EncodePublicKey marshals pub to PKIX DER and base64-encodes it (std encoding),
|
||||
// matching the public-key form the registration backend binds to the app.
|
||||
func EncodePublicKey(pub crypto.PublicKey) (string, error) {
|
||||
der, err := x509.MarshalPKIXPublicKey(pub)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("keysigner: encode public key: %w", err)
|
||||
}
|
||||
return base64.StdEncoding.EncodeToString(der), nil
|
||||
}
|
||||
|
||||
// PublicKeyJWK returns the RFC 7517 JSON Web Key for pub, used to embed the
|
||||
// public key in the attestation JWT's "jwk" header so the registration backend
|
||||
// can bind it to the app. EC keys use base64url fixed-width coordinates
|
||||
// (RFC 7518 §6.2.1); RSA keys use base64url-encoded modulus and exponent.
|
||||
func PublicKeyJWK(pub crypto.PublicKey) (map[string]any, error) {
|
||||
switch k := pub.(type) {
|
||||
case *ecdsa.PublicKey:
|
||||
if k.Curve != elliptic.P256() {
|
||||
return nil, fmt.Errorf("keysigner: JWK supports EC P-256 only, got %q", k.Curve.Params().Name)
|
||||
}
|
||||
const coordLen = 32 // P-256 field element size
|
||||
x := make([]byte, coordLen)
|
||||
y := make([]byte, coordLen)
|
||||
k.X.FillBytes(x)
|
||||
k.Y.FillBytes(y)
|
||||
return map[string]any{
|
||||
"use": "sig",
|
||||
"kty": "EC",
|
||||
"crv": "P-256",
|
||||
"x": base64.RawURLEncoding.EncodeToString(x),
|
||||
"y": base64.RawURLEncoding.EncodeToString(y),
|
||||
}, nil
|
||||
case *rsa.PublicKey:
|
||||
return map[string]any{
|
||||
"use": "sig",
|
||||
"kty": "RSA",
|
||||
"n": base64.RawURLEncoding.EncodeToString(k.N.Bytes()),
|
||||
"e": base64.RawURLEncoding.EncodeToString(big.NewInt(int64(k.E)).Bytes()),
|
||||
}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("keysigner: unsupported public key type %T for JWK", pub)
|
||||
}
|
||||
}
|
||||
@@ -1,240 +0,0 @@
|
||||
// Copyright (c) 2026 Lark Technologies Pte. Ltd.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package keysigner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/sha256"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAlgForKey(t *testing.T) {
|
||||
ec, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if alg, err := AlgForKey(ec.Public()); err != nil || alg != AlgES256 {
|
||||
t.Errorf("P-256: alg=%q err=%v, want ES256/nil", alg, err)
|
||||
}
|
||||
|
||||
rsaKey, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if alg, err := AlgForKey(rsaKey.Public()); err != nil || alg != AlgRS256 {
|
||||
t.Errorf("RSA: alg=%q err=%v, want RS256/nil", alg, err)
|
||||
}
|
||||
|
||||
ec384, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := AlgForKey(ec384.Public()); err == nil {
|
||||
t.Error("P-384: expected unsupported-curve error")
|
||||
}
|
||||
|
||||
if _, err := AlgForKey("not a key"); err == nil {
|
||||
t.Error("string: expected unsupported-type error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodePublicKeyRoundTrip(t *testing.T) {
|
||||
ec, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
enc, err := EncodePublicKey(ec.Public())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
der, err := base64.StdEncoding.DecodeString(enc)
|
||||
if err != nil {
|
||||
t.Fatalf("not valid base64: %v", err)
|
||||
}
|
||||
pub, err := x509.ParsePKIXPublicKey(der)
|
||||
if err != nil {
|
||||
t.Fatalf("not valid PKIX: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(pub, ec.Public()) {
|
||||
t.Error("public key did not round-trip")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicKeyJWK_EC(t *testing.T) {
|
||||
ec, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jwk, err := PublicKeyJWK(ec.Public())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if jwk["kty"] != "EC" || jwk["crv"] != "P-256" {
|
||||
t.Errorf("jwk = %v, want kty=EC crv=P-256", jwk)
|
||||
}
|
||||
if jwk["use"] != "sig" {
|
||||
t.Errorf("jwk use = %v, want sig", jwk["use"])
|
||||
}
|
||||
x, _ := jwk["x"].(string)
|
||||
xb, err := base64.RawURLEncoding.DecodeString(x)
|
||||
if err != nil || len(xb) != 32 {
|
||||
t.Errorf("x = %q (decoded %d bytes), want 32-byte base64url", x, len(xb))
|
||||
}
|
||||
if _, ok := jwk["y"].(string); !ok {
|
||||
t.Error("jwk missing y")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicKeyJWK_RSA(t *testing.T) {
|
||||
rsaKey, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jwk, err := PublicKeyJWK(rsaKey.Public())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if jwk["kty"] != "RSA" || jwk["n"] == "" || jwk["e"] == "" {
|
||||
t.Errorf("jwk = %v, want kty=RSA with n,e", jwk)
|
||||
}
|
||||
if jwk["use"] != "sig" {
|
||||
t.Errorf("jwk use = %v, want sig", jwk["use"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicKeyJWK_UnsupportedCurve(t *testing.T) {
|
||||
ec384, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := PublicKeyJWK(ec384.Public()); err == nil {
|
||||
t.Error("P-384: expected error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestECDSADERToJOSE(t *testing.T) {
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Iterate so we hit signatures whose r or s has its high bit set (ASN.1 pads
|
||||
// those with a leading 0x00) and whose scalars are short (need left-zero
|
||||
// padding) — verifying fixed-width conversion in both directions.
|
||||
for i := 0; i < 64; i++ {
|
||||
digest := sha256.Sum256([]byte{byte(i), byte(i >> 8), 'j', 'w', 't'})
|
||||
der, err := ecdsa.SignASN1(rand.Reader, key, digest[:])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jose, err := ecdsaDERToJOSE(der, 32)
|
||||
if err != nil {
|
||||
t.Fatalf("iter %d: %v", i, err)
|
||||
}
|
||||
if len(jose) != 64 {
|
||||
t.Fatalf("iter %d: len(jose)=%d, want 64 (fixed-width r||s)", i, len(jose))
|
||||
}
|
||||
r := new(big.Int).SetBytes(jose[:32])
|
||||
s := new(big.Int).SetBytes(jose[32:])
|
||||
if !ecdsa.Verify(&key.PublicKey, digest[:], r, s) {
|
||||
t.Fatalf("iter %d: converted r||s did not verify against the public key", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestECDSADERToJOSE_Errors(t *testing.T) {
|
||||
if _, err := ecdsaDERToJOSE([]byte{0x01, 0x02, 0x03}, 32); err == nil {
|
||||
t.Error("garbage DER: expected error")
|
||||
}
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := sha256.Sum256([]byte("trailing"))
|
||||
der, err := ecdsa.SignASN1(rand.Reader, key, digest[:])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ecdsaDERToJOSE(append(der, 0x00), 32); err == nil {
|
||||
t.Error("DER with trailing byte: expected error")
|
||||
}
|
||||
}
|
||||
|
||||
type stubSigner struct{}
|
||||
|
||||
func (stubSigner) EnsureKey(context.Context, KeyRef) (crypto.PublicKey, error) { return nil, nil }
|
||||
func (stubSigner) PublicKey(context.Context, KeyRef) (crypto.PublicKey, error) { return nil, nil }
|
||||
func (stubSigner) Sign(context.Context, KeyRef, []byte) ([]byte, string, error) { return nil, "", nil }
|
||||
|
||||
func TestCleanProbeError(t *testing.T) {
|
||||
cause := errors.New("open /dev/tpmrm0: permission denied")
|
||||
const p = "sks: error fetching Secure Hardware Vendor Data: "
|
||||
|
||||
// sks double-wraps with the same %w prefix → collapse to a single prefix.
|
||||
doubled := fmt.Errorf(p+"%w", fmt.Errorf(p+"%w", cause))
|
||||
if got, want := cleanProbeError(doubled), p+cause.Error(); got != want {
|
||||
t.Errorf("doubled: got %q, want %q", got, want)
|
||||
}
|
||||
// Triple wrap collapses too.
|
||||
if got, want := cleanProbeError(fmt.Errorf(p+"%w", doubled)), p+cause.Error(); got != want {
|
||||
t.Errorf("tripled: got %q, want %q", got, want)
|
||||
}
|
||||
// A layer adding genuinely new context is preserved.
|
||||
if got, want := cleanProbeError(fmt.Errorf("load: %w", cause)), "load: "+cause.Error(); got != want {
|
||||
t.Errorf("distinct prefix: got %q, want %q", got, want)
|
||||
}
|
||||
// nil and unwrapped-leaf cases.
|
||||
if got := cleanProbeError(nil); got != "" {
|
||||
t.Errorf("nil: got %q, want empty", got)
|
||||
}
|
||||
if got := cleanProbeError(cause); got != cause.Error() {
|
||||
t.Errorf("leaf: got %q, want %q", got, cause.Error())
|
||||
}
|
||||
}
|
||||
|
||||
type proberSigner struct {
|
||||
stubSigner
|
||||
info HardwareInfo
|
||||
}
|
||||
|
||||
func (p proberSigner) ProbeHardware(context.Context) (HardwareInfo, error) { return p.info, nil }
|
||||
|
||||
func TestProbeHardware(t *testing.T) {
|
||||
// nil signer and a signer that does not implement HardwareProber both yield ok=false.
|
||||
if _, ok, _ := probeHardware(context.Background(), nil); ok {
|
||||
t.Error("nil signer: ok should be false")
|
||||
}
|
||||
if _, ok, _ := probeHardware(context.Background(), stubSigner{}); ok {
|
||||
t.Error("non-prober signer: ok should be false")
|
||||
}
|
||||
|
||||
want := HardwareInfo{Backend: "tpm2", Available: true, VendorName: "ACME"}
|
||||
info, ok, err := probeHardware(context.Background(), proberSigner{info: want})
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("prober: ok=%v err=%v, want true/nil", ok, err)
|
||||
}
|
||||
if info != want {
|
||||
t.Errorf("info = %+v, want %+v", info, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistry(t *testing.T) {
|
||||
if Active() != nil {
|
||||
t.Skip("a signer is already registered in this build")
|
||||
}
|
||||
Register(stubSigner{})
|
||||
if _, ok := Active().(stubSigner); !ok {
|
||||
t.Error("Active did not return the registered signer")
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
// Copyright (c) 2026 Lark Technologies Pte. Ltd.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package keysigner
|
||||
|
||||
import "sync"
|
||||
|
||||
var (
|
||||
mu sync.RWMutex
|
||||
active Signer
|
||||
)
|
||||
|
||||
// Register sets the active Signer. It is typically called from the init() of a
|
||||
// build-tagged or extension package that provides the platform TEE/Keychain
|
||||
// implementation. The last registration wins (one backend per platform).
|
||||
func Register(s Signer) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
active = s
|
||||
}
|
||||
|
||||
// Active returns the registered Signer, or nil if none is available — in which
|
||||
// case private_key_jwt is unsupported on this build and only client_secret auth
|
||||
// can be used.
|
||||
func Active() Signer {
|
||||
mu.RLock()
|
||||
defer mu.RUnlock()
|
||||
return active
|
||||
}
|
||||
@@ -1,613 +0,0 @@
|
||||
//go:build darwin
|
||||
|
||||
// Copyright (c) 2026 Lark Technologies Pte. Ltd.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
// macOS non-exportable Keychain signer (compiled into every darwin build).
|
||||
//
|
||||
// It does NOT use the Secure Enclave / hardware TEE (which would require
|
||||
// code-signing entitlements that are unfriendly to open source). Instead it
|
||||
// generates an RSA-2048 key in software, imports it into a dedicated app
|
||||
// keychain as NON-EXTRACTABLE (`security import -x`), then deletes the software
|
||||
// copy — so the private key can sign but can never be exported. Signing is
|
||||
// RSASSA-PKCS1v15-SHA256 (RS256).
|
||||
//
|
||||
// Unlike the original revision, this implementation calls the Security and
|
||||
// CoreFoundation frameworks via RUNTIME FFI (github.com/ebitengine/purego)
|
||||
// instead of cgo. The security model is identical (the private key is still a
|
||||
// non-extractable keychain key and every signature is produced by the OS via
|
||||
// SecKeyCreateSignature), but the binary builds with CGO_ENABLED=0 and can be
|
||||
// cross-compiled for darwin from any host — so release binaries no longer
|
||||
// require a native macOS build runner.
|
||||
//
|
||||
// Build with: go build (cgo-free; compiled into every darwin build, no tag)
|
||||
package keysigner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/sha1"
|
||||
"crypto/sha256"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"unsafe"
|
||||
|
||||
"github.com/ebitengine/purego"
|
||||
"github.com/larksuite/cli/internal/vfs"
|
||||
)
|
||||
|
||||
// ---- Security / CoreFoundation runtime bindings (purego, no cgo) ----
|
||||
|
||||
const (
|
||||
cfFrameworkPath = "/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation"
|
||||
secFrameworkPath = "/System/Library/Frameworks/Security.framework/Security"
|
||||
|
||||
// kCFStringEncodingUTF8 (CFStringBuiltInEncodings).
|
||||
cfStringEncodingUTF8 = 0x08000100
|
||||
|
||||
// OSStatus values.
|
||||
errSecSuccess = 0
|
||||
)
|
||||
|
||||
var (
|
||||
ffiOnce sync.Once
|
||||
ffiErr error
|
||||
|
||||
cfDataCreate func(alloc uintptr, bytes *byte, length int) uintptr
|
||||
cfDataGetLength func(d uintptr) int
|
||||
cfDataGetBytePtr func(d uintptr) unsafe.Pointer
|
||||
cfStringCreate func(alloc uintptr, cstr *byte, encoding uint32) uintptr
|
||||
cfArrayCreate func(alloc uintptr, values *uintptr, numValues int, cb uintptr) uintptr
|
||||
cfDictCreateMutable func(alloc uintptr, capacity int, keyCB, valCB uintptr) uintptr
|
||||
cfDictSetValue func(dict, key, val uintptr)
|
||||
cfRelease func(ref uintptr)
|
||||
cfErrorGetCode func(e uintptr) int
|
||||
secKeychainOpen func(path *byte, out *uintptr) int32
|
||||
secItemCopyMatching func(query uintptr, result *uintptr) int32
|
||||
secItemUpdate func(query, attrs uintptr) int32
|
||||
secKeyCreateSignature func(key, algo, data uintptr, errOut *uintptr) uintptr
|
||||
|
||||
// CFTypeRef data-symbol constants (deref to obtain the held ref value).
|
||||
kSecClass uintptr
|
||||
kSecClassKey uintptr
|
||||
kSecAttrKeyClass uintptr
|
||||
kSecAttrKeyClassPrivate uintptr
|
||||
kSecAttrKeyType uintptr
|
||||
kSecAttrKeyTypeRSA uintptr
|
||||
kSecAttrApplicationLabel uintptr
|
||||
kSecReturnRef uintptr
|
||||
kSecMatchSearchList uintptr
|
||||
kSecAttrLabel uintptr
|
||||
kCFBooleanTrue uintptr
|
||||
algRSAPKCS1SHA256 uintptr
|
||||
|
||||
// Struct-symbol constants (passed BY ADDRESS, not dereferenced).
|
||||
cbTypeArray uintptr
|
||||
cbDictKey uintptr
|
||||
cbDictValue uintptr
|
||||
)
|
||||
|
||||
// loadFFI resolves the framework functions and constants once. Any failure
|
||||
// (framework missing, symbol absent) is returned to every caller so signing
|
||||
// fails cleanly rather than crashing.
|
||||
func loadFFI() error {
|
||||
ffiOnce.Do(func() {
|
||||
cf, err := purego.Dlopen(cfFrameworkPath, purego.RTLD_NOW|purego.RTLD_GLOBAL)
|
||||
if err != nil {
|
||||
ffiErr = fmt.Errorf("keysigner: dlopen CoreFoundation: %w", err)
|
||||
return
|
||||
}
|
||||
sec, err := purego.Dlopen(secFrameworkPath, purego.RTLD_NOW|purego.RTLD_GLOBAL)
|
||||
if err != nil {
|
||||
ffiErr = fmt.Errorf("keysigner: dlopen Security: %w", err)
|
||||
return
|
||||
}
|
||||
|
||||
purego.RegisterLibFunc(&cfDataCreate, cf, "CFDataCreate")
|
||||
purego.RegisterLibFunc(&cfDataGetLength, cf, "CFDataGetLength")
|
||||
purego.RegisterLibFunc(&cfDataGetBytePtr, cf, "CFDataGetBytePtr")
|
||||
purego.RegisterLibFunc(&cfStringCreate, cf, "CFStringCreateWithCString")
|
||||
purego.RegisterLibFunc(&cfArrayCreate, cf, "CFArrayCreate")
|
||||
purego.RegisterLibFunc(&cfDictCreateMutable, cf, "CFDictionaryCreateMutable")
|
||||
purego.RegisterLibFunc(&cfDictSetValue, cf, "CFDictionarySetValue")
|
||||
purego.RegisterLibFunc(&cfRelease, cf, "CFRelease")
|
||||
purego.RegisterLibFunc(&cfErrorGetCode, cf, "CFErrorGetCode")
|
||||
purego.RegisterLibFunc(&secKeychainOpen, sec, "SecKeychainOpen")
|
||||
purego.RegisterLibFunc(&secItemCopyMatching, sec, "SecItemCopyMatching")
|
||||
purego.RegisterLibFunc(&secItemUpdate, sec, "SecItemUpdate")
|
||||
purego.RegisterLibFunc(&secKeyCreateSignature, sec, "SecKeyCreateSignature")
|
||||
|
||||
// CFStringRef/CFBooleanRef constants: Dlsym gives the address of the
|
||||
// exported variable; deref once to read the ref it holds.
|
||||
derefs := []struct {
|
||||
dst *uintptr
|
||||
handle uintptr
|
||||
name string
|
||||
}{
|
||||
{&kSecClass, sec, "kSecClass"},
|
||||
{&kSecClassKey, sec, "kSecClassKey"},
|
||||
{&kSecAttrKeyClass, sec, "kSecAttrKeyClass"},
|
||||
{&kSecAttrKeyClassPrivate, sec, "kSecAttrKeyClassPrivate"},
|
||||
{&kSecAttrKeyType, sec, "kSecAttrKeyType"},
|
||||
{&kSecAttrKeyTypeRSA, sec, "kSecAttrKeyTypeRSA"},
|
||||
{&kSecAttrApplicationLabel, sec, "kSecAttrApplicationLabel"},
|
||||
{&kSecReturnRef, sec, "kSecReturnRef"},
|
||||
{&kSecMatchSearchList, sec, "kSecMatchSearchList"},
|
||||
{&kSecAttrLabel, sec, "kSecAttrLabel"},
|
||||
{&kCFBooleanTrue, cf, "kCFBooleanTrue"},
|
||||
{&algRSAPKCS1SHA256, sec, "kSecKeyAlgorithmRSASignatureDigestPKCS1v15SHA256"},
|
||||
}
|
||||
for _, d := range derefs {
|
||||
sym, e := purego.Dlsym(d.handle, d.name)
|
||||
if e != nil || sym == 0 {
|
||||
ffiErr = fmt.Errorf("keysigner: dlsym %s: %v", d.name, e)
|
||||
return
|
||||
}
|
||||
// deref of a stable dylib data-symbol address (not Go-managed memory), so safe.
|
||||
*d.dst = *(*uintptr)(unsafe.Pointer(sym)) //nolint:govet // unsafeptr: see comment above
|
||||
}
|
||||
|
||||
// Callback structs are passed by address (no deref).
|
||||
addrs := []struct {
|
||||
dst *uintptr
|
||||
handle uintptr
|
||||
name string
|
||||
}{
|
||||
{&cbTypeArray, cf, "kCFTypeArrayCallBacks"},
|
||||
{&cbDictKey, cf, "kCFTypeDictionaryKeyCallBacks"},
|
||||
{&cbDictValue, cf, "kCFTypeDictionaryValueCallBacks"},
|
||||
}
|
||||
for _, a := range addrs {
|
||||
sym, e := purego.Dlsym(a.handle, a.name)
|
||||
if e != nil || sym == 0 {
|
||||
ffiErr = fmt.Errorf("keysigner: dlsym %s: %v", a.name, e)
|
||||
return
|
||||
}
|
||||
*a.dst = sym
|
||||
}
|
||||
})
|
||||
return ffiErr
|
||||
}
|
||||
|
||||
// cstr returns a pointer to a NUL-terminated copy of s. The backing array stays
|
||||
// alive while the returned pointer is reachable.
|
||||
func cstr(s string) *byte {
|
||||
b := append([]byte(s), 0)
|
||||
return &b[0]
|
||||
}
|
||||
|
||||
// cfBytes wraps Go bytes in a CFData (CFDataCreate copies the bytes). Caller
|
||||
// releases the returned CFDataRef.
|
||||
func cfBytes(b []byte) uintptr {
|
||||
var p *byte
|
||||
if len(b) > 0 {
|
||||
p = &b[0]
|
||||
}
|
||||
d := cfDataCreate(0, p, len(b))
|
||||
runtime.KeepAlive(b)
|
||||
return d
|
||||
}
|
||||
|
||||
// keychainSearchArray opens the dedicated keychain file and wraps it in a
|
||||
// CFArray for kSecMatchSearchList. Caller releases the returned array.
|
||||
//
|
||||
// NOTE: SecKeychainOpen / the file-based keychain are deprecated by Apple in
|
||||
// favor of the data-protection keychain. They still function on current macOS;
|
||||
// migrating off them is tracked separately and is independent of the cgo→purego
|
||||
// change (the original cgo version used the same APIs).
|
||||
func keychainSearchArray(keychainPath string) (uintptr, error) {
|
||||
var kc uintptr
|
||||
if st := secKeychainOpen(cstr(keychainPath), &kc); st != errSecSuccess {
|
||||
return 0, keychainError("open keychain", int(st))
|
||||
}
|
||||
vals := [1]uintptr{kc}
|
||||
arr := cfArrayCreate(0, &vals[0], 1, cbTypeArray)
|
||||
cfRelease(kc) // the array retains it
|
||||
if arr == 0 {
|
||||
return 0, fmt.Errorf("keysigner: CFArrayCreate(search list) failed")
|
||||
}
|
||||
return arr, nil
|
||||
}
|
||||
|
||||
// findPrivateKey locates the non-extractable private key by its application
|
||||
// label within the dedicated keychain. Caller releases the returned SecKeyRef.
|
||||
func findPrivateKey(appLabel []byte, keychainPath string) (uintptr, error) {
|
||||
search, err := keychainSearchArray(keychainPath)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer cfRelease(search)
|
||||
|
||||
labelData := cfBytes(appLabel)
|
||||
defer cfRelease(labelData)
|
||||
|
||||
q := cfDictCreateMutable(0, 0, cbDictKey, cbDictValue)
|
||||
if q == 0 {
|
||||
return 0, fmt.Errorf("keysigner: CFDictionaryCreateMutable(query) failed")
|
||||
}
|
||||
defer cfRelease(q)
|
||||
cfDictSetValue(q, kSecClass, kSecClassKey)
|
||||
cfDictSetValue(q, kSecAttrKeyClass, kSecAttrKeyClassPrivate)
|
||||
cfDictSetValue(q, kSecAttrKeyType, kSecAttrKeyTypeRSA)
|
||||
cfDictSetValue(q, kSecAttrApplicationLabel, labelData)
|
||||
cfDictSetValue(q, kSecReturnRef, kCFBooleanTrue)
|
||||
cfDictSetValue(q, kSecMatchSearchList, search)
|
||||
|
||||
var keyRef uintptr
|
||||
if st := secItemCopyMatching(q, &keyRef); st != errSecSuccess {
|
||||
return 0, keychainError("find private key", int(st))
|
||||
}
|
||||
return keyRef, nil
|
||||
}
|
||||
|
||||
// securityBin is invoked by absolute path so a poisoned PATH cannot hijack it.
|
||||
const securityBin = "/usr/bin/security"
|
||||
|
||||
// keychainSigner implements Signer using a macOS non-exportable Keychain key.
|
||||
type keychainSigner struct{}
|
||||
|
||||
func init() { Register(keychainSigner{}) }
|
||||
|
||||
// ProbeHardware reports the macOS Keychain backend backing this signer. The
|
||||
// keychain signer is compiled into every darwin build and needs no special
|
||||
// hardware, so it reports available whenever the Security tooling is present.
|
||||
// It performs no key access, so it never prompts. Implementing HardwareProber
|
||||
// is what lets `doctor` report the signer as present rather than treating the
|
||||
// (prober-less) signer as "no TEE signer in this build".
|
||||
func (keychainSigner) ProbeHardware(_ context.Context) (HardwareInfo, error) {
|
||||
info := HardwareInfo{Backend: "keychain", VendorName: "macOS Keychain"}
|
||||
// A missing security tool is a status (Available=false via Reason), not a
|
||||
// probe error — so we deliberately return a nil error here.
|
||||
if _, err := vfs.Stat(securityBin); err != nil {
|
||||
info.Reason = securityBin + " not found"
|
||||
return info, nil //nolint:nilerr // absence is reported via Reason, not as an error
|
||||
}
|
||||
info.Available = true
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func (keychainSigner) EnsureKey(_ context.Context, ref KeyRef) (crypto.PublicKey, error) {
|
||||
if md, err := readKeyMetadata(ref.Label); err == nil {
|
||||
return decodePublicKey(md.PublicKey)
|
||||
} else if !os.IsNotExist(err) {
|
||||
return nil, err
|
||||
}
|
||||
return createKeychainKey(ref.Label)
|
||||
}
|
||||
|
||||
func (keychainSigner) PublicKey(_ context.Context, ref KeyRef) (crypto.PublicKey, error) {
|
||||
md, err := readKeyMetadata(ref.Label)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return decodePublicKey(md.PublicKey)
|
||||
}
|
||||
|
||||
func (keychainSigner) Sign(_ context.Context, ref KeyRef, signingInput []byte) ([]byte, string, error) {
|
||||
if err := loadFFI(); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
md, err := readKeyMetadata(ref.Label)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
appLabel, err := hex.DecodeString(md.AppLabel)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("keysigner: decode app label: %w", err)
|
||||
}
|
||||
if len(appLabel) == 0 {
|
||||
// Guard the &appLabel[0] pointer below against corrupted metadata.
|
||||
return nil, "", fmt.Errorf("keysigner: key metadata for %q has empty app label", ref.Label)
|
||||
}
|
||||
keychain, err := ensureKeychain()
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
keyRef, err := findPrivateKey(appLabel, keychain)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
defer cfRelease(keyRef)
|
||||
|
||||
digest := sha256.Sum256(signingInput)
|
||||
digestData := cfBytes(digest[:])
|
||||
defer cfRelease(digestData)
|
||||
|
||||
var errRef uintptr
|
||||
sigRef := secKeyCreateSignature(keyRef, algRSAPKCS1SHA256, digestData, &errRef)
|
||||
if sigRef == 0 {
|
||||
code := 0
|
||||
if errRef != 0 {
|
||||
code = cfErrorGetCode(errRef)
|
||||
cfRelease(errRef)
|
||||
}
|
||||
return nil, "", fmt.Errorf("keysigner: SecKeyCreateSignature failed (CFError %d)", code)
|
||||
}
|
||||
defer cfRelease(sigRef)
|
||||
|
||||
n := cfDataGetLength(sigRef)
|
||||
bp := cfDataGetBytePtr(sigRef)
|
||||
out := make([]byte, n)
|
||||
copy(out, unsafe.Slice((*byte)(bp), n))
|
||||
// RS256: the SecKey PKCS1v15-SHA256 signature is the JOSE signature as-is.
|
||||
return out, AlgRS256, nil
|
||||
}
|
||||
|
||||
// keyMetadata records the public key + the keychain application-label used to
|
||||
// locate the non-extractable private key.
|
||||
type keyMetadata struct {
|
||||
PublicKey string `json:"public_key"` // PKIX DER, std base64 (see EncodePublicKey)
|
||||
AppLabel string `json:"app_label"` // hex(sha1(PKCS1 public key))
|
||||
}
|
||||
|
||||
func createKeychainKey(label string) (crypto.PublicKey, error) {
|
||||
metadataPath, err := keyMetadataPath(label)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("keysigner: generate RSA key: %w", err)
|
||||
}
|
||||
appLabel := sha1.Sum(x509.MarshalPKCS1PublicKey(&privateKey.PublicKey))
|
||||
|
||||
pemFile, err := vfs.CreateTemp("", "lark-keysigner-*.pem")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("keysigner: temp key file: %w", err)
|
||||
}
|
||||
pemPath := pemFile.Name()
|
||||
defer vfs.Remove(pemPath)
|
||||
if err := pemFile.Chmod(0600); err != nil {
|
||||
pemFile.Close()
|
||||
return nil, err
|
||||
}
|
||||
der := x509.MarshalPKCS1PrivateKey(privateKey)
|
||||
if _, err := pemFile.WriteString("-----BEGIN RSA PRIVATE KEY-----\n" +
|
||||
base64Wrap(der) + "-----END RSA PRIVATE KEY-----\n"); err != nil {
|
||||
pemFile.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := pemFile.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
executable, err := vfs.Executable()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("keysigner: resolve executable: %w", err)
|
||||
}
|
||||
keychain, err := ensureKeychain()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// -x: import as NON-EXTRACTABLE; the software copy (pemPath) is then removed.
|
||||
importCmd := exec.Command(securityBin, "import", pemPath, "-k", keychain, "-t", "priv", "-f", "openssl", "-x", "-A", "-T", executable)
|
||||
if out, err := importCmd.CombinedOutput(); err != nil {
|
||||
return nil, fmt.Errorf("keysigner: import non-extractable key: %w: %s", err, summarizeCmdOutput(out))
|
||||
}
|
||||
if err := setKeychainKeyLabel(appLabel[:], keychain, label); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
encodedPub, err := EncodePublicKey(&privateKey.PublicKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := writeKeyMetadata(metadataPath, keyMetadata{PublicKey: encodedPub, AppLabel: hex.EncodeToString(appLabel[:])}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &privateKey.PublicKey, nil
|
||||
}
|
||||
|
||||
func setKeychainKeyLabel(appLabel []byte, keychain, label string) error {
|
||||
if err := loadFFI(); err != nil {
|
||||
return err
|
||||
}
|
||||
search, err := keychainSearchArray(keychain)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cfRelease(search)
|
||||
|
||||
labelData := cfBytes(appLabel)
|
||||
defer cfRelease(labelData)
|
||||
|
||||
q := cfDictCreateMutable(0, 0, cbDictKey, cbDictValue)
|
||||
if q == 0 {
|
||||
return fmt.Errorf("keysigner: CFDictionaryCreateMutable(query) failed")
|
||||
}
|
||||
defer cfRelease(q)
|
||||
cfDictSetValue(q, kSecClass, kSecClassKey)
|
||||
cfDictSetValue(q, kSecAttrKeyClass, kSecAttrKeyClassPrivate)
|
||||
cfDictSetValue(q, kSecAttrKeyType, kSecAttrKeyTypeRSA)
|
||||
cfDictSetValue(q, kSecAttrApplicationLabel, labelData)
|
||||
cfDictSetValue(q, kSecMatchSearchList, search)
|
||||
|
||||
cfLabel := cfStringCreate(0, cstr(label), cfStringEncodingUTF8)
|
||||
if cfLabel == 0 {
|
||||
return fmt.Errorf("keysigner: CFStringCreateWithCString failed")
|
||||
}
|
||||
defer cfRelease(cfLabel)
|
||||
attrs := cfDictCreateMutable(0, 0, cbDictKey, cbDictValue)
|
||||
if attrs == 0 {
|
||||
return fmt.Errorf("keysigner: CFDictionaryCreateMutable(attrs) failed")
|
||||
}
|
||||
defer cfRelease(attrs)
|
||||
cfDictSetValue(attrs, kSecAttrLabel, cfLabel)
|
||||
|
||||
if st := secItemUpdate(q, attrs); st != errSecSuccess {
|
||||
return keychainError("set keychain key label", int(st))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func decodePublicKey(encoded string) (crypto.PublicKey, error) {
|
||||
der, err := base64.StdEncoding.DecodeString(encoded)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("keysigner: decode public key: %w", err)
|
||||
}
|
||||
return x509.ParsePKIXPublicKey(der)
|
||||
}
|
||||
|
||||
// base64Wrap PEM-wraps DER bytes at 64 columns.
|
||||
func base64Wrap(der []byte) string {
|
||||
enc := base64.StdEncoding.EncodeToString(der)
|
||||
var b strings.Builder
|
||||
for i := 0; i < len(enc); i += 64 {
|
||||
end := i + 64
|
||||
if end > len(enc) {
|
||||
end = len(enc)
|
||||
}
|
||||
b.WriteString(enc[i:end])
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func readKeyMetadata(label string) (*keyMetadata, error) {
|
||||
path, err := keyMetadataPath(label)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data, err := vfs.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err // preserves os.ErrNotExist for EnsureKey
|
||||
}
|
||||
var md keyMetadata
|
||||
if err := json.Unmarshal(data, &md); err != nil {
|
||||
return nil, fmt.Errorf("keysigner: parse key metadata: %w", err)
|
||||
}
|
||||
return &md, nil
|
||||
}
|
||||
|
||||
func writeKeyMetadata(path string, md keyMetadata) error {
|
||||
if err := vfs.MkdirAll(filepath.Dir(path), 0700); err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := json.MarshalIndent(md, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return vfs.WriteFile(path, data, 0600)
|
||||
}
|
||||
|
||||
func ensureKeychain() (string, error) {
|
||||
keychainPath, err := keychainFilePath()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
password, err := keychainPassword()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := vfs.Stat(keychainPath); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return "", fmt.Errorf("keysigner: stat keychain: %w", err)
|
||||
}
|
||||
if err := vfs.MkdirAll(filepath.Dir(keychainPath), 0700); err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, args := range [][]string{
|
||||
{"create-keychain", "-p", password, keychainPath},
|
||||
{"set-keychain-settings", keychainPath},
|
||||
{"unlock-keychain", "-p", password, keychainPath},
|
||||
} {
|
||||
if out, err := exec.Command(securityBin, args...).CombinedOutput(); err != nil {
|
||||
return "", fmt.Errorf("keysigner: security %s: %w: %s", args[0], err, summarizeCmdOutput(out))
|
||||
}
|
||||
}
|
||||
}
|
||||
return keychainPath, nil
|
||||
}
|
||||
|
||||
func keysignerDir() (string, error) {
|
||||
configDir, err := os.UserConfigDir()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("keysigner: resolve config dir: %w", err)
|
||||
}
|
||||
return filepath.Join(configDir, "lark-cli", "keysigner"), nil
|
||||
}
|
||||
|
||||
func keychainFilePath() (string, error) {
|
||||
dir, err := keysignerDir()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Join(dir, "lark-cli.keychain"), nil
|
||||
}
|
||||
|
||||
func keychainPassword() (string, error) {
|
||||
dir, err := keysignerDir()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
path := filepath.Join(dir, "keychain.pass")
|
||||
if data, err := vfs.ReadFile(path); err == nil {
|
||||
if pw := strings.TrimSpace(string(data)); pw != "" {
|
||||
return pw, nil
|
||||
}
|
||||
return "", fmt.Errorf("keysigner: empty keychain password")
|
||||
} else if !os.IsNotExist(err) {
|
||||
return "", err
|
||||
}
|
||||
buf := make([]byte, 32)
|
||||
if _, err := rand.Read(buf); err != nil {
|
||||
return "", err
|
||||
}
|
||||
pw := hex.EncodeToString(buf)
|
||||
if err := vfs.MkdirAll(filepath.Dir(path), 0700); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := vfs.WriteFile(path, []byte(pw+"\n"), 0600); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return pw, nil
|
||||
}
|
||||
|
||||
func keyMetadataPath(label string) (string, error) {
|
||||
dir, err := keysignerDir()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
id := sha256.Sum256([]byte(label))
|
||||
return filepath.Join(dir, "keys", hex.EncodeToString(id[:])+".json"), nil
|
||||
}
|
||||
|
||||
// summarizeCmdOutput bounds external command output before it is embedded in
|
||||
// an error: first line only, capped at 200 chars.
|
||||
func summarizeCmdOutput(out []byte) string {
|
||||
s := strings.TrimSpace(string(out))
|
||||
if i := strings.IndexByte(s, '\n'); i >= 0 {
|
||||
s = strings.TrimSpace(s[:i])
|
||||
}
|
||||
const maxLen = 200
|
||||
if len(s) > maxLen {
|
||||
s = s[:maxLen] + "..."
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func keychainError(operation string, status int) error {
|
||||
switch status {
|
||||
case -25299:
|
||||
return fmt.Errorf("keysigner: %s: key already exists", operation)
|
||||
case -25300:
|
||||
return fmt.Errorf("keysigner: %s: key not found", operation)
|
||||
case -2:
|
||||
return fmt.Errorf("keysigner: %s: allocation failed", operation)
|
||||
default:
|
||||
return fmt.Errorf("keysigner: %s: Security framework status %d", operation, status)
|
||||
}
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
//go:build darwin
|
||||
|
||||
// Copyright (c) 2026 Lark Technologies Pte. Ltd.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package keysigner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto"
|
||||
"crypto/rsa"
|
||||
"crypto/sha256"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestKeychainSignerRegistered confirms the keychain_signer build self-registers
|
||||
// (init → Register), so keysigner.Active() is non-nil. No keychain access.
|
||||
func TestKeychainSignerRegistered(t *testing.T) {
|
||||
if _, ok := Active().(keychainSigner); !ok {
|
||||
t.Fatalf("Active() = %T, want keychainSigner (keychain_signer build must self-register)", Active())
|
||||
}
|
||||
}
|
||||
|
||||
// TestKeychainSignerRoundTrip creates a real non-extractable RSA key, signs, and
|
||||
// verifies RS256 against the returned public key. Gated by LARK_KEYCHAIN_IT
|
||||
// because it mutates the dedicated lark-cli keychain store. The signer is now
|
||||
// cgo-free (purego runtime FFI), so it runs with CGO_ENABLED=0. Run with:
|
||||
//
|
||||
// LARK_KEYCHAIN_IT=1 go test -run RoundTrip ./internal/keysigner/
|
||||
func TestKeychainSignerRoundTrip(t *testing.T) {
|
||||
if os.Getenv("LARK_KEYCHAIN_IT") == "" {
|
||||
t.Skip("set LARK_KEYCHAIN_IT=1 to run (mutates the macOS keychain)")
|
||||
}
|
||||
s := keychainSigner{}
|
||||
ref := KeyRef{Label: "lark-cli-keychain-it"}
|
||||
|
||||
pub, err := s.EnsureKey(context.Background(), ref)
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureKey: %v", err)
|
||||
}
|
||||
rsaPub, ok := pub.(*rsa.PublicKey)
|
||||
if !ok {
|
||||
t.Fatalf("public key = %T, want *rsa.PublicKey", pub)
|
||||
}
|
||||
if alg, err := AlgForKey(pub); err != nil || alg != AlgRS256 {
|
||||
t.Fatalf("AlgForKey = %q, %v; want RS256", alg, err)
|
||||
}
|
||||
|
||||
input := []byte("header.payload")
|
||||
sig, alg, err := s.Sign(context.Background(), ref, input)
|
||||
if err != nil {
|
||||
t.Fatalf("Sign: %v", err)
|
||||
}
|
||||
if alg != AlgRS256 {
|
||||
t.Errorf("Sign alg = %q, want RS256", alg)
|
||||
}
|
||||
h := sha256.Sum256(input)
|
||||
if err := rsa.VerifyPKCS1v15(rsaPub, crypto.SHA256, h[:], sig); err != nil {
|
||||
t.Errorf("RS256 signature did not verify: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,135 +0,0 @@
|
||||
//go:build linux || (windows && amd64)
|
||||
|
||||
// Copyright (c) 2026 Lark Technologies Pte. Ltd.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
// TPM 2.0 signer (compiled into every linux and windows/amd64 build, no build
|
||||
// tag required), backed by github.com/facebookincubator/sks.
|
||||
//
|
||||
// sks holds a non-exportable ECDSA P-256 key in the platform TPM and signs
|
||||
// SHA-256 digests. On Linux it talks to /dev/tpmrm0; on Windows it uses the
|
||||
// Microsoft Platform Crypto Provider (CNG). Both backends return an ASN.1 DER
|
||||
// ECDSA signature, which we convert to the fixed-width r||s form JWS requires for
|
||||
// ES256 (see ecdsaDERToJOSE). One key is created on the first private_key_jwt
|
||||
// registration (DefaultKeyLabel) and reused for subsequent app registrations and
|
||||
// every client_assertion on the same device.
|
||||
//
|
||||
// Excluded from windows/arm64: the sks Windows dependency stack (go-ole) has no
|
||||
// arm64 VARIANT and fails to compile, so windows/arm64 falls back to
|
||||
// client_secret only (keysigner.Active() is nil). On darwin the keychain signer
|
||||
// is used instead. CGO is never required.
|
||||
package keysigner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto"
|
||||
"crypto/ecdsa"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/facebookincubator/flog"
|
||||
"github.com/facebookincubator/sks"
|
||||
)
|
||||
|
||||
// p256ByteLen is the P-256 coordinate width. sks regular keys are always ECDSA
|
||||
// P-256, so ES256 signatures are 2*p256ByteLen bytes of r||s.
|
||||
const p256ByteLen = 32
|
||||
|
||||
// keyTag is the sks key tag. Both the Linux and Windows sks backends address
|
||||
// keys by label and ignore the tag, but the macOS backend uses it, so we set a
|
||||
// stable namespaced value for forward compatibility.
|
||||
const keyTag = "com.larksuite.cli"
|
||||
|
||||
// sksSigner implements Signer (and HardwareProber) using a non-exportable
|
||||
// TPM 2.0 ECDSA key via sks.
|
||||
type sksSigner struct{}
|
||||
|
||||
func init() {
|
||||
Register(sksSigner{})
|
||||
// This sks version logs verbose TPM-operation chatter to stderr via flog (a
|
||||
// glog fork it owns exclusively) — e.g. "Loaded TPM device", "Found handle
|
||||
// for key" on every sign. The CLI does not use flog, so silence it
|
||||
// process-wide here; real failures are returned as errors, never relied upon
|
||||
// from these logs. (Newer sks switched to slog, but that lands only on its
|
||||
// go-1.24 line, which we avoid to keep the module on go 1.23.)
|
||||
flog.SetOutput(io.Discard)
|
||||
}
|
||||
|
||||
// EnsureKey returns the public key for ref, creating the TPM key if absent.
|
||||
// sks.NewKey is find-or-create: it returns the existing key when one is present.
|
||||
func (sksSigner) EnsureKey(_ context.Context, ref KeyRef) (crypto.PublicKey, error) {
|
||||
key, err := sks.NewKey(ref.Label, keyTag, false, true, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("keysigner: ensure TPM key %q: %w", ref.Label, err)
|
||||
}
|
||||
defer key.Close()
|
||||
return ecdsaPublic(ref.Label, key.Public())
|
||||
}
|
||||
|
||||
// PublicKey returns the public key for ref without creating it. FromLabelTag does
|
||||
// not touch the TPM until Public() loads the sealed key; a missing key yields a
|
||||
// nil public key, which we surface as an error — at runtime the key MUST already
|
||||
// exist (it was bound to the app at registration), so we never silently mint a
|
||||
// new, unbound one here.
|
||||
func (sksSigner) PublicKey(_ context.Context, ref KeyRef) (crypto.PublicKey, error) {
|
||||
pub := sks.FromLabelTag(ref.Label).Public()
|
||||
if pub == nil {
|
||||
return nil, fmt.Errorf("keysigner: TPM key %q not found", ref.Label)
|
||||
}
|
||||
return ecdsaPublic(ref.Label, pub)
|
||||
}
|
||||
|
||||
// Sign signs signingInput with the TPM key and returns a JOSE-format ES256
|
||||
// signature (fixed-width r||s) plus its alg.
|
||||
func (sksSigner) Sign(_ context.Context, ref KeyRef, signingInput []byte) ([]byte, string, error) {
|
||||
key, err := sks.NewKey(ref.Label, keyTag, false, true, nil)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("keysigner: load TPM key %q: %w", ref.Label, err)
|
||||
}
|
||||
defer key.Close()
|
||||
|
||||
// ES256 signs the SHA-256 digest of the JWS signing input.
|
||||
digest := sha256.Sum256(signingInput)
|
||||
der, err := key.Sign(nil, digest[:], crypto.SHA256)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("keysigner: TPM sign with key %q: %w", ref.Label, err)
|
||||
}
|
||||
// Both sks backends emit ASN.1 DER; JWS ES256 requires fixed-width r||s
|
||||
// (RFC 7518 §3.4).
|
||||
rs, err := ecdsaDERToJOSE(der, p256ByteLen)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return rs, AlgES256, nil
|
||||
}
|
||||
|
||||
// ProbeHardware reports on the TPM backing this signer without touching any key.
|
||||
// A failure to reach the TPM (no device, permission denied, not TPM 2.0) is
|
||||
// reported as Available=false with Reason set, NOT as a Go error — the probe
|
||||
// still succeeded in determining that the TEE is currently unusable.
|
||||
func (sksSigner) ProbeHardware(_ context.Context) (HardwareInfo, error) {
|
||||
info := HardwareInfo{Backend: "tpm2"}
|
||||
data, err := sks.GetSecureHardwareVendorData()
|
||||
if err != nil {
|
||||
info.Reason = cleanProbeError(err)
|
||||
return info, nil
|
||||
}
|
||||
info.VendorName = data.VendorName
|
||||
info.VendorInfo = data.VendorInfo
|
||||
info.Available = data.IsTPM20CompliantDevice
|
||||
if !info.Available {
|
||||
info.Reason = "secure hardware is not a TPM 2.0 compliant device"
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// ecdsaPublic asserts that an sks public key is an ECDSA key (it always is for
|
||||
// regular sks keys) so the caller gets the concrete type AlgForKey/PublicKeyJWK expect.
|
||||
func ecdsaPublic(label string, pub crypto.PublicKey) (*ecdsa.PublicKey, error) {
|
||||
ecPub, ok := pub.(*ecdsa.PublicKey)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("keysigner: TPM key %q public is %T, want *ecdsa.PublicKey", label, pub)
|
||||
}
|
||||
return ecPub, nil
|
||||
}
|
||||
@@ -1,122 +0,0 @@
|
||||
//go:build linux || (windows && amd64)
|
||||
|
||||
// Copyright (c) 2026 Lark Technologies Pte. Ltd.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package keysigner
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/sha256"
|
||||
"io"
|
||||
"math/big"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/facebookincubator/flog"
|
||||
"github.com/facebookincubator/sks"
|
||||
)
|
||||
|
||||
// TestFlogSilenced verifies the mechanism init() relies on to keep sks's flog
|
||||
// TPM chatter off the CLI's stderr: SetOutput redirects flog, and io.Discard
|
||||
// drops it. Cleanup restores io.Discard so init()'s silencing holds for the
|
||||
// rest of the package's tests.
|
||||
func TestFlogSilenced(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
flog.SetOutput(&buf)
|
||||
t.Cleanup(func() { flog.SetOutput(io.Discard) })
|
||||
|
||||
flog.Info("captured-line")
|
||||
if !strings.Contains(buf.String(), "captured-line") {
|
||||
t.Fatalf("flog.SetOutput(buffer) did not capture output: %q", buf.String())
|
||||
}
|
||||
|
||||
flog.SetOutput(io.Discard)
|
||||
buf.Reset()
|
||||
flog.Info("should-be-discarded")
|
||||
if buf.Len() != 0 {
|
||||
t.Errorf("flog output not discarded: %q", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
// requireTEE skips the test unless the TPM is present and usable. On a Linux
|
||||
// machine with a TPM but a restrictive device owner (`/dev/tpmrm0` is `tss:tss`
|
||||
// by default), grant access with `sudo usermod -aG tss $USER` then re-login, or
|
||||
// run the test under sudo.
|
||||
func requireTEE(t *testing.T) {
|
||||
t.Helper()
|
||||
info, err := sksSigner{}.ProbeHardware(context.Background())
|
||||
if err != nil || !info.Available {
|
||||
reason := info.Reason
|
||||
if err != nil {
|
||||
reason = err.Error()
|
||||
}
|
||||
t.Skipf("TEE not available (%s)", reason)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSKSSignerRoundTrip exercises the full registration→assertion contract
|
||||
// against the real TPM: create the key, read it back without creating, derive
|
||||
// the JWS alg + JWK, sign, and verify the fixed-width r||s output.
|
||||
func TestSKSSignerRoundTrip(t *testing.T) {
|
||||
requireTEE(t)
|
||||
|
||||
var s sksSigner
|
||||
ctx := context.Background()
|
||||
ref := KeyRef{Label: "larksuite-cli-test"}
|
||||
|
||||
// Best-effort cleanup so the test key does not linger in the TPM-sealed store.
|
||||
t.Cleanup(func() {
|
||||
if k, err := sks.NewKey(ref.Label, keyTag, false, true, nil); err == nil {
|
||||
_ = k.Remove()
|
||||
_ = k.Close()
|
||||
}
|
||||
})
|
||||
|
||||
pub, err := s.EnsureKey(ctx, ref)
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureKey: %v", err)
|
||||
}
|
||||
ecPub, ok := pub.(*ecdsa.PublicKey)
|
||||
if !ok {
|
||||
t.Fatalf("EnsureKey returned %T, want *ecdsa.PublicKey", pub)
|
||||
}
|
||||
|
||||
// PublicKey (no-create) must return the same key bound at EnsureKey.
|
||||
pub2, err := s.PublicKey(ctx, ref)
|
||||
if err != nil {
|
||||
t.Fatalf("PublicKey: %v", err)
|
||||
}
|
||||
if !ecPub.Equal(pub2) {
|
||||
t.Fatal("PublicKey returned a different key than EnsureKey")
|
||||
}
|
||||
|
||||
// The JWT layer derives alg + JWK from the public key; both must work.
|
||||
if alg, err := AlgForKey(pub); err != nil || alg != AlgES256 {
|
||||
t.Fatalf("AlgForKey = %q, %v; want ES256", alg, err)
|
||||
}
|
||||
if _, err := PublicKeyJWK(pub); err != nil {
|
||||
t.Fatalf("PublicKeyJWK: %v", err)
|
||||
}
|
||||
|
||||
// Sign a representative JWS signing input and verify the converted r||s.
|
||||
input := []byte("eyJhbGciOiJFUzI1NiJ9.eyJzdWIiOiJjbGkifQ")
|
||||
sig, alg, err := s.Sign(ctx, ref, input)
|
||||
if err != nil {
|
||||
t.Fatalf("Sign: %v", err)
|
||||
}
|
||||
if alg != AlgES256 {
|
||||
t.Fatalf("Sign alg = %q, want ES256", alg)
|
||||
}
|
||||
if len(sig) != 2*p256ByteLen {
|
||||
t.Fatalf("len(sig) = %d, want %d (fixed-width r||s)", len(sig), 2*p256ByteLen)
|
||||
}
|
||||
digest := sha256.Sum256(input)
|
||||
r := new(big.Int).SetBytes(sig[:p256ByteLen])
|
||||
ss := new(big.Int).SetBytes(sig[p256ByteLen:])
|
||||
if !ecdsa.Verify(ecPub, digest[:], r, ss) {
|
||||
t.Fatal("TPM signature did not verify against the public key")
|
||||
}
|
||||
}
|
||||
@@ -113,7 +113,8 @@ type EnumOption struct {
|
||||
}
|
||||
|
||||
// EnumOptions returns the field's allowed values paired with their descriptions
|
||||
// — from enum, or from options when enum is absent — coerced to the canonical
|
||||
// — from enum (with descriptions backfilled from options when the field carries
|
||||
// both forms), or from options when enum is absent — coerced to the canonical
|
||||
// type and ordered: numeric and boolean values are sorted; string values keep
|
||||
// source order (which can encode priority). Uncoercible literals are dropped.
|
||||
// Returns nil when the field declares no enum constraint.
|
||||
@@ -122,9 +123,14 @@ func (f Field) EnumOptions() []EnumOption {
|
||||
var out []EnumOption
|
||||
switch {
|
||||
case len(f.Enum) > 0:
|
||||
// key by raw literal so enum "1" and option 1 align across JSON types
|
||||
desc := make(map[string]string, len(f.Options))
|
||||
for _, o := range f.Options {
|
||||
desc[fmt.Sprintf("%v", o.Value)] = o.Description
|
||||
}
|
||||
for _, e := range f.Enum {
|
||||
if v, ok := coerceLiteral(ct, e); ok {
|
||||
out = append(out, EnumOption{Value: v})
|
||||
out = append(out, EnumOption{Value: v, Description: desc[fmt.Sprintf("%v", e)]})
|
||||
}
|
||||
}
|
||||
case len(f.Options) > 0:
|
||||
|
||||
@@ -80,6 +80,39 @@ func TestField_EnumOptions(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestField_EnumOptions_BothEnumAndOptions(t *testing.T) {
|
||||
// enum is the value set; descriptions backfilled from options, empty where absent
|
||||
f := Field{Type: "string", Enum: []any{"1", "2", "3", "4", "6"}, Options: []Option{
|
||||
{Value: "1", Description: "from"},
|
||||
{Value: "2", Description: "to"},
|
||||
{Value: "6", Description: "subject"},
|
||||
}}
|
||||
want := []EnumOption{
|
||||
{Value: "1", Description: "from"},
|
||||
{Value: "2", Description: "to"},
|
||||
{Value: "3", Description: ""},
|
||||
{Value: "4", Description: ""},
|
||||
{Value: "6", Description: "subject"},
|
||||
}
|
||||
if got := f.EnumOptions(); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("EnumOptions(enum+options) = %+v, want %+v", got, want)
|
||||
}
|
||||
|
||||
// enum values stored as strings match option values stored as numbers
|
||||
fi := Field{Type: "integer", Enum: []any{"10", "2", "1"}, Options: []Option{
|
||||
{Value: 1, Description: "one"},
|
||||
{Value: 2, Description: "two"},
|
||||
}}
|
||||
wantI := []EnumOption{
|
||||
{Value: int64(1), Description: "one"},
|
||||
{Value: int64(2), Description: "two"},
|
||||
{Value: int64(10), Description: ""},
|
||||
}
|
||||
if got := fi.EnumOptions(); !reflect.DeepEqual(got, wantI) {
|
||||
t.Errorf("EnumOptions(integer enum+options) = %+v, want %+v", got, wantI)
|
||||
}
|
||||
}
|
||||
|
||||
func TestField_Enum_NumberAndBoolean(t *testing.T) {
|
||||
// number: string-stored floats coerced to float64 and numerically sorted
|
||||
if got := (Field{Type: "number", Enum: []any{"2.5", "1.5", "10"}}).EnumValues(); !reflect.DeepEqual(got, []any{1.5, 2.5, float64(10)}) {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user