mirror of
https://github.com/larksuite/cli.git
synced 2026-07-08 02:00:19 +08:00
(im) support im oapi range download large file (#283)
Add range download support for IM OAPI resources so lark-cli can reliably download large files. This improves stability for large payloads and network interruptions. Change-Id: I38e6f6f9cf8b8711dc40650d19c77503f4e44989
This commit is contained in:
committed by
GitHub
parent
74f7de386a
commit
06e7ae267c
@@ -6,6 +6,7 @@ package im
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -13,6 +14,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"unsafe"
|
||||
@@ -286,6 +288,46 @@ func TestDownloadIMResourceToPathSuccess(t *testing.T) {
|
||||
if gotHeaders.Get(cmdutil.HeaderExecutionId) != "exec-123" {
|
||||
t.Fatalf("%s = %q, want %q", cmdutil.HeaderExecutionId, gotHeaders.Get(cmdutil.HeaderExecutionId), "exec-123")
|
||||
}
|
||||
if gotHeaders.Get("Range") != fmt.Sprintf("bytes=0-%d", probeChunkSize-1) {
|
||||
t.Fatalf("Range header = %q, want %q", gotHeaders.Get("Range"), fmt.Sprintf("bytes=0-%d", probeChunkSize-1))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadIMResourceToPathImageUsesSingleRequestWithoutRange(t *testing.T) {
|
||||
var gotHeaders http.Header
|
||||
payload := []byte("image download")
|
||||
runtime := newBotShortcutRuntime(t, shortcutRoundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
switch {
|
||||
case strings.Contains(req.URL.Path, "/open-apis/im/v1/messages/om_img/resources/img_123"):
|
||||
gotHeaders = req.Header.Clone()
|
||||
return shortcutRawResponse(200, payload, http.Header{"Content-Type": []string{"image/png"}}), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected request: %s", req.URL.String())
|
||||
}
|
||||
}))
|
||||
|
||||
cmdutil.TestChdir(t, t.TempDir())
|
||||
|
||||
gotPath, size, err := downloadIMResourceToPath(context.Background(), runtime, "om_img", "img_123", "image", "image")
|
||||
if err != nil {
|
||||
t.Fatalf("downloadIMResourceToPath() error = %v", err)
|
||||
}
|
||||
if size != int64(len(payload)) {
|
||||
t.Fatalf("downloadIMResourceToPath() size = %d, want %d", size, len(payload))
|
||||
}
|
||||
if gotHeaders.Get("Range") != "" {
|
||||
t.Fatalf("Range header = %q, want empty", gotHeaders.Get("Range"))
|
||||
}
|
||||
if !strings.HasSuffix(gotPath, "image.png") {
|
||||
t.Fatalf("saved path = %q, want suffix %q", gotPath, "image.png")
|
||||
}
|
||||
data, err := os.ReadFile("image.png")
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile() error = %v", err)
|
||||
}
|
||||
if string(data) != string(payload) {
|
||||
t.Fatalf("downloaded payload = %q, want %q", string(data), string(payload))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadIMResourceToPathHTTPErrorBody(t *testing.T) {
|
||||
@@ -306,6 +348,348 @@ func TestDownloadIMResourceToPathHTTPErrorBody(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadIMResourceToPathRetriesNetworkError(t *testing.T) {
|
||||
attempts := 0
|
||||
payload := []byte("retry success")
|
||||
runtime := newBotShortcutRuntime(t, shortcutRoundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
switch {
|
||||
case strings.Contains(req.URL.Path, "tenant_access_token"):
|
||||
return shortcutJSONResponse(200, map[string]interface{}{
|
||||
"code": 0,
|
||||
"tenant_access_token": "tenant-token",
|
||||
"expire": 7200,
|
||||
}), nil
|
||||
case strings.Contains(req.URL.Path, "/open-apis/im/v1/messages/om_retry/resources/file_retry"):
|
||||
attempts++
|
||||
if attempts < 3 {
|
||||
return nil, fmt.Errorf("temporary network failure")
|
||||
}
|
||||
return shortcutRawResponse(200, payload, http.Header{"Content-Type": []string{"application/octet-stream"}}), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected request: %s", req.URL.String())
|
||||
}
|
||||
}))
|
||||
|
||||
cmdutil.TestChdir(t, t.TempDir())
|
||||
target := "out.bin"
|
||||
_, size, err := downloadIMResourceToPath(context.Background(), runtime, "om_retry", "file_retry", "file", target)
|
||||
if err != nil {
|
||||
t.Fatalf("downloadIMResourceToPath() error = %v", err)
|
||||
}
|
||||
if attempts != 3 {
|
||||
t.Fatalf("download attempts = %d, want 3", attempts)
|
||||
}
|
||||
if size != int64(len(payload)) {
|
||||
t.Fatalf("downloadIMResourceToPath() size = %d, want %d", size, len(payload))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadIMResourceToPathRetrySecondAttemptSuccess(t *testing.T) {
|
||||
attempts := 0
|
||||
payload := []byte("second retry success")
|
||||
runtime := newBotShortcutRuntime(t, shortcutRoundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
switch {
|
||||
case strings.Contains(req.URL.Path, "tenant_access_token"):
|
||||
return shortcutJSONResponse(200, map[string]interface{}{
|
||||
"code": 0,
|
||||
"tenant_access_token": "tenant-token",
|
||||
"expire": 7200,
|
||||
}), nil
|
||||
case strings.Contains(req.URL.Path, "/open-apis/im/v1/messages/om_retry2/resources/file_retry2"):
|
||||
attempts++
|
||||
if attempts < 2 {
|
||||
return nil, fmt.Errorf("temporary network failure")
|
||||
}
|
||||
return shortcutRawResponse(200, payload, http.Header{"Content-Type": []string{"application/octet-stream"}}), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected request: %s", req.URL.String())
|
||||
}
|
||||
}))
|
||||
|
||||
cmdutil.TestChdir(t, t.TempDir())
|
||||
target := "out.bin"
|
||||
_, size, err := downloadIMResourceToPath(context.Background(), runtime, "om_retry2", "file_retry2", "file", target)
|
||||
if err != nil {
|
||||
t.Fatalf("downloadIMResourceToPath() error = %v", err)
|
||||
}
|
||||
if attempts != 2 {
|
||||
t.Fatalf("download attempts = %d, want 2", attempts)
|
||||
}
|
||||
if size != int64(len(payload)) {
|
||||
t.Fatalf("downloadIMResourceToPath() size = %d, want %d", size, len(payload))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadIMResourceToPathRetryContextCanceled(t *testing.T) {
|
||||
attempts := 0
|
||||
runtime := newBotShortcutRuntime(t, shortcutRoundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
switch {
|
||||
case strings.Contains(req.URL.Path, "tenant_access_token"):
|
||||
return shortcutJSONResponse(200, map[string]interface{}{
|
||||
"code": 0,
|
||||
"tenant_access_token": "tenant-token",
|
||||
"expire": 7200,
|
||||
}), nil
|
||||
case strings.Contains(req.URL.Path, "/open-apis/im/v1/messages/om_cancel/resources/file_cancel"):
|
||||
attempts++
|
||||
return nil, fmt.Errorf("temporary network failure")
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected request: %s", req.URL.String())
|
||||
}
|
||||
}))
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
// Cancel context immediately to trigger context error on first retry
|
||||
cancel()
|
||||
|
||||
cmdutil.TestChdir(t, t.TempDir())
|
||||
target := "out.bin"
|
||||
_, _, err := downloadIMResourceToPath(ctx, runtime, "om_cancel", "file_cancel", "file", target)
|
||||
if err != context.Canceled {
|
||||
t.Fatalf("downloadIMResourceToPath() error = %v, want context.Canceled", err)
|
||||
}
|
||||
// First attempt is made, then retry checks ctx.Err() and returns
|
||||
if attempts != 1 {
|
||||
t.Fatalf("download attempts = %d, want 1", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadIMResourceToPathRangeDownload(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
payloadLen int64
|
||||
wantRanges []string
|
||||
}{
|
||||
{
|
||||
name: "single small chunk",
|
||||
payloadLen: 16,
|
||||
wantRanges: []string{"bytes=0-131071"},
|
||||
},
|
||||
{
|
||||
name: "exact probe chunk",
|
||||
payloadLen: probeChunkSize,
|
||||
wantRanges: []string{"bytes=0-131071"},
|
||||
},
|
||||
{
|
||||
name: "multiple chunks with tail",
|
||||
payloadLen: probeChunkSize + normalChunkSize + 1234,
|
||||
wantRanges: []string{
|
||||
"bytes=0-131071",
|
||||
fmt.Sprintf("bytes=%d-%d", probeChunkSize, probeChunkSize+normalChunkSize-1),
|
||||
fmt.Sprintf("bytes=%d-%d", probeChunkSize+normalChunkSize, probeChunkSize+normalChunkSize+1233),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "multiple chunks exact 8mb tail",
|
||||
payloadLen: probeChunkSize + 2*normalChunkSize,
|
||||
wantRanges: []string{
|
||||
"bytes=0-131071",
|
||||
fmt.Sprintf("bytes=%d-%d", probeChunkSize, probeChunkSize+normalChunkSize-1),
|
||||
fmt.Sprintf("bytes=%d-%d", probeChunkSize+normalChunkSize, probeChunkSize+2*normalChunkSize-1),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range cases {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
payload := bytes.Repeat([]byte("range-download-"), int(tt.payloadLen/15)+1)
|
||||
payload = payload[:tt.payloadLen]
|
||||
|
||||
var gotRanges []string
|
||||
runtime := newBotShortcutRuntime(t, shortcutRoundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
switch {
|
||||
case strings.Contains(req.URL.Path, "tenant_access_token"):
|
||||
return shortcutJSONResponse(200, map[string]interface{}{
|
||||
"code": 0,
|
||||
"tenant_access_token": "tenant-token",
|
||||
"expire": 7200,
|
||||
}), nil
|
||||
case strings.Contains(req.URL.Path, "/open-apis/im/v1/messages/om_range/resources/file_range"):
|
||||
rangeHeader := req.Header.Get("Range")
|
||||
gotRanges = append(gotRanges, rangeHeader)
|
||||
if req.Header.Get("Authorization") != "Bearer tenant-token" {
|
||||
return nil, fmt.Errorf("missing authorization header")
|
||||
}
|
||||
start, end, err := parseRangeHeader(rangeHeader, int64(len(payload)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return shortcutRawResponse(http.StatusPartialContent, payload[start:end+1], http.Header{
|
||||
"Content-Type": []string{"application/octet-stream"},
|
||||
"Content-Range": []string{fmt.Sprintf("bytes %d-%d/%d", start, end, len(payload))},
|
||||
}), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected request: %s", req.URL.String())
|
||||
}
|
||||
}))
|
||||
|
||||
cmdutil.TestChdir(t, t.TempDir())
|
||||
target := filepath.Join("nested", "resource.bin")
|
||||
_, size, err := downloadIMResourceToPath(context.Background(), runtime, "om_range", "file_range", "file", target)
|
||||
if err != nil {
|
||||
t.Fatalf("downloadIMResourceToPath() error = %v", err)
|
||||
}
|
||||
if size != int64(len(payload)) {
|
||||
t.Fatalf("downloadIMResourceToPath() size = %d, want %d", size, len(payload))
|
||||
}
|
||||
if !reflect.DeepEqual(gotRanges, tt.wantRanges) {
|
||||
t.Fatalf("Range requests = %#v, want %#v", gotRanges, tt.wantRanges)
|
||||
}
|
||||
|
||||
got, err := os.ReadFile(target)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile() error = %v", err)
|
||||
}
|
||||
if md5.Sum(got) != md5.Sum(payload) {
|
||||
t.Fatalf("downloaded payload MD5 = %x, want %x", md5.Sum(got), md5.Sum(payload))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadIMResourceToPathInvalidContentRange(t *testing.T) {
|
||||
runtime := newBotShortcutRuntime(t, shortcutRoundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
switch {
|
||||
case strings.Contains(req.URL.Path, "tenant_access_token"):
|
||||
return shortcutJSONResponse(200, map[string]interface{}{
|
||||
"code": 0,
|
||||
"tenant_access_token": "tenant-token",
|
||||
"expire": 7200,
|
||||
}), nil
|
||||
case strings.Contains(req.URL.Path, "/open-apis/im/v1/messages/om_bad/resources/file_bad"):
|
||||
return shortcutRawResponse(http.StatusPartialContent, []byte("bad"), http.Header{
|
||||
"Content-Type": []string{"application/octet-stream"},
|
||||
"Content-Range": []string{"bytes 0-2/not-a-number"},
|
||||
}), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected request: %s", req.URL.String())
|
||||
}
|
||||
}))
|
||||
|
||||
cmdutil.TestChdir(t, t.TempDir())
|
||||
_, _, err := downloadIMResourceToPath(context.Background(), runtime, "om_bad", "file_bad", "file", "out.bin")
|
||||
if err == nil || !strings.Contains(err.Error(), "invalid Content-Range header") {
|
||||
t.Fatalf("downloadIMResourceToPath() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadIMResourceToPathRangeChunkFailureCleansOutput(t *testing.T) {
|
||||
payload := bytes.Repeat([]byte("range-download-"), int((probeChunkSize+1024)/15)+1)
|
||||
payload = payload[:probeChunkSize+1024]
|
||||
|
||||
runtime := newBotShortcutRuntime(t, shortcutRoundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
switch {
|
||||
case strings.Contains(req.URL.Path, "/open-apis/im/v1/messages/om_miderr/resources/file_miderr"):
|
||||
rangeHeader := req.Header.Get("Range")
|
||||
if rangeHeader == fmt.Sprintf("bytes=0-%d", probeChunkSize-1) {
|
||||
return shortcutRawResponse(http.StatusPartialContent, payload[:probeChunkSize], http.Header{
|
||||
"Content-Type": []string{"application/octet-stream"},
|
||||
"Content-Range": []string{fmt.Sprintf("bytes 0-%d/%d", probeChunkSize-1, len(payload))},
|
||||
}), nil
|
||||
}
|
||||
return shortcutRawResponse(http.StatusInternalServerError, []byte("chunk failed"), http.Header{"Content-Type": []string{"text/plain"}}), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected request: %s", req.URL.String())
|
||||
}
|
||||
}))
|
||||
|
||||
cmdutil.TestChdir(t, t.TempDir())
|
||||
|
||||
target := "out.bin"
|
||||
_, _, err := downloadIMResourceToPath(context.Background(), runtime, "om_miderr", "file_miderr", "file", target)
|
||||
if err == nil || !strings.Contains(err.Error(), "HTTP 500: chunk failed") {
|
||||
t.Fatalf("downloadIMResourceToPath() error = %v", err)
|
||||
}
|
||||
if _, statErr := os.Stat(target); !os.IsNotExist(statErr) {
|
||||
t.Fatalf("output file exists after failed download, stat error = %v", statErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadIMResourceToPathRangeOverflowCleansOutput(t *testing.T) {
|
||||
payload := []byte("overflow-payload")
|
||||
runtime := newBotShortcutRuntime(t, shortcutRoundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
switch {
|
||||
case strings.Contains(req.URL.Path, "/open-apis/im/v1/messages/om_overflow/resources/file_overflow"):
|
||||
return shortcutRawResponse(http.StatusPartialContent, payload, http.Header{
|
||||
"Content-Type": []string{"application/octet-stream"},
|
||||
"Content-Range": []string{"bytes 0-3/4"},
|
||||
}), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected request: %s", req.URL.String())
|
||||
}
|
||||
}))
|
||||
|
||||
cmdutil.TestChdir(t, t.TempDir())
|
||||
|
||||
target := "out.bin"
|
||||
_, _, err := downloadIMResourceToPath(context.Background(), runtime, "om_overflow", "file_overflow", "file", target)
|
||||
if err == nil || !strings.Contains(err.Error(), "chunk overflow") {
|
||||
t.Fatalf("downloadIMResourceToPath() error = %v", err)
|
||||
}
|
||||
if _, statErr := os.Stat(target); !os.IsNotExist(statErr) {
|
||||
t.Fatalf("output file exists after overflow, stat error = %v", statErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadIMResourceToPathRangeShortChunkSizeMismatch(t *testing.T) {
|
||||
payload := bytes.Repeat([]byte("range-download-"), int((probeChunkSize+1024)/15)+1)
|
||||
payload = payload[:probeChunkSize+1024]
|
||||
|
||||
runtime := newBotShortcutRuntime(t, shortcutRoundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
switch {
|
||||
case strings.Contains(req.URL.Path, "/open-apis/im/v1/messages/om_short/resources/file_short"):
|
||||
rangeHeader := req.Header.Get("Range")
|
||||
start, end, err := parseRangeHeader(rangeHeader, int64(len(payload)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body := payload[start : end+1]
|
||||
if start == probeChunkSize {
|
||||
body = body[:len(body)-10]
|
||||
}
|
||||
return shortcutRawResponse(http.StatusPartialContent, body, http.Header{
|
||||
"Content-Type": []string{"application/octet-stream"},
|
||||
"Content-Range": []string{fmt.Sprintf("bytes %d-%d/%d", start, end, len(payload))},
|
||||
}), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected request: %s", req.URL.String())
|
||||
}
|
||||
}))
|
||||
|
||||
cmdutil.TestChdir(t, t.TempDir())
|
||||
|
||||
_, _, err := downloadIMResourceToPath(context.Background(), runtime, "om_short", "file_short", "file", "out.bin")
|
||||
if err == nil || !strings.Contains(err.Error(), "file size mismatch") {
|
||||
t.Fatalf("downloadIMResourceToPath() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func parseRangeHeader(header string, totalSize int64) (int64, int64, error) {
|
||||
if !strings.HasPrefix(header, "bytes=") {
|
||||
return 0, 0, fmt.Errorf("unexpected range header: %q", header)
|
||||
}
|
||||
parts := strings.SplitN(strings.TrimPrefix(header, "bytes="), "-", 2)
|
||||
if len(parts) != 2 {
|
||||
return 0, 0, fmt.Errorf("unexpected range header: %q", header)
|
||||
}
|
||||
|
||||
start, err := strconv.ParseInt(parts[0], 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("parse start: %w", err)
|
||||
}
|
||||
end, err := strconv.ParseInt(parts[1], 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("parse end: %w", err)
|
||||
}
|
||||
if start < 0 || end < start || start >= totalSize {
|
||||
return 0, 0, fmt.Errorf("invalid range bounds: %d-%d for size %d", start, end, totalSize)
|
||||
}
|
||||
if end >= totalSize {
|
||||
end = totalSize - 1
|
||||
}
|
||||
return start, end, nil
|
||||
}
|
||||
|
||||
func TestUploadImageToIMSuccess(t *testing.T) {
|
||||
var gotBody string
|
||||
runtime := newBotShortcutRuntime(t, shortcutRoundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
|
||||
@@ -599,6 +599,44 @@ func TestDownloadIMResourceToPathHTTPClientError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTotalSize(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
contentRange string
|
||||
want int64
|
||||
wantErr string
|
||||
}{
|
||||
{name: "normal", contentRange: "bytes 0-131071/104857600", want: 104857600},
|
||||
{name: "single probe chunk", contentRange: "bytes 0-131071/131072", want: 131072},
|
||||
{name: "single small chunk", contentRange: "bytes 0-15/16", want: 16},
|
||||
{name: "empty", contentRange: "", wantErr: "content-range is empty"},
|
||||
{name: "invalid prefix", contentRange: "items 0-15/16", wantErr: `unsupported content-range: "items 0-15/16"`},
|
||||
{name: "missing total", contentRange: "bytes 0-15/", wantErr: `unsupported content-range: "bytes 0-15/"`},
|
||||
{name: "wildcard", contentRange: "bytes */16", wantErr: `unsupported content-range: "bytes */16"`},
|
||||
{name: "unknown total size", contentRange: "bytes 0-99/*", wantErr: `unknown total size in content-range: "bytes 0-99/*"`},
|
||||
{name: "invalid total", contentRange: "bytes 0-15/not-a-number", wantErr: "parse total size:"},
|
||||
{name: "zero total size", contentRange: "bytes 0-0/0", wantErr: "invalid total size: 0"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := parseTotalSize(tt.contentRange)
|
||||
if tt.wantErr != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||
t.Fatalf("parseTotalSize() error = %v, want substring %q", err, tt.wantErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("parseTotalSize() unexpected error = %v", err)
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Fatalf("parseTotalSize() = %d, want %d", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestShortcuts(t *testing.T) {
|
||||
var commands []string
|
||||
for _, shortcut := range Shortcuts() {
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -67,6 +68,9 @@ var ImMessagesResourcesDownload = common.Shortcut{
|
||||
if err != nil {
|
||||
return output.ErrValidation("invalid output path: %s", err)
|
||||
}
|
||||
if _, err := runtime.ResolveSavePath(relPath); err != nil {
|
||||
return output.ErrValidation("unsafe output path: %s", err)
|
||||
}
|
||||
|
||||
finalPath, sizeBytes, err := downloadIMResourceToPath(ctx, runtime, messageId, fileKey, fileType, relPath)
|
||||
if err != nil {
|
||||
@@ -102,7 +106,13 @@ func normalizeDownloadOutputPath(fileKey, outputPath string) (string, error) {
|
||||
return outputPath, nil
|
||||
}
|
||||
|
||||
const defaultIMResourceDownloadTimeout = 120 * time.Second
|
||||
const (
|
||||
defaultIMResourceDownloadTimeout = 120 * time.Second
|
||||
probeChunkSize = int64(128 * 1024)
|
||||
normalChunkSize = int64(8 * 1024 * 1024)
|
||||
imDownloadRequestRetries = 2
|
||||
imDownloadRetryDelay = 300 * time.Millisecond
|
||||
)
|
||||
|
||||
var imMimeToExt = map[string]string{
|
||||
"image/png": ".png",
|
||||
@@ -135,10 +145,199 @@ var imMimeToExt = map[string]string{
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",
|
||||
}
|
||||
|
||||
func downloadIMResourceToPath(ctx context.Context, runtime *common.RuntimeContext, messageID, fileKey, fileType, safePath string) (string, int64, error) {
|
||||
type rangeChunkReader struct {
|
||||
ctx context.Context
|
||||
runtime *common.RuntimeContext
|
||||
messageID string
|
||||
fileKey string
|
||||
fileType string
|
||||
totalSize int64
|
||||
delivered int64
|
||||
current io.ReadCloser
|
||||
nextOffset int64
|
||||
}
|
||||
|
||||
func newRangeChunkReader(
|
||||
ctx context.Context,
|
||||
runtime *common.RuntimeContext,
|
||||
messageID, fileKey, fileType string,
|
||||
probeBody io.ReadCloser,
|
||||
totalSize int64,
|
||||
) *rangeChunkReader {
|
||||
return &rangeChunkReader{
|
||||
ctx: ctx,
|
||||
runtime: runtime,
|
||||
messageID: messageID,
|
||||
fileKey: fileKey,
|
||||
fileType: fileType,
|
||||
totalSize: totalSize,
|
||||
current: probeBody,
|
||||
nextOffset: probeChunkSize,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *rangeChunkReader) Read(p []byte) (int, error) {
|
||||
for {
|
||||
if r.current != nil {
|
||||
n, err := r.current.Read(p)
|
||||
r.delivered += int64(n)
|
||||
|
||||
if r.delivered > r.totalSize {
|
||||
if err == io.EOF {
|
||||
closeErr := r.current.Close()
|
||||
r.current = nil
|
||||
if closeErr != nil {
|
||||
return 0, closeErr
|
||||
}
|
||||
}
|
||||
return 0, output.ErrNetwork("chunk overflow: delivered %d, expected %d", r.delivered, r.totalSize)
|
||||
}
|
||||
|
||||
switch err {
|
||||
case nil:
|
||||
return n, nil
|
||||
case io.EOF:
|
||||
closeErr := r.current.Close()
|
||||
r.current = nil
|
||||
if closeErr != nil {
|
||||
return n, closeErr
|
||||
}
|
||||
if r.delivered == r.totalSize {
|
||||
if n > 0 {
|
||||
return n, nil
|
||||
}
|
||||
return 0, io.EOF
|
||||
}
|
||||
if n > 0 {
|
||||
return n, nil
|
||||
}
|
||||
default:
|
||||
return n, err
|
||||
}
|
||||
}
|
||||
|
||||
if r.nextOffset >= r.totalSize {
|
||||
if r.delivered == r.totalSize {
|
||||
return 0, io.EOF
|
||||
}
|
||||
return 0, output.ErrNetwork("file size mismatch: expected %d, got %d", r.totalSize, r.delivered)
|
||||
}
|
||||
|
||||
end := min(r.nextOffset+normalChunkSize-1, r.totalSize-1)
|
||||
resp, err := doIMResourceDownloadRequest(r.ctx, r.runtime, r.messageID, r.fileKey, r.fileType, map[string]string{
|
||||
"Range": fmt.Sprintf("bytes=%d-%d", r.nextOffset, end),
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if resp.StatusCode >= 400 {
|
||||
defer resp.Body.Close()
|
||||
return 0, downloadResponseError(resp)
|
||||
}
|
||||
if resp.StatusCode != http.StatusPartialContent {
|
||||
resp.Body.Close()
|
||||
return 0, output.ErrNetwork("unexpected status code: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
r.current = resp.Body
|
||||
r.nextOffset = end + 1
|
||||
}
|
||||
}
|
||||
|
||||
func (r *rangeChunkReader) Close() error {
|
||||
if r.current == nil {
|
||||
return nil
|
||||
}
|
||||
err := r.current.Close()
|
||||
r.current = nil
|
||||
return err
|
||||
}
|
||||
|
||||
func initialIMResourceDownloadHeaders(fileType string) map[string]string {
|
||||
if fileType != "file" {
|
||||
return nil
|
||||
}
|
||||
return map[string]string{
|
||||
"Range": fmt.Sprintf("bytes=0-%d", probeChunkSize-1),
|
||||
}
|
||||
}
|
||||
|
||||
func downloadIMResourceToPath(ctx context.Context, runtime *common.RuntimeContext, messageID, fileKey, fileType, outputPath string) (string, int64, error) {
|
||||
downloadResp, err := doIMResourceDownloadRequest(ctx, runtime, messageID, fileKey, fileType, initialIMResourceDownloadHeaders(fileType))
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
if downloadResp.StatusCode >= 400 {
|
||||
defer downloadResp.Body.Close()
|
||||
return "", 0, downloadResponseError(downloadResp)
|
||||
}
|
||||
|
||||
finalPath := resolveIMResourceDownloadPath(outputPath, downloadResp.Header.Get("Content-Type"))
|
||||
|
||||
var (
|
||||
body io.ReadCloser
|
||||
sizeBytes int64
|
||||
)
|
||||
switch downloadResp.StatusCode {
|
||||
case http.StatusPartialContent:
|
||||
totalSize, err := parseTotalSize(downloadResp.Header.Get("Content-Range"))
|
||||
if err != nil {
|
||||
downloadResp.Body.Close()
|
||||
return "", 0, output.ErrNetwork("invalid Content-Range header on range response: %s", err)
|
||||
}
|
||||
body = newRangeChunkReader(ctx, runtime, messageID, fileKey, fileType, downloadResp.Body, totalSize)
|
||||
sizeBytes = totalSize
|
||||
|
||||
case http.StatusOK:
|
||||
body = downloadResp.Body
|
||||
sizeBytes = downloadResp.ContentLength
|
||||
|
||||
default:
|
||||
downloadResp.Body.Close()
|
||||
return "", 0, output.ErrNetwork("unexpected status code: %d", downloadResp.StatusCode)
|
||||
}
|
||||
defer body.Close()
|
||||
|
||||
result, err := runtime.FileIO().Save(finalPath, fileio.SaveOptions{
|
||||
ContentType: downloadResp.Header.Get("Content-Type"),
|
||||
ContentLength: sizeBytes,
|
||||
}, body)
|
||||
if err != nil {
|
||||
return "", 0, common.WrapSaveErrorByCategory(err, "api_error")
|
||||
}
|
||||
if sizeBytes >= 0 && result.Size() != sizeBytes {
|
||||
return "", 0, output.ErrNetwork("file size mismatch: expected %d, got %d", sizeBytes, result.Size())
|
||||
}
|
||||
savedPath, resolveErr := runtime.ResolveSavePath(finalPath)
|
||||
if resolveErr != nil || savedPath == "" {
|
||||
savedPath = finalPath
|
||||
}
|
||||
return savedPath, result.Size(), nil
|
||||
}
|
||||
|
||||
func resolveIMResourceDownloadPath(safePath, contentType string) string {
|
||||
if filepath.Ext(safePath) != "" {
|
||||
return safePath
|
||||
}
|
||||
mimeType := strings.Split(contentType, ";")[0]
|
||||
mimeType = strings.TrimSpace(mimeType)
|
||||
if ext, ok := imMimeToExt[mimeType]; ok {
|
||||
return safePath + ext
|
||||
}
|
||||
return safePath
|
||||
}
|
||||
|
||||
func doIMResourceDownloadRequest(ctx context.Context, runtime *common.RuntimeContext, messageID, fileKey, fileType string, headers map[string]string) (*http.Response, error) {
|
||||
query := larkcore.QueryParams{}
|
||||
query.Set("type", fileType)
|
||||
downloadResp, err := runtime.DoAPIStream(ctx, &larkcore.ApiReq{
|
||||
|
||||
headerValues := make(http.Header, len(headers))
|
||||
for key, value := range headers {
|
||||
headerValues.Set(key, value)
|
||||
}
|
||||
|
||||
req := &larkcore.ApiReq{
|
||||
HttpMethod: http.MethodGet,
|
||||
ApiPath: "/open-apis/im/v1/messages/:message_id/resources/:file_key",
|
||||
PathParams: larkcore.PathParams{
|
||||
@@ -146,44 +345,73 @@ func downloadIMResourceToPath(ctx context.Context, runtime *common.RuntimeContex
|
||||
"file_key": fileKey,
|
||||
},
|
||||
QueryParams: query,
|
||||
}, client.WithTimeout(defaultIMResourceDownloadTimeout))
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
defer downloadResp.Body.Close()
|
||||
|
||||
if downloadResp.StatusCode >= 400 {
|
||||
body, _ := io.ReadAll(io.LimitReader(downloadResp.Body, 4096))
|
||||
if len(body) > 0 {
|
||||
return "", 0, output.ErrNetwork("download failed: HTTP %d: %s", downloadResp.StatusCode, strings.TrimSpace(string(body)))
|
||||
var lastErr error
|
||||
for attempt := 0; attempt <= imDownloadRequestRetries; attempt++ {
|
||||
resp, err := runtime.DoAPIStream(ctx, req, client.WithTimeout(defaultIMResourceDownloadTimeout), client.WithHeaders(headerValues))
|
||||
if err == nil {
|
||||
return resp, nil
|
||||
}
|
||||
return "", 0, output.ErrNetwork("download failed: HTTP %d", downloadResp.StatusCode)
|
||||
}
|
||||
|
||||
// Auto-detect extension from Content-Type if missing
|
||||
finalPath := safePath
|
||||
if filepath.Ext(safePath) == "" {
|
||||
contentType := downloadResp.Header.Get("Content-Type")
|
||||
mimeType := strings.Split(contentType, ";")[0]
|
||||
mimeType = strings.TrimSpace(mimeType)
|
||||
if ext, ok := imMimeToExt[mimeType]; ok {
|
||||
finalPath = safePath + ext
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
lastErr = err
|
||||
if attempt == imDownloadRequestRetries {
|
||||
break
|
||||
}
|
||||
sleepIMDownloadRetry(ctx, attempt)
|
||||
}
|
||||
|
||||
result, err := runtime.FileIO().Save(finalPath, fileio.SaveOptions{
|
||||
ContentType: downloadResp.Header.Get("Content-Type"),
|
||||
ContentLength: downloadResp.ContentLength,
|
||||
}, downloadResp.Body)
|
||||
if err != nil {
|
||||
return "", 0, output.Errorf(output.ExitInternal, "api_error", "%s",
|
||||
common.WrapSaveError(err, "unsafe output path", "cannot create parent directory", "cannot create file"))
|
||||
if lastErr != nil {
|
||||
return nil, lastErr
|
||||
}
|
||||
savedPath, resolveErr := runtime.ResolveSavePath(finalPath)
|
||||
if resolveErr != nil {
|
||||
// Save succeeded — file is on disk. Fall back to the relative path
|
||||
// rather than returning an error for a successfully written file.
|
||||
savedPath = finalPath
|
||||
}
|
||||
return savedPath, result.Size(), nil
|
||||
return nil, output.ErrNetwork("download request failed")
|
||||
}
|
||||
|
||||
func sleepIMDownloadRetry(ctx context.Context, attempt int) {
|
||||
delay := imDownloadRetryDelay * (1 << uint(attempt))
|
||||
timer := time.NewTimer(delay)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
|
||||
func downloadResponseError(resp *http.Response) error {
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||||
if len(body) > 0 {
|
||||
return output.ErrNetwork("download failed: HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
|
||||
}
|
||||
return output.ErrNetwork("download failed: HTTP %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
func parseTotalSize(contentRange string) (int64, error) {
|
||||
contentRange = strings.TrimSpace(contentRange)
|
||||
if contentRange == "" {
|
||||
return 0, fmt.Errorf("content-range is empty")
|
||||
}
|
||||
if !strings.HasPrefix(contentRange, "bytes ") {
|
||||
return 0, fmt.Errorf("unsupported content-range: %q", contentRange)
|
||||
}
|
||||
|
||||
parts := strings.SplitN(strings.TrimPrefix(contentRange, "bytes "), "/", 2)
|
||||
if len(parts) != 2 || parts[1] == "" {
|
||||
return 0, fmt.Errorf("unsupported content-range: %q", contentRange)
|
||||
}
|
||||
if parts[0] == "*" {
|
||||
return 0, fmt.Errorf("unsupported content-range: %q", contentRange)
|
||||
}
|
||||
if parts[1] == "*" {
|
||||
return 0, fmt.Errorf("unknown total size in content-range: %q", contentRange)
|
||||
}
|
||||
|
||||
totalSize, err := strconv.ParseInt(parts[1], 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("parse total size: %w", err)
|
||||
}
|
||||
if totalSize <= 0 {
|
||||
return 0, fmt.Errorf("invalid total size: %d", totalSize)
|
||||
}
|
||||
return totalSize, nil
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
name: lark-im
|
||||
version: 1.0.0
|
||||
description: "飞书即时通讯:收发消息和管理群聊。发送和回复消息、搜索聊天记录、管理群聊成员、上传下载图片和文件、管理表情回复。当用户需要发消息、查看或搜索聊天记录、下载聊天中的文件、查看群成员时使用。"
|
||||
description: "飞书即时通讯:收发消息和管理群聊。发送和回复消息、搜索聊天记录、管理群聊成员、上传下载图片和文件(支持大文件分片下载)、管理表情回复。当用户需要发消息、查看或搜索聊天记录、下载聊天中的文件、查看群成员时使用。"
|
||||
metadata:
|
||||
requires:
|
||||
bins: ["lark-cli"]
|
||||
@@ -62,7 +62,7 @@ Shortcut 是对常用操作的高级封装(`lark-cli im +<verb> [flags]`)。
|
||||
| [`+chat-update`](references/lark-im-chat-update.md) | Update group chat name or description; user/bot; updates a chat's name or description |
|
||||
| [`+messages-mget`](references/lark-im-messages-mget.md) | Batch get messages by IDs; user/bot; fetches up to 50 om_ message IDs, formats sender names, expands thread replies |
|
||||
| [`+messages-reply`](references/lark-im-messages-reply.md) | Reply to a message (supports thread replies); user/bot; supports text/markdown/post/media replies, reply-in-thread, idempotency key |
|
||||
| [`+messages-resources-download`](references/lark-im-messages-resources-download.md) | Download images/files from a message; user/bot; downloads image/file resources by message-id and file-key to a safe relative output path |
|
||||
| [`+messages-resources-download`](references/lark-im-messages-resources-download.md) | Download images/files from a message; user/bot; supports automatic chunked download for large files (8MB chunks), auto-detects file extension from Content-Type |
|
||||
| [`+messages-search`](references/lark-im-messages-search.md) | Search messages across chats (supports keyword, sender, time range filters) with user identity; user-only; filters by chat/sender/attachment/time, supports auto-pagination via `--page-all` / `--page-limit`, enriches results via batched mget and chats batch_query |
|
||||
| [`+messages-send`](references/lark-im-messages-send.md) | Send a message to a chat or direct message; user/bot; sends to chat-id or user-id with text/markdown/post/media, supports idempotency key |
|
||||
| [`+threads-messages-list`](references/lark-im-threads-messages-list.md) | List messages in a thread; user/bot; accepts om_/omt_ input, resolves message IDs to thread_id, supports sort/pagination |
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
> **Prerequisite:** Read [`../lark-shared/SKILL.md`](../../lark-shared/SKILL.md) first to understand authentication, global parameters, and safety rules.
|
||||
|
||||
Download image or file resources from a message. Resources are identified by the combination of `message_id` + `file_key`, both of which come directly from message content returned by `im +chat-messages-list`.
|
||||
Download image or file resources from a message. Supports **automatic chunked download for large files** using HTTP Range requests. Resources are identified by the combination of `message_id` + `file_key`, both of which come directly from message content returned by `im +chat-messages-list`.
|
||||
|
||||
> **Note:** read-only message commands render resource keys in message content, but they do not download binaries automatically. Use this command whenever you need to fetch the actual image/file bytes or save them to a specific path.
|
||||
|
||||
@@ -34,10 +34,26 @@ lark-cli im +messages-resources-download --message-id om_xxx --file-key img_v3_x
|
||||
| `--message-id <id>` | Yes | Message ID (`om_xxx` format) |
|
||||
| `--file-key <key>` | Yes | Resource key (`img_xxx` or `file_xxx`) |
|
||||
| `--type <type>` | Yes | Resource type: `image` or `file` |
|
||||
| `--output <path>` | No | Output path (relative paths only; `..` traversal is not allowed; defaults to `file_key` as the file name) |
|
||||
| `--output <path>` | No | Output path (relative paths only; `..` traversal is not allowed; defaults to `file_key` as the file name). File extension is automatically added based on Content-Type if not provided |
|
||||
| `--as <identity>` | No | Identity type: `user` (default) or `bot` |
|
||||
| `--dry-run` | No | Print the request only, do not execute it |
|
||||
|
||||
## Large File Download (Auto Chunking)
|
||||
|
||||
When downloading large files, the command automatically uses **HTTP Range requests** for reliable chunked downloading:
|
||||
|
||||
| Behavior | Details |
|
||||
|----------|---------|
|
||||
| Probe chunk | First 128 KB to detect file size and Content-Type |
|
||||
| Chunk size | 8 MB per subsequent request |
|
||||
| Workers | Single-threaded sequential download (ensures reliability) |
|
||||
| Retries | Up to 2 retries for transient request failures, with exponential backoff |
|
||||
|
||||
**Benefits:**
|
||||
- Reduces the impact of transient request failures during large downloads
|
||||
- Automatically detects and appends correct file extension from Content-Type
|
||||
- Validates file size integrity after download completion
|
||||
|
||||
## `file_key` Sources
|
||||
|
||||
Different resource markers in message content correspond to different `file_key` and `type` values:
|
||||
@@ -69,7 +85,8 @@ lark-cli im +messages-resources-download --message-id om_xxx --file-key img_v3_x
|
||||
| Download failed | `file_key` does not match the `message_id` | Make sure the `file_key` came from that message's content |
|
||||
| Hit error code 234002 or 14005 | No permission, **not** missing API scope | no access to this chat or file was deleted — do not retry, return the error to the user |
|
||||
| Permission denied | `im:message:readonly` is not authorized | Run `auth login --scope "im:message:readonly"` |
|
||||
| File too large | Over the 100 MB limit | This is a Feishu API limitation and cannot be bypassed with this endpoint |
|
||||
| File size mismatch | Chunked download integrity check failed | Network instability during download; retry the command |
|
||||
| Content-Range error | Server returned invalid range header | Transient API issue; retry the command |
|
||||
|
||||
## References
|
||||
|
||||
|
||||
Reference in New Issue
Block a user