diff --git a/go.mod b/go.mod index 8c76bd7d7..8f6dedef4 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,8 @@ require ( gopkg.in/yaml.v3 v3.0.1 ) +require github.com/apache/arrow/go/v17 v17.0.0 + require ( github.com/atotto/clipboard v0.1.4 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect @@ -42,13 +44,17 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/goccy/go-json v0.10.3 // indirect github.com/godbus/dbus/v5 v5.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/flatbuffers v24.3.25+incompatible // indirect github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/itchyny/timefmt-go v0.1.6 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect @@ -57,10 +63,16 @@ require ( github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/smarty/assertions v1.15.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + github.com/zeebo/xxh3 v1.0.2 // indirect + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect + golang.org/x/mod v0.18.0 // indirect + golang.org/x/tools v0.22.0 // indirect + golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect ) diff --git a/go.sum b/go.sum index 0e68ff05a..e5ba69b3c 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/apache/arrow/go/v17 v17.0.0 h1:RRR2bdqKcdbss9Gxy2NS/hK8i4LDMh23L6BbkN5+F54= +github.com/apache/arrow/go/v17 v17.0.0/go.mod h1:jR7QHkODl15PfYyjM2nU+yTLScZ/qfj7OSUZmJ8putc= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= @@ -52,12 +54,16 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godbus/dbus/v5 v5.2.2 h1:TUR3TgtSVDmjiXOgAAyaZbYmIeP3DPkld3jgKGV8mXQ= github.com/godbus/dbus/v5 v5.2.2/go.mod h1:3AAv2+hPq5rdnr5txxxRwiGjPXamgoIHgz9FPBfOp3c= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI= +github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= @@ -74,11 +80,16 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/larksuite/oapi-sdk-go/v3 v3.5.4 h1:U2S9x9LrfH++ZqJ+YAiUlqzCWJmVXhFdS8Z7rIBH8H0= github.com/larksuite/oapi-sdk-go/v3 v3.5.4/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= @@ -97,6 +108,8 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -133,14 +146,20 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/zalando/go-keyring v0.2.8 h1:6sD/Ucpl7jNq10rM2pgqTs0sZ9V3qMrqfIIy5YPccHs= github.com/zalando/go-keyring v0.2.8/go.mod h1:tsMo+VpRq5NGyKfxoBVjCuMrG47yj8cmakZDO5QGii0= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -156,6 +175,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= @@ -169,10 +189,16 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ= +gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/shortcuts/sheets/data/flag-defs.json b/shortcuts/sheets/data/flag-defs.json index a83acfb9d..f975989e9 100644 --- a/shortcuts/sheets/data/flag-defs.json +++ b/shortcuts/sheets/data/flag-defs.json @@ -526,7 +526,7 @@ "kind": "own", "type": "string", "required": "optional", - "desc": "Typed table payload as JSON (same shape as `+table-put`): a top-level `sheets` array, each item `{name, start_cell?, mode?, header?, allow_overwrite?, columns:[\"colA\",\"colB\",...], data:[[...]], dtypes?:{colA:pandasDtype, ...}, formats?:{colA:numberFormat, ...}}`. Agents typically build it from a DataFrame via `{**json.loads(df.to_json(orient=\"split\")), \"dtypes\": df.dtypes.astype(str).to_dict()}`. Mutually exclusive with --values. Creates the workbook, then writes typed type-faithful data (dates land as real dates, numbers keep precision).", + "desc": "Typed table payload as JSON (same shape as `+table-put`): a top-level `sheets` array, each item `{name, start_cell?, mode?, header?, allow_overwrite?, columns:[\"colA\",\"colB\",...], data:[[...]], dtypes?:{colA:pandasDtype, ...}, formats?:{colA:numberFormat, ...}}`. Agents typically build it from a DataFrame via `{**json.loads(df.to_json(orient=\"split\")), \"dtypes\": df.dtypes.astype(str).to_dict()}`. Mutually exclusive with --values and --dataframe. Creates the workbook, then writes typed type-faithful data (dates land as real dates, numbers keep precision).", "input": [ "file", "stdin" @@ -543,6 +543,13 @@ "stdin" ] }, + { + "name": "dataframe", + "kind": "own", + "type": "string", + "required": "optional", + "desc": "Single-sheet typed table from one Arrow IPC file (Feather v2 — what `pandas.DataFrame.to_feather()` writes), mutually exclusive with --values and --sheets. Pass `@` for a file or `-` for binary stdin (same convention as other input flags). Arrow bytes are read raw — no TrimSpace / BOM strip — so the IPC magic survives intact (unlike text input flags). Column types come from the Arrow schema; per-column `number_format` may be set via Arrow field metadata. Creates the workbook and fills its default sheet (`Sheet1` — adopted in place, no empty Sheet1 left behind). For multi-sheet or non-default placement, use `--sheets` instead." + }, { "name": "dry-run", "kind": "system", @@ -1382,6 +1389,13 @@ "required": "optional", "desc": "Treat the first row as data instead of a header (columns get positional names col1, col2, ...)" }, + { + "name": "dataframe-out", + "kind": "own", + "type": "string", + "required": "optional", + "desc": "Write the typed table as one Arrow IPC file (Feather v2) instead of the default JSON. Pass `@` for a file or `-` for binary stdout (same convention as other binary I/O flags). Mirror of the input-side `--dataframe` on `+table-put` / `+workbook-create` — pandas users round-trip via `df = pd.read_feather(\"x.arrow\")` or `pd.read_feather(io.BytesIO(stdout))`. Single-sheet only: requires `--sheet-id` or `--sheet-name`; whole-workbook reads keep the default JSON path. Column types come from the typed read-back (string/number/date/bool); per-column `number_format` is preserved as Arrow field metadata so the Arrow file can round-trip straight back through `+table-put --dataframe`." + }, { "name": "dry-run", "kind": "system", @@ -2064,13 +2078,20 @@ "name": "sheets", "kind": "own", "type": "string", - "required": "required", - "desc": "Typed table payload (pandas-DataFrame-shaped) as JSON: a top-level `sheets` array, each item `{name, start_cell?, mode?, header?, allow_overwrite?, columns:[\"colA\",\"colB\",...], data:[[...]], dtypes?:{colA:pandasDtype, ...}, formats?:{colA:numberFormat, ...}}`. Agents typically build it with `{**json.loads(df.to_json(orient=\"split\")), \"dtypes\": df.dtypes.astype(str).to_dict()}`. `dtypes` values are pandas dtype strings (`int64`, `float64`, `Int64`, `bool`, `boolean`, `datetime64[ns]`, `object`, ...); the writer maps them to internal string/number/date/bool — omit `dtypes` and a column writes as text (good for raw CSV-shaped data). `formats[col]` is an Excel number_format string (e.g. `#,##0.00`, `0.0%`, `yyyy-mm`); when absent, date columns default to `yyyy-mm-dd` and string columns to text format (`@`).", + "required": "xor", + "desc": "Typed table payload (pandas-DataFrame-shaped) as JSON, XOR with `--dataframe`: a top-level `sheets` array, each item `{name, start_cell?, mode?, header?, allow_overwrite?, columns:[\"colA\",\"colB\",...], data:[[...]], dtypes?:{colA:pandasDtype, ...}, formats?:{colA:numberFormat, ...}}`. Agents typically build it with `{**json.loads(df.to_json(orient=\"split\")), \"dtypes\": df.dtypes.astype(str).to_dict()}`. `dtypes` values are pandas dtype strings (`int64`, `float64`, `Int64`, `bool`, `boolean`, `datetime64[ns]`, `object`, ...); the writer maps them to internal string/number/date/bool — omit `dtypes` and a column writes as text (good for raw CSV-shaped data). `formats[col]` is an Excel number_format string (e.g. `#,##0.00`, `0.0%`, `yyyy-mm`); when absent, date columns default to `yyyy-mm-dd` and string columns to text format (`@`).", "input": [ "file", "stdin" ] }, + { + "name": "dataframe", + "kind": "own", + "type": "string", + "required": "xor", + "desc": "Single-sheet typed table from one Arrow IPC file (a.k.a. Feather v2 — what `pandas.DataFrame.to_feather()` writes), XOR with `--sheets`. Pass `@` for a file or `-` for binary stdin (same convention as other input flags). Arrow bytes are read raw — no TrimSpace / BOM strip — so the IPC magic survives intact (unlike text input flags). Column types come from the Arrow schema (int*/uint*/float* → number, date32/date64/timestamp → date, utf8/large_utf8 → string, bool → bool); per-column `number_format` may be set via Arrow field metadata (`pa.field(\"price\", pa.float64(), metadata={b\"number_format\": b\"$#,##0.00\"})`). Writes the sheet at default placement: name `Sheet1` (created when absent), overwrite from A1 with header. For a different sheet name, anchor, mode, or to write multiple sheets, use `--sheets` instead." + }, { "name": "dry-run", "kind": "system", diff --git a/shortcuts/sheets/flag_defs_gen.go b/shortcuts/sheets/flag_defs_gen.go index 41358ca65..a7bbe4bb0 100644 --- a/shortcuts/sheets/flag_defs_gen.go +++ b/shortcuts/sheets/flag_defs_gen.go @@ -923,6 +923,7 @@ var flagDefs = map[string]commandDef{ {Name: "sheet-name", Kind: "own", Type: "string", Required: "optional", Desc: "Read only this sheet (by name); omit to read all sheets"}, {Name: "range", Kind: "own", Type: "string", Required: "optional", Desc: "A1 range to read; omit to read each sheet current region"}, {Name: "no-header", Kind: "own", Type: "bool", Required: "optional", Desc: "Treat the first row as data instead of a header (columns get positional names col1, col2, ...)"}, + {Name: "dataframe-out", Kind: "own", Type: "string", Required: "optional", Desc: "Write the typed table as one Arrow IPC file (Feather v2) instead of the default JSON. Pass `@` for a file or `-` for binary stdout (same convention as other binary I/O flags). Mirror of the input-side `--dataframe` on `+table-put` / `+workbook-create` — pandas users round-trip via `df = pd.read_feather(\"x.arrow\")` or `pd.read_feather(io.BytesIO(stdout))`. Single-sheet only: requires `--sheet-id` or `--sheet-name`; whole-workbook reads keep the default JSON path. Column types come from the typed read-back (string/number/date/bool); per-column `number_format` is preserved as Arrow field metadata so the Arrow file can round-trip straight back through `+table-put --dataframe`."}, {Name: "dry-run", Kind: "system", Type: "bool", Required: "optional"}, }, }, @@ -931,7 +932,8 @@ var flagDefs = map[string]commandDef{ Flags: []flagDef{ {Name: "url", Kind: "public", Type: "string", Required: "xor", Desc: "Spreadsheet URL to write into (XOR with `--spreadsheet-token`)"}, {Name: "spreadsheet-token", Kind: "public", Type: "string", Required: "xor", Desc: "Spreadsheet token to write into (XOR with `--url`)"}, - {Name: "sheets", Kind: "own", Type: "string", Required: "required", Desc: "Typed table payload (pandas-DataFrame-shaped) as JSON: a top-level `sheets` array, each item `{name, start_cell?, mode?, header?, allow_overwrite?, columns:[\"colA\",\"colB\",...], data:[[...]], dtypes?:{colA:pandasDtype, ...}, formats?:{colA:numberFormat, ...}}`. Agents typically build it with `{**json.loads(df.to_json(orient=\"split\")), \"dtypes\": df.dtypes.astype(str).to_dict()}`. `dtypes` values are pandas dtype strings (`int64`, `float64`, `Int64`, `bool`, `boolean`, `datetime64[ns]`, `object`, ...); the writer maps them to internal string/number/date/bool — omit `dtypes` and a column writes as text (good for raw CSV-shaped data). `formats[col]` is an Excel number_format string (e.g. `#,##0.00`, `0.0%`, `yyyy-mm`); when absent, date columns default to `yyyy-mm-dd` and string columns to text format (`@`).", Input: []string{"file", "stdin"}}, + {Name: "sheets", Kind: "own", Type: "string", Required: "xor", Desc: "Typed table payload (pandas-DataFrame-shaped) as JSON, XOR with `--dataframe`: a top-level `sheets` array, each item `{name, start_cell?, mode?, header?, allow_overwrite?, columns:[\"colA\",\"colB\",...], data:[[...]], dtypes?:{colA:pandasDtype, ...}, formats?:{colA:numberFormat, ...}}`. Agents typically build it with `{**json.loads(df.to_json(orient=\"split\")), \"dtypes\": df.dtypes.astype(str).to_dict()}`. `dtypes` values are pandas dtype strings (`int64`, `float64`, `Int64`, `bool`, `boolean`, `datetime64[ns]`, `object`, ...); the writer maps them to internal string/number/date/bool — omit `dtypes` and a column writes as text (good for raw CSV-shaped data). `formats[col]` is an Excel number_format string (e.g. `#,##0.00`, `0.0%`, `yyyy-mm`); when absent, date columns default to `yyyy-mm-dd` and string columns to text format (`@`).", Input: []string{"file", "stdin"}}, + {Name: "dataframe", Kind: "own", Type: "string", Required: "xor", Desc: "Single-sheet typed table from one Arrow IPC file (a.k.a. Feather v2 — what `pandas.DataFrame.to_feather()` writes), XOR with `--sheets`. Pass `@` for a file or `-` for binary stdin (same convention as other input flags). Arrow bytes are read raw — no TrimSpace / BOM strip — so the IPC magic survives intact (unlike text input flags). Column types come from the Arrow schema (int*/uint*/float* → number, date32/date64/timestamp → date, utf8/large_utf8 → string, bool → bool); per-column `number_format` may be set via Arrow field metadata (`pa.field(\"price\", pa.float64(), metadata={b\"number_format\": b\"$#,##0.00\"})`). Writes the sheet at default placement: name `Sheet1` (created when absent), overwrite from A1 with header. For a different sheet name, anchor, mode, or to write multiple sheets, use `--sheets` instead."}, {Name: "dry-run", Kind: "system", Type: "bool", Required: "optional"}, }, }, @@ -941,8 +943,9 @@ var flagDefs = map[string]commandDef{ {Name: "title", Kind: "own", Type: "string", Required: "required", Desc: "Spreadsheet title"}, {Name: "folder-token", Kind: "own", Type: "string", Required: "optional", Desc: "Target folder token; placed at the drive root when omitted"}, {Name: "values", Kind: "own", Type: "string", Required: "optional", Desc: "Untyped initial data as one 2D JSON array (`[[\"alice\",95]]`); values are written as-is with their type auto-detected, through the same batched set_cell_range path as --sheets — pair with --styles for number formats, colors, merges, and row/col sizes", Input: []string{"file", "stdin"}}, - {Name: "sheets", Kind: "own", Type: "string", Required: "optional", Desc: "Typed table payload as JSON (same shape as `+table-put`): a top-level `sheets` array, each item `{name, start_cell?, mode?, header?, allow_overwrite?, columns:[\"colA\",\"colB\",...], data:[[...]], dtypes?:{colA:pandasDtype, ...}, formats?:{colA:numberFormat, ...}}`. Agents typically build it from a DataFrame via `{**json.loads(df.to_json(orient=\"split\")), \"dtypes\": df.dtypes.astype(str).to_dict()}`. Mutually exclusive with --values. Creates the workbook, then writes typed type-faithful data (dates land as real dates, numbers keep precision).", Input: []string{"file", "stdin"}}, + {Name: "sheets", Kind: "own", Type: "string", Required: "optional", Desc: "Typed table payload as JSON (same shape as `+table-put`): a top-level `sheets` array, each item `{name, start_cell?, mode?, header?, allow_overwrite?, columns:[\"colA\",\"colB\",...], data:[[...]], dtypes?:{colA:pandasDtype, ...}, formats?:{colA:numberFormat, ...}}`. Agents typically build it from a DataFrame via `{**json.loads(df.to_json(orient=\"split\")), \"dtypes\": df.dtypes.astype(str).to_dict()}`. Mutually exclusive with --values and --dataframe. Creates the workbook, then writes typed type-faithful data (dates land as real dates, numbers keep precision).", Input: []string{"file", "stdin"}}, {Name: "styles", Kind: "own", Type: "string", Required: "optional", Desc: "Initial visual operations as JSON: top-level `{styles:[...]}`. Each item corresponds to one target sheet and must include `name`, plus at least one of `cell_styles` / `row_sizes` / `col_sizes` / `cell_merges`. `cell_styles` entries use +cells-set-style fields with a cell range; row/col sizes use dimension ranges plus type/size; merges use cell ranges plus optional merge_type. With --sheets, styles array length/order/name must match --sheets.sheets. With --values, pass exactly one styles item for the initial sheet (its name is ignored).", Input: []string{"file", "stdin"}}, + {Name: "dataframe", Kind: "own", Type: "string", Required: "optional", Desc: "Single-sheet typed table from one Arrow IPC file (Feather v2 — what `pandas.DataFrame.to_feather()` writes), mutually exclusive with --values and --sheets. Pass `@` for a file or `-` for binary stdin (same convention as other input flags). Arrow bytes are read raw — no TrimSpace / BOM strip — so the IPC magic survives intact (unlike text input flags). Column types come from the Arrow schema; per-column `number_format` may be set via Arrow field metadata. Creates the workbook and fills its default sheet (`Sheet1` — adopted in place, no empty Sheet1 left behind). For multi-sheet or non-default placement, use `--sheets` instead."}, {Name: "dry-run", Kind: "system", Type: "bool", Required: "optional"}, }, }, diff --git a/shortcuts/sheets/lark_sheet_dataframe.go b/shortcuts/sheets/lark_sheet_dataframe.go new file mode 100644 index 000000000..1a54b8118 --- /dev/null +++ b/shortcuts/sheets/lark_sheet_dataframe.go @@ -0,0 +1,553 @@ +// Copyright (c) 2026 Lark Technologies Pte. Ltd. +// SPDX-License-Identifier: MIT + +package sheets + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "strconv" + "strings" + "time" + + "github.com/apache/arrow/go/v17/arrow" + "github.com/apache/arrow/go/v17/arrow/array" + "github.com/apache/arrow/go/v17/arrow/ipc" + "github.com/apache/arrow/go/v17/arrow/memory" + + "github.com/larksuite/cli/extension/fileio" + "github.com/larksuite/cli/internal/cmdutil" + "github.com/larksuite/cli/shortcuts/common" +) + +// ─── --dataframe (Arrow IPC / Feather v2 binary input) ──────────────── +// +// --dataframe is the binary-typed twin of --sheets. The wire payload is one +// Arrow IPC file (a.k.a. Feather v2 — what `pandas.DataFrame.to_feather()` +// writes), single schema, optionally multi-batch. Type / format are read off +// the Arrow schema (no separate dtypes/formats maps), and per-column number +// format can be set via the field's `number_format` metadata key: +// +// pa.field("price", pa.float64(), metadata={b"number_format": b"$#,##0.00"}) +// +// One DataFrame writes into one sub-sheet at fixed defaults: name `Sheet1` +// (adopted in place by +workbook-create; created when absent by +table-put), +// overwrite from A1 with header on, allow_overwrite=true. The shortcut +// surface is deliberately the one flag — anything that needs a different +// sheet name / anchor / mode / multi-sheet falls back to --sheets, whose +// JSON payload already carries every knob. +// +// Binary IO note: --dataframe bypasses the text-oriented Input resolver +// (`runtime.Str("dataframe")` carries a *path*, not file contents). Reading +// the Arrow bytes through that resolver would TrimSpace the trailing IPC +// magic / corrupt non-UTF8 bytes. Path → FileIO.Open → io.ReadAll keeps the +// stream byte-exact. "-" reads from stdin directly. + +// dataframeDefaultSheetName is the sub-sheet name --dataframe writes into. +// Matches valuesSheetName so +workbook-create adopts the brand-new +// workbook's default sheet in place (no stray empty Sheet1 left behind); +// +table-put creates Sheet1 if it doesn't already exist. +const dataframeDefaultSheetName = valuesSheetName + +// parseDataframePayload reads the --dataframe path (Arrow IPC file) and +// composes a single-sheet tablePayload at the fixed default placement. +// Network-free: safe from Validate and DryRun. The resulting tableSheetSpec +// rides the same buildSheetMatrix / buildTypedCell path as a --sheets entry, +// so downstream is unaware of where the rows came from. +func parseDataframePayload(rctx *common.RuntimeContext) (*tablePayload, error) { + raw := strings.TrimSpace(rctx.Str("dataframe")) + if raw == "" { + return nil, common.FlagErrorf("--dataframe is required") + } + data, err := readDataframeBytes(rctx, raw) + if err != nil { + return nil, err + } + spec, err := decodeArrowToSheet(data, dataframeDefaultSheetName) + if err != nil { + return nil, common.FlagErrorf("--dataframe: %v", err) + } + payload := &tablePayload{Sheets: []tableSheetSpec{spec}} + if err := payload.validate(); err != nil { + return nil, err + } + return payload, nil +} + +// dataframeStdinCache holds the bytes read from stdin on the first call so a +// later call (Validate → Execute / DryRun) gets the same bytes instead of an +// empty stream — stdin is single-shot, but parseDataframePayload runs +// multiple times per command invocation. Process-wide is fine: lark-cli is +// one-shot (one command per process). Tests reset by setting it back to nil. +var dataframeStdinCache []byte + +// readDataframeBytes resolves --dataframe to raw binary. A literal `@` prefix +// is tolerated for symmetry with --sheets (`@/tmp/x.arrow` and `/tmp/x.arrow` +// both work). `-` reads stdin verbatim — cached on first call so Validate / +// Execute / DryRun all see the same bytes. Bytes are returned untouched: no +// TrimSpace, no BOM strip — both would corrupt an Arrow IPC stream. +func readDataframeBytes(rctx *common.RuntimeContext, raw string) ([]byte, error) { + if raw == "-" { + if dataframeStdinCache != nil { + return dataframeStdinCache, nil + } + io := rctx.IO() + if io == nil || io.In == nil { + return nil, common.FlagErrorf("--dataframe: stdin is not available") + } + data, err := readAllBytes(io.In) + if err != nil { + return nil, common.FlagErrorf("--dataframe: read stdin: %v", err) + } + if len(data) == 0 { + return nil, common.FlagErrorf("--dataframe: stdin is empty") + } + dataframeStdinCache = data + return data, nil + } + path := strings.TrimPrefix(raw, "@") + data, err := cmdutil.ReadInputFile(rctx.FileIO(), path) + if err != nil { + return nil, common.FlagErrorf("--dataframe: %v", err) + } + if len(data) == 0 { + return nil, common.FlagErrorf("--dataframe: file %q is empty", path) + } + return data, nil +} + +// readAllBytes is a thin wrapper so tests can fake the io.Reader without +// importing io. Mirrors io.ReadAll exactly. +func readAllBytes(r io.Reader) ([]byte, error) { return io.ReadAll(r) } + +// decodeArrowToSheet reads `data` as an Arrow IPC file (single schema, +// possibly multi-batch) and produces a tableSheetSpec with name + columns + +// rows filled in. Sheet placement (start_cell / mode / header / overwrite) is +// not touched here — parseDataframePayload layers those on from CLI flags. +func decodeArrowToSheet(data []byte, sheetName string) (tableSheetSpec, error) { + reader, err := ipc.NewFileReader(bytes.NewReader(data)) + if err != nil { + return tableSheetSpec{}, fmt.Errorf("invalid Arrow IPC file (expected pandas df.to_feather output): %v", err) + } + defer reader.Close() + + schema := reader.Schema() + if schema == nil || schema.NumFields() == 0 { + return tableSheetSpec{}, fmt.Errorf("Arrow schema has no fields") + } + + ncols := schema.NumFields() + cols := make([]tableColumnSpec, ncols) + seen := make(map[string]bool, ncols) + for i := 0; i < ncols; i++ { + f := schema.Field(i) + name := f.Name + if strings.TrimSpace(name) == "" { + return tableSheetSpec{}, fmt.Errorf("column %d has empty name", i) + } + if seen[name] { + return tableSheetSpec{}, fmt.Errorf("duplicate column name %q", name) + } + seen[name] = true + typ, format, err := arrowFieldToTypeFormat(f) + if err != nil { + return tableSheetSpec{}, fmt.Errorf("column %q: %v", name, err) + } + cols[i] = tableColumnSpec{Name: name, Type: typ, Format: format} + } + + var rows [][]interface{} + for b := 0; b < reader.NumRecords(); b++ { + rec, err := reader.RecordAt(b) + if err != nil { + return tableSheetSpec{}, fmt.Errorf("read record batch %d: %v", b, err) + } + batchRows, err := arrowRecordToRows(rec, cols) + rec.Release() + if err != nil { + return tableSheetSpec{}, err + } + rows = append(rows, batchRows...) + } + + return tableSheetSpec{Name: sheetName, Columns: cols, Rows: rows}, nil +} + +// arrowFieldToTypeFormat maps an Arrow field to the internal (type, format) +// pair. The field's `number_format` metadata key — when present — sets the +// Excel number_format string verbatim; otherwise sensible defaults are +// applied per type (`@` text for strings, `yyyy-mm-dd` for dates). +func arrowFieldToTypeFormat(f arrow.Field) (typ, format string, err error) { + if v, ok := f.Metadata.GetValue("number_format"); ok { + format = strings.TrimSpace(v) + } + switch f.Type.(type) { + case *arrow.StringType, *arrow.LargeStringType: + if format == "" { + format = "@" + } + return "string", format, nil + case *arrow.BooleanType: + return "bool", format, nil + case *arrow.Date32Type, *arrow.Date64Type, *arrow.TimestampType: + if format == "" { + format = "yyyy-mm-dd" + } + return "date", format, nil + } + if isArrowNumericType(f.Type) { + return "number", format, nil + } + return "", "", fmt.Errorf("unsupported Arrow type %s (want string/number/date/bool)", f.Type.Name()) +} + +func isArrowNumericType(t arrow.DataType) bool { + switch t.ID() { + case arrow.INT8, arrow.INT16, arrow.INT32, arrow.INT64, + arrow.UINT8, arrow.UINT16, arrow.UINT32, arrow.UINT64, + arrow.FLOAT16, arrow.FLOAT32, arrow.FLOAT64: + return true + } + return false +} + +// arrowRecordToRows transposes one column-batch into row-major +// [][]interface{} matched to `cols`. Cells are stamped with the same value +// shapes buildTypedCell expects from the JSON path: nil for nulls, +// json.Number for numerics (precision-preserving), `yyyy-mm-dd` strings for +// dates/timestamps, bool for booleans, string for strings. +func arrowRecordToRows(rec arrow.Record, cols []tableColumnSpec) ([][]interface{}, error) { + if int(rec.NumCols()) != len(cols) { + return nil, fmt.Errorf("record has %d cols, schema declared %d", rec.NumCols(), len(cols)) + } + nrows := int(rec.NumRows()) + rows := make([][]interface{}, nrows) + for r := range rows { + rows[r] = make([]interface{}, len(cols)) + } + for c := range cols { + arr := rec.Column(c) + for r := 0; r < nrows; r++ { + if arr.IsNull(r) { + continue + } + v, err := arrowCellValue(arr, r) + if err != nil { + return nil, fmt.Errorf("row %d column %q: %v", r, cols[c].Name, err) + } + rows[r][c] = v + } + } + return rows, nil +} + +func arrowCellValue(arr arrow.Array, i int) (interface{}, error) { + switch a := arr.(type) { + case *array.String: + return a.Value(i), nil + case *array.LargeString: + return a.Value(i), nil + case *array.Boolean: + return a.Value(i), nil + case *array.Int8: + return json.Number(strconv.FormatInt(int64(a.Value(i)), 10)), nil + case *array.Int16: + return json.Number(strconv.FormatInt(int64(a.Value(i)), 10)), nil + case *array.Int32: + return json.Number(strconv.FormatInt(int64(a.Value(i)), 10)), nil + case *array.Int64: + return json.Number(strconv.FormatInt(a.Value(i), 10)), nil + case *array.Uint8: + return json.Number(strconv.FormatUint(uint64(a.Value(i)), 10)), nil + case *array.Uint16: + return json.Number(strconv.FormatUint(uint64(a.Value(i)), 10)), nil + case *array.Uint32: + return json.Number(strconv.FormatUint(uint64(a.Value(i)), 10)), nil + case *array.Uint64: + return json.Number(strconv.FormatUint(a.Value(i), 10)), nil + case *array.Float16: + return json.Number(strconv.FormatFloat(float64(a.Value(i).Float32()), 'f', -1, 32)), nil + case *array.Float32: + return json.Number(strconv.FormatFloat(float64(a.Value(i)), 'f', -1, 32)), nil + case *array.Float64: + return json.Number(strconv.FormatFloat(a.Value(i), 'f', -1, 64)), nil + case *array.Date32: + // Date32: days since 1970-01-01 (epoch). Multiply to seconds, format + // in UTC so timezone offset can't flip the calendar date. + t := time.Unix(int64(a.Value(i))*86400, 0).UTC() + return t.Format("2006-01-02"), nil + case *array.Date64: + t := time.UnixMilli(int64(a.Value(i))).UTC() + return t.Format("2006-01-02"), nil + case *array.Timestamp: + ts := int64(a.Value(i)) + unit := a.DataType().(*arrow.TimestampType).Unit + var t time.Time + switch unit { + case arrow.Second: + t = time.Unix(ts, 0).UTC() + case arrow.Millisecond: + t = time.UnixMilli(ts).UTC() + case arrow.Microsecond: + t = time.UnixMicro(ts).UTC() + case arrow.Nanosecond: + t = time.Unix(0, ts).UTC() + default: + return nil, fmt.Errorf("unsupported timestamp unit %v", unit) + } + return t.Format("2006-01-02"), nil + } + return nil, fmt.Errorf("unsupported Arrow array %T", arr) +} + +// ─── --dataframe-out (Arrow IPC binary output, mirror of --dataframe) ── +// +// +table-get's binary read-back: encode one sheet's typed read-back as an +// Arrow IPC file (Feather v2), so pandas can `pd.read_feather(path)` / +// `pd.read_feather(BytesIO(stdout))` symmetrically with the put side. +// Single-sheet only — Arrow IPC carries one schema per file. The JSON path +// is unchanged; --dataframe-out swaps the encoder for callers that already +// have pandas / pyarrow in their pipeline. + +// encodeSheetMapToArrowIPC turns one readSheetAsSpec output into an Arrow IPC +// file blob. Internal column types are recovered from `dtypes` (the wire +// proxy for the typed protocol), and per-column `number_format` rides through +// as Arrow field metadata so the file feeds straight back into +// `+table-put --dataframe`. +func encodeSheetMapToArrowIPC(sheet map[string]interface{}) ([]byte, error) { + columns, _ := sheet["columns"].([]interface{}) + if len(columns) == 0 { + return nil, fmt.Errorf("sheet has no columns") + } + dtypes, _ := sheet["dtypes"].(map[string]interface{}) + formats, _ := sheet["formats"].(map[string]interface{}) + // `data` arrives as either []interface{} (when the sheet came through a + // JSON round-trip / unit-test fixture) or [][]interface{} (the shape + // readSheetAsSpec directly emits in production). Accept both — anything + // else falls through to a zero-row table. + var rawData [][]interface{} + switch d := sheet["data"].(type) { + case [][]interface{}: + rawData = d + case []interface{}: + rawData = make([][]interface{}, len(d)) + for i, r := range d { + rawData[i], _ = r.([]interface{}) + } + } + + ncols := len(columns) + colNames := make([]string, ncols) + colTypes := make([]string, ncols) + fields := make([]arrow.Field, ncols) + for i, c := range columns { + name, _ := c.(string) + if name == "" { + return nil, fmt.Errorf("column %d has empty name", i) + } + colNames[i] = name + dt, _ := dtypes[name].(string) + colTypes[i] = dtypeToInternalType(dt) + var meta arrow.Metadata + if formats != nil { + if nf, ok := formats[name].(string); ok && strings.TrimSpace(nf) != "" { + meta = arrow.NewMetadata([]string{"number_format"}, []string{nf}) + } + } + fields[i] = arrow.Field{ + Name: name, + Type: internalTypeToArrowType(colTypes[i]), + Nullable: true, + Metadata: meta, + } + } + schema := arrow.NewSchema(fields, nil) + + mem := memory.NewGoAllocator() + rb := array.NewRecordBuilder(mem, schema) + defer rb.Release() + for r, row := range rawData { + if len(row) != ncols { + return nil, fmt.Errorf("row %d has %d cells, want %d", r, len(row), ncols) + } + for c := 0; c < ncols; c++ { + if err := appendArrowCell(rb.Field(c), colTypes[c], row[c]); err != nil { + return nil, fmt.Errorf("row %d column %q: %v", r, colNames[c], err) + } + } + } + rec := rb.NewRecord() + defer rec.Release() + + var buf bytesWriterSeeker + w, err := ipc.NewFileWriter(&buf, ipc.WithSchema(schema), ipc.WithAllocator(mem)) + if err != nil { + return nil, fmt.Errorf("ipc.NewFileWriter: %v", err) + } + if err := w.Write(rec); err != nil { + return nil, fmt.Errorf("write record: %v", err) + } + if err := w.Close(); err != nil { + return nil, fmt.Errorf("close writer: %v", err) + } + return buf.buf, nil +} + +// dtypeToInternalType inverts typeToDtype so the Arrow encoder can pick an +// internal column type from the wire-level dtype string. Unknown / object +// falls back to string (lossless: every cell is already typed as such). +func dtypeToInternalType(dtype string) string { + switch strings.ToLower(strings.TrimSpace(dtype)) { + case "float64", "float32", "int64", "int32", "int16", "int8", + "uint64", "uint32", "uint16", "uint8": + return "number" + case "bool", "boolean": + return "bool" + } + if strings.HasPrefix(strings.ToLower(dtype), "datetime") { + return "date" + } + return "string" +} + +// internalTypeToArrowType is the put-side dtypeToTypeFormat dual: maps the +// internal column type to the Arrow data type the encoder builds a column +// with. Numbers go to float64 because +table-get can't tell int from float +// from a number_format alone — float64 covers both losslessly for the cell +// ranges Lark Sheets accepts. +func internalTypeToArrowType(typ string) arrow.DataType { + switch typ { + case "number": + return arrow.PrimitiveTypes.Float64 + case "date": + return arrow.FixedWidthTypes.Date32 + case "bool": + return arrow.FixedWidthTypes.Boolean + } + return arrow.BinaryTypes.String +} + +// appendArrowCell stamps one cell into its column builder. Cell shape matches +// what cellToTyped emits on the JSON path: json.Number for numbers, ISO +// `yyyy-mm-dd` string for dates, plain string for strings, bool for bools, +// nil for empty. Anything off-shape errors so the caller doesn't silently +// emit nulls for malformed data. +func appendArrowCell(b array.Builder, typ string, v interface{}) error { + if v == nil { + b.AppendNull() + return nil + } + switch typ { + case "string": + s, ok := v.(string) + if !ok { + return fmt.Errorf("string expects string value, got %T", v) + } + b.(*array.StringBuilder).Append(s) + case "number": + f, err := arrowNumber(v) + if err != nil { + return err + } + b.(*array.Float64Builder).Append(f) + case "date": + s, ok := v.(string) + if !ok { + return fmt.Errorf("date expects ISO yyyy-mm-dd string, got %T", v) + } + t, err := time.Parse("2006-01-02", strings.TrimSpace(s)) + if err != nil { + return fmt.Errorf("date parse %q: %v", s, err) + } + b.(*array.Date32Builder).Append(arrow.Date32FromTime(t)) + case "bool": + bb, ok := v.(bool) + if !ok { + return fmt.Errorf("bool expects bool, got %T", v) + } + b.(*array.BooleanBuilder).Append(bb) + default: + return fmt.Errorf("unsupported internal type %q", typ) + } + return nil +} + +// arrowNumber converts the number cell shape readSheetAsSpec emits +// (json.Number) plus the float fallback to float64 for the Arrow builder. +func arrowNumber(v interface{}) (float64, error) { + switch n := v.(type) { + case json.Number: + f, err := n.Float64() + if err != nil { + return 0, fmt.Errorf("number parse %q: %v", n.String(), err) + } + return f, nil + case float64: + return n, nil + } + return 0, fmt.Errorf("number expects numeric value, got %T", v) +} + +// bytesWriterSeeker is a 10-line in-memory io.WriteSeeker for +// ipc.NewFileWriter, which seeks back to patch a footer offset. Using a +// buffer (instead of a temp file or os.Stdout, which isn't seekable) keeps +// --dataframe-out's stdout path zero-IO and stays straightforward. +type bytesWriterSeeker struct { + buf []byte + pos int64 +} + +func (w *bytesWriterSeeker) Write(p []byte) (int, error) { + end := w.pos + int64(len(p)) + if end > int64(len(w.buf)) { + w.buf = append(w.buf, make([]byte, end-int64(len(w.buf)))...) + } + n := copy(w.buf[w.pos:], p) + w.pos = end + return n, nil +} + +func (w *bytesWriterSeeker) Seek(offset int64, whence int) (int64, error) { + switch whence { + case io.SeekStart: + w.pos = offset + case io.SeekCurrent: + w.pos += offset + case io.SeekEnd: + w.pos = int64(len(w.buf)) + offset + default: + return 0, fmt.Errorf("unknown whence %d", whence) + } + return w.pos, nil +} + +// writeDataframeOut dispatches the encoded Arrow bytes to wherever --dataframe-out +// points: `-` → process stdout, `@` or plain path → local file. Symmetric +// with readDataframeBytes on the input side: same `@` tolerance, same TrimPrefix +// semantics, and an absolute path will still get rejected by FileIO's SafePath. +func writeDataframeOut(rctx *common.RuntimeContext, raw string, data []byte) error { + if raw == "-" { + out := rctx.IO() + if out == nil || out.Out == nil { + return common.FlagErrorf("--dataframe-out: stdout is not available") + } + if _, err := out.Out.Write(data); err != nil { + return fmt.Errorf("--dataframe-out: write stdout: %v", err) + } + return nil + } + path := strings.TrimPrefix(raw, "@") + fio := rctx.FileIO() + if fio == nil { + return common.FlagErrorf("--dataframe-out: file output is not available in this context") + } + // FileIO.Save validates the path via SafeOutputPath (the same sandbox + // readDataframeBytes hits on the input side) and writes atomically, so we + // don't need an extra ValidatePath call here. + if _, err := fio.Save(path, fileio.SaveOptions{ContentLength: int64(len(data))}, bytes.NewReader(data)); err != nil { + return fmt.Errorf("--dataframe-out: write %q: %v", path, err) + } + return nil +} diff --git a/shortcuts/sheets/lark_sheet_dataframe_test.go b/shortcuts/sheets/lark_sheet_dataframe_test.go new file mode 100644 index 000000000..4eb471eb4 --- /dev/null +++ b/shortcuts/sheets/lark_sheet_dataframe_test.go @@ -0,0 +1,378 @@ +// Copyright (c) 2026 Lark Technologies Pte. Ltd. +// SPDX-License-Identifier: MIT + +package sheets + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/apache/arrow/go/v17/arrow" + "github.com/apache/arrow/go/v17/arrow/array" + "github.com/apache/arrow/go/v17/arrow/ipc" + "github.com/apache/arrow/go/v17/arrow/memory" +) + +// buildArrowIPC writes one record into a Feather v2 (Arrow IPC file) blob. +// Used by the round-trip tests below to stand in for what +// `pandas.DataFrame.to_feather(path)` would produce; saves the tests from +// depending on a pandas-shaped fixture file. +// +// ipc.NewFileWriter wants an io.WriteSeeker (it back-patches a footer +// offset), so we write to a temp file and read the bytes back — simpler than +// re-implementing a seekable in-memory buffer. +func buildArrowIPC(t *testing.T, schema *arrow.Schema, build func(b *array.RecordBuilder)) []byte { + t.Helper() + mem := memory.NewGoAllocator() + rb := array.NewRecordBuilder(mem, schema) + defer rb.Release() + build(rb) + rec := rb.NewRecord() + defer rec.Release() + + path := filepath.Join(t.TempDir(), "df.arrow") + f, err := os.Create(path) + if err != nil { + t.Fatalf("create temp arrow file: %v", err) + } + w, err := ipc.NewFileWriter(f, ipc.WithSchema(schema), ipc.WithAllocator(mem)) + if err != nil { + f.Close() + t.Fatalf("ipc.NewFileWriter: %v", err) + } + if err := w.Write(rec); err != nil { + t.Fatalf("write record: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("close writer: %v", err) + } + if err := f.Close(); err != nil { + t.Fatalf("close file: %v", err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read temp arrow file: %v", err) + } + return data +} + +// TestDataframe_RoundTripCoreTypes pins down the Arrow-schema → internal +// (type, format) mapping and the per-cell value shape that buildTypedCell +// expects: number cells are json.Number (precision-preserving), date cells +// are `yyyy-mm-dd` strings, bool/string come through verbatim. Numbers, dates, +// strings, bools, and nulls all in one record so a future Arrow-Go bump can't +// quietly regress any one family. +func TestDataframe_RoundTripCoreTypes(t *testing.T) { + t.Parallel() + + schema := arrow.NewSchema([]arrow.Field{ + {Name: "name", Type: arrow.BinaryTypes.String}, + {Name: "qty", Type: arrow.PrimitiveTypes.Int64}, + {Name: "price", Type: arrow.PrimitiveTypes.Float64, Metadata: arrow.NewMetadata( + []string{"number_format"}, []string{"$#,##0.00"}, + )}, + {Name: "active", Type: arrow.FixedWidthTypes.Boolean}, + {Name: "shipped_on", Type: arrow.FixedWidthTypes.Date32}, + }, nil) + + jan15 := arrow.Date32FromTime(time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC)) + feb02 := arrow.Date32FromTime(time.Date(2024, 2, 2, 0, 0, 0, 0, time.UTC)) + + buf := buildArrowIPC(t, schema, func(b *array.RecordBuilder) { + b.Field(0).(*array.StringBuilder).AppendValues([]string{"alice", ""}, []bool{true, false}) + b.Field(1).(*array.Int64Builder).AppendValues([]int64{42, 0}, []bool{true, false}) + b.Field(2).(*array.Float64Builder).AppendValues([]float64{19.95, 0}, []bool{true, false}) + b.Field(3).(*array.BooleanBuilder).AppendValues([]bool{true, false}, []bool{true, true}) + b.Field(4).(*array.Date32Builder).AppendValues([]arrow.Date32{jan15, feb02}, []bool{true, true}) + }) + + spec, err := decodeArrowToSheet(buf, "S1") + if err != nil { + t.Fatalf("decodeArrowToSheet: %v", err) + } + if spec.Name != "S1" { + t.Errorf("sheet name = %q, want S1", spec.Name) + } + if len(spec.Columns) != 5 { + t.Fatalf("got %d columns, want 5", len(spec.Columns)) + } + want := []struct{ typ, format string }{ + {"string", "@"}, + {"number", ""}, + {"number", "$#,##0.00"}, + {"bool", ""}, + {"date", "yyyy-mm-dd"}, + } + for i, w := range want { + if spec.Columns[i].Type != w.typ { + t.Errorf("columns[%d].Type = %q, want %q", i, spec.Columns[i].Type, w.typ) + } + if spec.Columns[i].Format != w.format { + t.Errorf("columns[%d].Format = %q, want %q", i, spec.Columns[i].Format, w.format) + } + } + + if len(spec.Rows) != 2 { + t.Fatalf("got %d rows, want 2", len(spec.Rows)) + } + // Row 0: every field present, types match what buildTypedCell will accept. + row0 := spec.Rows[0] + if row0[0] != "alice" { + t.Errorf("row0[name] = %#v, want \"alice\"", row0[0]) + } + if n, ok := row0[1].(json.Number); !ok || n.String() != "42" { + t.Errorf("row0[qty] = %#v, want json.Number(\"42\")", row0[1]) + } + if n, ok := row0[2].(json.Number); !ok || n.String() != "19.95" { + t.Errorf("row0[price] = %#v, want json.Number(\"19.95\")", row0[2]) + } + if row0[3] != true { + t.Errorf("row0[active] = %#v, want true", row0[3]) + } + if row0[4] != "2024-01-15" { + t.Errorf("row0[shipped_on] = %#v, want \"2024-01-15\"", row0[4]) + } + + // Row 1: nulls on name/qty/price (despite the buffer values) must become nil + // so buildTypedCell paints an empty cell that still carries number_format. + row1 := spec.Rows[1] + for _, c := range []int{0, 1, 2} { + if row1[c] != nil { + t.Errorf("row1[%d] = %#v, want nil (null in arrow)", c, row1[c]) + } + } + if row1[3] != false { + t.Errorf("row1[active] = %#v, want false", row1[3]) + } + if row1[4] != "2024-02-02" { + t.Errorf("row1[shipped_on] = %#v, want \"2024-02-02\"", row1[4]) + } +} + +// TestDataframe_Timestamp pins the timestamp → date conversion for the +// timestamp[us] case (pandas default for `pd.Timestamp` columns once written +// via `to_feather`). Only the calendar date matters for our `yyyy-mm-dd` +// landing — guard against TZ drift from the wrong unit pick. +func TestDataframe_Timestamp(t *testing.T) { + t.Parallel() + schema := arrow.NewSchema([]arrow.Field{ + {Name: "ts", Type: &arrow.TimestampType{Unit: arrow.Microsecond}}, + }, nil) + ts := arrow.Timestamp(time.Date(2024, 6, 12, 14, 30, 0, 0, time.UTC).UnixMicro()) + buf := buildArrowIPC(t, schema, func(b *array.RecordBuilder) { + b.Field(0).(*array.TimestampBuilder).AppendValues([]arrow.Timestamp{ts}, []bool{true}) + }) + spec, err := decodeArrowToSheet(buf, "S") + if err != nil { + t.Fatal(err) + } + if spec.Columns[0].Type != "date" { + t.Errorf("type = %q, want date", spec.Columns[0].Type) + } + if got := spec.Rows[0][0]; got != "2024-06-12" { + t.Errorf("ts = %#v, want \"2024-06-12\"", got) + } +} + +// TestDataframe_EmptySchema rejects an Arrow file whose schema has no fields: +// a 0-column "DataFrame" would write a header-less, data-less block that +// validates as "writer ran successfully" but produces nothing — the test ties +// that off as an explicit error rather than letting it slip through. +func TestDataframe_EmptySchema(t *testing.T) { + t.Parallel() + schema := arrow.NewSchema(nil, nil) + buf := buildArrowIPC(t, schema, func(b *array.RecordBuilder) {}) + _, err := decodeArrowToSheet(buf, "S") + if err == nil || !strings.Contains(err.Error(), "no fields") { + t.Errorf("err = %v, want 'no fields' error", err) + } +} + +// TestDataframe_DuplicateColumn catches duplicate-name columns at decode +// time. Validate already rejects duplicate column names for the JSON path; +// the Arrow path mirrors that so the error surfaces with the same shape. +func TestDataframe_DuplicateColumn(t *testing.T) { + t.Parallel() + schema := arrow.NewSchema([]arrow.Field{ + {Name: "x", Type: arrow.BinaryTypes.String}, + {Name: "x", Type: arrow.PrimitiveTypes.Int64}, + }, nil) + buf := buildArrowIPC(t, schema, func(b *array.RecordBuilder) { + b.Field(0).(*array.StringBuilder).Append("") + b.Field(1).(*array.Int64Builder).Append(0) + }) + _, err := decodeArrowToSheet(buf, "S") + if err == nil || !strings.Contains(err.Error(), "duplicate") { + t.Errorf("err = %v, want duplicate-column error", err) + } +} + +// TestDataframe_BadBytes rejects a non-Arrow blob with a hint pointing at +// pandas df.to_feather so users see what producer is expected without having +// to grep the docs. +func TestDataframe_BadBytes(t *testing.T) { + t.Parallel() + _, err := decodeArrowToSheet([]byte("not arrow"), "S") + if err == nil || !strings.Contains(err.Error(), "Arrow") { + t.Errorf("err = %v, want Arrow-decode error", err) + } +} + +// TestDataframe_EncodeRoundTrip checks --dataframe-out's encoder against its +// own decoder: build a +table-get-shaped sheet map (the same one +// readSheetAsSpec emits), encode to Arrow IPC, decode back via the put-side +// decoder, and require the column types / formats / row values to match. If +// any encoder choice drifts from what the decoder expects, the round-trip +// breaks here long before a real put → get round-trip in production would. +func TestDataframe_EncodeRoundTrip(t *testing.T) { + t.Parallel() + sheet := map[string]interface{}{ + "name": "S1", + "columns": []interface{}{"name", "qty", "price", "active", "ts"}, + "dtypes": map[string]interface{}{ + "name": "object", + "qty": "float64", + "price": "float64", + "active": "bool", + "ts": "datetime64[ns]", + }, + "formats": map[string]interface{}{ + // `@` is the writer convention for string columns; readSheetAsSpec + // strips it via isTextNumberFormat, so an Arrow file built from a + // real read won't carry @ either. Keep it absent here to mirror + // the production wire shape. + "price": "$#,##0.00", + }, + "data": []interface{}{ + []interface{}{"alice", json.Number("42"), json.Number("19.95"), true, "2024-01-15"}, + []interface{}{"bob", nil, json.Number("8.5"), false, "2024-02-02"}, + }, + } + blob, err := encodeSheetMapToArrowIPC(sheet) + if err != nil { + t.Fatalf("encodeSheetMapToArrowIPC: %v", err) + } + spec, err := decodeArrowToSheet(blob, "S1") + if err != nil { + t.Fatalf("decodeArrowToSheet: %v", err) + } + wantTypes := []string{"string", "number", "number", "bool", "date"} + wantFormats := []string{"@", "", "$#,##0.00", "", "yyyy-mm-dd"} + if len(spec.Columns) != len(wantTypes) { + t.Fatalf("got %d columns, want %d", len(spec.Columns), len(wantTypes)) + } + for i, w := range wantTypes { + if spec.Columns[i].Type != w { + t.Errorf("columns[%d].Type = %q, want %q", i, spec.Columns[i].Type, w) + } + if spec.Columns[i].Format != wantFormats[i] { + t.Errorf("columns[%d].Format = %q, want %q", i, spec.Columns[i].Format, wantFormats[i]) + } + } + if len(spec.Rows) != 2 { + t.Fatalf("got %d rows, want 2", len(spec.Rows)) + } + if spec.Rows[0][0] != "alice" { + t.Errorf("row0[name] = %#v, want alice", spec.Rows[0][0]) + } + if n, ok := spec.Rows[0][1].(json.Number); !ok || n.String() != "42" { + t.Errorf("row0[qty] = %#v, want json.Number(\"42\")", spec.Rows[0][1]) + } + if spec.Rows[0][3] != true { + t.Errorf("row0[active] = %#v, want true", spec.Rows[0][3]) + } + if spec.Rows[0][4] != "2024-01-15" { + t.Errorf("row0[ts] = %#v, want 2024-01-15", spec.Rows[0][4]) + } + // qty is null on row1, must come back as nil (not a zero-valued + // json.Number that would later round-trip as 0). + if spec.Rows[1][1] != nil { + t.Errorf("row1[qty] = %#v, want nil (null arrow cell)", spec.Rows[1][1]) + } +} + +// TestDataframe_EncodeAcceptsBothRowShapes pins the encoder against the two +// shapes `sheet["data"]` actually arrives in: `[][]interface{}` from a live +// readSheetAsSpec call (production), and `[]interface{}` from a JSON +// unmarshal (round-trip / fixtures). Either must produce non-empty Arrow +// output — early on the production shape silently fell through the +// `[]interface{}` type assertion and we shipped a 0-row Arrow blob. +func TestDataframe_EncodeAcceptsBothRowShapes(t *testing.T) { + t.Parallel() + base := func(data interface{}) map[string]interface{} { + return map[string]interface{}{ + "name": "S", + "columns": []interface{}{"city"}, + "dtypes": map[string]interface{}{"city": "object"}, + "data": data, + } + } + for label, data := range map[string]interface{}{ + "production [][]interface{}": [][]interface{}{{"BJ"}, {"SH"}}, + "unmarshal []interface{}": []interface{}{[]interface{}{"BJ"}, []interface{}{"SH"}}, + } { + blob, err := encodeSheetMapToArrowIPC(base(data)) + if err != nil { + t.Errorf("%s: encode: %v", label, err) + continue + } + spec, err := decodeArrowToSheet(blob, "S") + if err != nil { + t.Errorf("%s: decode: %v", label, err) + continue + } + if len(spec.Rows) != 2 { + t.Errorf("%s: got %d rows, want 2", label, len(spec.Rows)) + } + } +} + +// TestDataframe_DtypeToInternalType pins the inverse of typeToDtype so +// readSheetAsSpec's dtype labels recover the right internal type. Covers the +// dtype families +table-get emits today plus the safe fallback for unknown +// labels (string, lossless). +func TestDataframe_DtypeToInternalType(t *testing.T) { + t.Parallel() + cases := map[string]string{ + "float64": "number", + "int64": "number", + "Int64": "number", + "bool": "bool", + "boolean": "bool", + "datetime64[ns]": "date", + "datetime64[ms]": "date", + "object": "string", + "": "string", + "weird-new-dtype": "string", + } + for in, want := range cases { + if got := dtypeToInternalType(in); got != want { + t.Errorf("dtypeToInternalType(%q) = %q, want %q", in, got, want) + } + } +} + +// TestDataframe_BytesWriterSeeker confirms the in-memory WriteSeeker handles +// the Seek-and-overwrite pattern ipc.NewFileWriter uses to patch the footer +// offset: write some bytes, seek back to the middle, overwrite, end up with +// the buffer reflecting the overwritten bytes (not a tail-extended duplicate). +func TestDataframe_BytesWriterSeeker(t *testing.T) { + t.Parallel() + var w bytesWriterSeeker + if _, err := w.Write([]byte("hello world")); err != nil { + t.Fatal(err) + } + if _, err := w.Seek(6, 0); err != nil { + t.Fatal(err) + } + if _, err := w.Write([]byte("WORLD")); err != nil { + t.Fatal(err) + } + if got := string(w.buf); got != "hello WORLD" { + t.Errorf("buf = %q, want \"hello WORLD\"", got) + } +} diff --git a/shortcuts/sheets/lark_sheet_table_io.go b/shortcuts/sheets/lark_sheet_table_io.go index 9d0e14fa6..535b03e23 100644 --- a/shortcuts/sheets/lark_sheet_table_io.go +++ b/shortcuts/sheets/lark_sheet_table_io.go @@ -8,6 +8,7 @@ import ( "encoding/json" "fmt" "math" + "strconv" "strings" "time" @@ -52,7 +53,7 @@ var TablePut = common.Shortcut{ if _, err := resolveSpreadsheetToken(runtime); err != nil { return err } - _, err := parseTablePutPayload(runtime) + _, err := resolveTablePayload(runtime) return err }, DryRun: func(ctx context.Context, runtime *common.RuntimeContext) *common.DryRunAPI { @@ -63,7 +64,7 @@ var TablePut = common.Shortcut{ if err != nil { return err } - payload, err := parseTablePutPayload(runtime) + payload, err := resolveTablePayload(runtime) if err != nil { return err } @@ -72,9 +73,32 @@ var TablePut = common.Shortcut{ Tips: []string{ "Writes into an existing spreadsheet — pass --url or --spreadsheet-token. To create a new workbook first, use +workbook-create, then point --spreadsheet-token here.", "Payload sheets are matched to existing sub-sheets by name (created when absent). Date columns take ISO yyyy-mm-dd strings — converted to real dates (serial + date format).", + "Two equivalent producers: --sheets (multi-sheet JSON, the pandas-split convention) or --dataframe (single-sheet Arrow IPC binary, what `df.to_feather()` writes). Mutually exclusive; pick by what your producer already emits.", }, } +// resolveTablePayload picks between --sheets (JSON, multi-sheet) and +// --dataframe (Arrow IPC, single-sheet), enforces XOR, and returns the +// unified internal tablePayload. Both +table-put and +workbook-create funnel +// through here so the two entry points stay in lockstep; Validate / Execute / +// DryRun / workbookCreateData all share this one decision. Network-free. +func resolveTablePayload(rctx *common.RuntimeContext) (*tablePayload, error) { + sheetsGiven := rctx.Changed("sheets") && strings.TrimSpace(rctx.Str("sheets")) != "" + dfGiven := rctx.Changed("dataframe") && strings.TrimSpace(rctx.Str("dataframe")) != "" + if sheetsGiven && dfGiven { + return nil, common.FlagErrorf("--sheets and --dataframe are mutually exclusive") + } + if !sheetsGiven && !dfGiven { + // Mirror the original "--sheets is required" message but list both + // alternatives so users discover the binary entry from the error. + return nil, common.FlagErrorf("one of --sheets or --dataframe is required") + } + if dfGiven { + return parseDataframePayload(rctx) + } + return parseTablePutPayload(rctx) +} + // ─── protocol ───────────────────────────────────────────────────────── type tablePayload struct { @@ -601,6 +625,16 @@ func writeSheetData(ctx context.Context, runtime *common.RuntimeContext, token, }, nil } + // Grow the sub-sheet to fit the write block before the first batch. + // Without this, writes past the sheet's initial dimensions (typically the + // backend default of 200 rows × 20 cols — which also covers + // `+workbook-create`'s adopted Sheet1) fail with [900015206] range exceeds + // sheet bounds. Best-effort: if reading dims fails the downstream write + // will surface the same out-of-bounds error it did before this helper. + if err := ensureSheetCapacity(ctx, runtime, token, sheetID, baseRow+len(matrix), col0+ncols); err != nil { + return nil, fmt.Errorf("ensuring sheet capacity: %w", err) + } + startCol := columnIndexToLetter(col0) endCol := columnIndexToLetter(col0 + ncols - 1) allowOverwrite := s.AllowOverwrite == nil || *s.AllowOverwrite @@ -819,6 +853,91 @@ func sheetCreateDims(s *tableSheetSpec) (rows, cols int) { return rows, cols } +// ensureSheetCapacity grows the sub-sheet's row / column count enough to +// fit a needRows × needCols write block before the next set_cell_range call. +// Both +table-put writing into an existing sheet *and* +workbook-create's +// adopted default Sheet1 inherit the backend's 200×20 default — anything +// past that would error with `[900015206] range exceeds sheet bounds`. +// Read failures (e.g. mock stubs without `row_count`) silently fall through; +// the downstream write surfaces the same out-of-bounds error it did before, +// so the helper can't make things worse. Backend hard ceilings (50000 rows, +// 200 cols) are honored. +func ensureSheetCapacity(ctx context.Context, runtime *common.RuntimeContext, token, sheetID string, needRows, needCols int) error { + if needRows <= 0 && needCols <= 0 { + return nil + } + out, err := callTool(ctx, runtime, token, ToolKindRead, "get_sheet_structure", map[string]interface{}{ + "excel_id": token, + "sheet_id": sheetID, + }) + if err != nil { + // best-effort: skip if we can't see current dims (mock without stub, + // permissions, transient failure). The write below will still bounce + // if the sheet really is too small, so degrading silently here is + // fail-open by design. + return nil //nolint:nilerr + } + m, _ := out.(map[string]interface{}) + // get_sheet_structure reports current dims as the `range` field + // (e.g. "A1:T200" → 20 cols × 200 rows). splitCellRef parses the + // bottom-right corner into 0-based (col, row). + curRows, curCols := 0, 0 + if rng, _ := m["range"].(string); rng != "" { + parts := strings.SplitN(rng, ":", 2) + if len(parts) == 2 { + if c, r, ok := splitCellRef(parts[1]); ok { + curCols = c + 1 + curRows = r + 1 + } + } + } + if needRows > curRows && curRows > 0 { + target := needRows + if target > 50000 { + target = 50000 + } + if target > curRows { + // position is 1-based and must be ≤ curRows (the backend rejects + // "before row N+1" as out-of-range). Inserting before the last + // existing row pushes that row down and effectively appends + // `count` blank rows — the data writes that follow will overwrite + // the existing rows from row 1, so the placement of the inserted + // blanks doesn't matter as long as the total dimension grows. + input := map[string]interface{}{ + "excel_id": token, + "sheet_id": sheetID, + "operation": "insert", + "position": strconv.Itoa(curRows), + "count": target - curRows, + } + if _, err := callTool(ctx, runtime, token, ToolKindWrite, "modify_sheet_structure", input); err != nil { + return fmt.Errorf("growing rows %d → %d: %w", curRows, target, err) + } + } + } + if needCols > curCols && curCols > 0 { + target := needCols + if target > 200 { + target = 200 + } + if target > curCols { + // Same 1-based, ≤-current constraint as rows: insert before the + // last existing column letter. + input := map[string]interface{}{ + "excel_id": token, + "sheet_id": sheetID, + "operation": "insert", + "position": columnIndexToLetter(curCols - 1), + "count": target - curCols, + } + if _, err := callTool(ctx, runtime, token, ToolKindWrite, "modify_sheet_structure", input); err != nil { + return fmt.Errorf("growing cols %d → %d: %w", curCols, target, err) + } + } + } + return nil +} + // listSheetIDsByName maps every existing sub-sheet's display name to its id via // a single get_workbook_structure read. Used by write mode to decide which // payload sheets already exist. @@ -894,7 +1013,7 @@ func tablePutPartial(token string, spreadsheet interface{}, written []interface{ // by Validate, so errors here degrade to an empty preview rather than twice. func tablePutDryRun(runtime *common.RuntimeContext) *common.DryRunAPI { dry := common.NewDryRunAPI() - payload, err := parseTablePutPayload(runtime) + payload, err := resolveTablePayload(runtime) if err != nil { return dry } @@ -951,6 +1070,15 @@ var TableGet = common.Shortcut{ if strings.TrimSpace(runtime.Str("sheet-id")) != "" && strings.TrimSpace(runtime.Str("sheet-name")) != "" { return common.FlagErrorf("--sheet-id and --sheet-name are mutually exclusive") } + // --dataframe-out is Arrow IPC, which carries one schema per file — a + // whole-workbook read can't ride that shape. Surface the constraint + // before we round-trip to the API instead of after the read fails to + // encode. + if strings.TrimSpace(runtime.Str("dataframe-out")) != "" { + if strings.TrimSpace(runtime.Str("sheet-id")) == "" && strings.TrimSpace(runtime.Str("sheet-name")) == "" { + return common.FlagErrorf("--dataframe-out requires --sheet-id or --sheet-name (single-sheet only); for the whole workbook, drop --dataframe-out and use the default JSON output") + } + } return nil }, DryRun: func(ctx context.Context, runtime *common.RuntimeContext) *common.DryRunAPI { @@ -992,12 +1120,38 @@ var TableGet = common.Shortcut{ } sheets = append(sheets, spec) } + // Arrow IPC binary branch: Validate already guards single-sheet so the + // sheets slice has exactly one entry here. Stdout mode owns stdout for + // the binary stream — no envelope, raw Arrow only. File mode writes + // the Arrow blob to disk and still emits the lark-cli JSON envelope + // (output_path / bytes / sheet_name) so scripted callers can detect + // success the same way they do for every other shortcut. + if dfOut := strings.TrimSpace(runtime.Str("dataframe-out")); dfOut != "" { + spec, _ := sheets[0].(map[string]interface{}) + data, err := encodeSheetMapToArrowIPC(spec) + if err != nil { + return common.FlagErrorf("--dataframe-out: encode arrow: %v", err) + } + if err := writeDataframeOut(runtime, dfOut, data); err != nil { + return err + } + if dfOut == "-" { + return nil + } + runtime.Out(map[string]interface{}{ + "output_path": strings.TrimPrefix(dfOut, "@"), + "bytes": len(data), + "sheet_name": spec["name"], + }, nil) + return nil + } runtime.Out(map[string]interface{}{"sheets": sheets}, nil) return nil }, Tips: []string{ "Output is the same shape +table-put consumes — pipe it back in, or load sheets[].rows into a DataFrame keyed by columns[].name.", "Column types are inferred per column, but only when every non-empty cell agrees; a column mixing types (e.g. numbers + \"暂无\") degrades to string — lossless and round-trips cleanly. Numeric coercion of dirty cells is the caller's job (pandas to_numeric(errors=\"coerce\") on the string column).", + "For a pandas round-trip, use --dataframe-out (single sheet, Arrow IPC / Feather v2) — `@./x.arrow` writes a file, `-` streams binary to stdout for `pd.read_feather(BytesIO(stdout))`. Multi-sheet reads stay on the JSON path.", }, } diff --git a/shortcuts/sheets/lark_sheet_workbook.go b/shortcuts/sheets/lark_sheet_workbook.go index 9e6498dfc..40c741dae 100644 --- a/shortcuts/sheets/lark_sheet_workbook.go +++ b/shortcuts/sheets/lark_sheet_workbook.go @@ -571,17 +571,23 @@ var WorkbookCreate = common.Shortcut{ if strings.TrimSpace(runtime.Str("title")) == "" { return common.FlagErrorf("--title is required") } - // --sheets (typed) is an alternative, mutually exclusive data entry to the - // untyped --values. Gated on Changed (not just non-empty): an explicitly- - // given but empty --sheets (e.g. empty stdin / file) is an error, not a - // silent fall-through to creating an empty workbook. - if runtime.Changed("sheets") { + // --sheets (typed JSON) and --dataframe (typed Arrow IPC) are two + // alternative typed data entries; both are mutually exclusive with + // the untyped --values. Gating on Changed (not just non-empty) catches + // an explicitly-given but empty payload as an error instead of letting + // it fall through to creating an empty workbook. + sheetsGiven := runtime.Changed("sheets") + dfGiven := runtime.Changed("dataframe") + if sheetsGiven && dfGiven { + return common.FlagErrorf("--sheets and --dataframe are mutually exclusive") + } + if (sheetsGiven || dfGiven) && runtime.Str("values") != "" { + return common.FlagErrorf("--values is mutually exclusive with --sheets/--dataframe") + } + if sheetsGiven { if strings.TrimSpace(runtime.Str("sheets")) == "" { return common.FlagErrorf("--sheets was given but resolved to empty (empty stdin/file?); pass a typed payload, or drop --sheets to create an empty workbook") } - if runtime.Str("values") != "" { - return common.FlagErrorf("--sheets is mutually exclusive with --values") - } payload, err := parseTablePutPayload(runtime) if err != nil { return err @@ -589,6 +595,17 @@ var WorkbookCreate = common.Shortcut{ _, err = parseWorkbookCreateSheetStyles(runtime, payload) return err } + if dfGiven { + if strings.TrimSpace(runtime.Str("dataframe")) == "" { + return common.FlagErrorf("--dataframe was given but resolved to empty; pass a path to an Arrow IPC file, or drop --dataframe to create an empty workbook") + } + payload, err := parseDataframePayload(runtime) + if err != nil { + return err + } + _, err = parseWorkbookCreateSheetStyles(runtime, payload) + return err + } // Untyped --values path: parse (and validate) --styles as a single sheet // style item, then synthesize --values into a type-less typed payload — // the same construction buildValuesPayload runs at execute time, so any @@ -731,6 +748,17 @@ func workbookCreateData(runtime *common.RuntimeContext) (*tablePayload, *workboo } return payload, styles, nil } + if runtime.Changed("dataframe") { + payload, err := parseDataframePayload(runtime) + if err != nil { + return nil, nil, err + } + styles, err := parseWorkbookCreateSheetStyles(runtime, payload) + if err != nil { + return nil, nil, err + } + return payload, styles, nil + } styles, err := parseValuesSheetStyles(runtime) if err != nil { return nil, nil, err diff --git a/skills/lark-sheets/references/lark-sheets-read-data.md b/skills/lark-sheets/references/lark-sheets-read-data.md index baca5ea46..c85f9071d 100644 --- a/skills/lark-sheets/references/lark-sheets-read-data.md +++ b/skills/lark-sheets/references/lark-sheets-read-data.md @@ -130,6 +130,7 @@ _公共：URL/token（无 sheet 定位） · 系统：`--dry-run`_ | `--sheet-name` | string | optional | 只读该子表（按名）；省略则读所有子表 | | `--range` | string | optional | 读取的 A1 范围；省略则读每个子表的当前数据区 | | `--no-header` | bool | optional | 把第一行当数据而非表头（列名取 col1/col2 …） | +| `--dataframe-out` | string | optional | 以一份 Arrow IPC 文件（Feather v2）格式输出 typed 表格，替代默认的 JSON 输出。用 `@` 传文件或 `-` 写二进制 stdout（同其他 binary I/O flag 的约定）。是 `+table-put` / `+workbook-create` 入口 `--dataframe` 的镜像 —— pandas 端 `pd.read_feather("x.arrow")` 或 `pd.read_feather(io.BytesIO(stdout))` 一行读回。仅支持单 sheet：必须给 `--sheet-id` 或 `--sheet-name`；读整本 workbook 仍走默认 JSON。列类型沿用 typed 读回（string/number/date/bool）；`number_format` 以 Arrow Field metadata 保留，Arrow 文件可直接喂回 `+table-put --dataframe`。 | ## Examples @@ -200,6 +201,34 @@ df_sales = sheets["销售"] > 显示格式（千分位、百分比、自定义日期）在 `sheet["formats"]`，pandas 不消费；改完数据 round-trip 回去时透传给 `+table-put` 即可，飞书侧显示不变。 +#### `--dataframe-out`（Arrow IPC / Feather v2 二进制读出） + +`--dataframe-out` 是 `+table-put` 入口 `--dataframe` 的镜像：把 typed 读回直接编码成 Arrow IPC 文件，pandas 端一行 `pd.read_feather()` 读回——省掉 JSON 解析 + `astype(dtypes)`，列类型 / `number_format` 走 Arrow schema + Field metadata 保真。**仅支持单 sheet**（Arrow 文件一 schema 容器），必须给 `--sheet-id` 或 `--sheet-name`；读整本 workbook 仍走默认 JSON。 + +```bash +# 文件 +lark-cli sheets +table-get --url "<表URL>" --sheet-name "销售" --dataframe-out @./out.arrow +# binary stdout（不落盘） +lark-cli sheets +table-get --url "<表URL>" --sheet-name "销售" --dataframe-out - +``` + +```python +import io, pandas as pd, subprocess + +# 1) 文件 +subprocess.run(["lark-cli","sheets","+table-get","--url",URL, + "--sheet-name","销售","--dataframe-out","@./out.arrow"], check=True) +df = pd.read_feather("./out.arrow") + +# 2) stdin/stdout 管道（不落盘）—— 跟 --dataframe 写入侧对称的一行 +res = subprocess.run(["lark-cli","sheets","+table-get","--url",URL, + "--sheet-name","销售","--dataframe-out","-"], + capture_output=True, check=True) +df = pd.read_feather(io.BytesIO(res.stdout)) +``` + +> `number_format` 进 Arrow Field metadata（key=`number_format`），Arrow 文件可以直接喂回 `+table-put --dataframe` round-trip 写回，types / formats 一路保真。 + #### round-trip：读 → 改 → 写回（写读对偶） `sheet_to_df` 和 write-cells reference 里的 `df_to_sheet` 是一对镜像 helper，round-trip 三段读 / 改 / 写各一行： diff --git a/skills/lark-sheets/references/lark-sheets-workbook.md b/skills/lark-sheets/references/lark-sheets-workbook.md index 304c35d2b..50678690d 100644 --- a/skills/lark-sheets/references/lark-sheets-workbook.md +++ b/skills/lark-sheets/references/lark-sheets-workbook.md @@ -139,8 +139,9 @@ _系统：`--dry-run`_ | `--title` | string | required | 新 spreadsheet 标题 | | `--folder-token` | string | optional | 目标文件夹 token；省略时放在云空间根目录 | | `--values` | string + File + Stdin（简单 JSON） | optional | untyped 初始数据，一个 JSON 二维数组（表头并入第一行）：`[["列A","列B"],["alice",95]]`；值原样写入、类型由飞书自动识别，走与 --sheets 相同的分批 `+cells-set`；配 --styles 控制格式/颜色/合并/行列尺寸 | -| `--sheets` | string + File + Stdin（复合 JSON） | optional | 建表后写入的 typed 表格协议 JSON（同 +table-put）：顶层 sheets 数组，每项 `{name, start_cell?, mode?, header?, allow_overwrite?, columns:["colA","colB",...], data:[[...]], dtypes?:{colA:pandasDtype, ...}, formats?:{colA:numberFormat, ...}}`。Agents 通常用 `{**json.loads(df.to_json(orient="split")), "dtypes": df.dtypes.astype(str).to_dict()}` 一行构造。与 --values 互斥；新表默认子表复用为第一个子表，日期/数字类型保真。 | +| `--sheets` | string + File + Stdin（复合 JSON） | optional | 建表后写入的 typed 表格协议 JSON（同 +table-put）：顶层 sheets 数组，每项 `{name, start_cell?, mode?, header?, allow_overwrite?, columns:["colA","colB",...], data:[[...]], dtypes?:{colA:pandasDtype, ...}, formats?:{colA:numberFormat, ...}}`。Agents 通常用 `{**json.loads(df.to_json(orient="split")), "dtypes": df.dtypes.astype(str).to_dict()}` 一行构造。与 --values、--dataframe 互斥；新表默认子表复用为第一个子表，日期/数字类型保真。 | | `--styles` | string + File + Stdin（复合 JSON） | optional | 建表时同时写入的视觉处理操作 JSON：顶层 `{styles:[...]}`，每项对应一个目标子表、含 `name`，并至少给 `cell_styles` / `row_sizes` / `col_sizes` / `cell_merges` 之一。`cell_styles` 用 A1 单元格 range + 扁平样式字段（字段同 +cells-set-style，含 number_format / 颜色 / 对齐 / border_styles）；row/col sizes 用行/列范围 + type/size；merges 用单元格 range + 可选 merge_type。与 --sheets 搭配时 styles 数组长度/顺序/name 必须与 --sheets.sheets 对应；与 --values 搭配时只给一个 styles 项（其 name 忽略）。 | +| `--dataframe` | string | optional | 单 sheet 类型保真表格的二进制入口，从一个 Arrow IPC 文件（Feather v2，pandas `df.to_feather()` 直接写出）读入，与 --values / --sheets 互斥。用 `@` 传文件或 `-` 读二进制 stdin（同其他输入 flag 的约定）。Arrow 字节按原样读 —— 不做 TrimSpace / BOM strip，IPC magic 字节完整保留（区别于文本类输入 flag）。列类型从 Arrow schema 推导；每列的 `number_format` 可写在 Arrow Field metadata 里。建表后写入默认子表（`Sheet1` —— 直接复用，不残留空 Sheet1）。要多子表或换落点，请改用 `--sheets`。 | ### `+workbook-export` @@ -199,7 +200,7 @@ _一个或多个子表的 typed 数据，每个数组元素写入一张子表； ### `+workbook-create` -新建电子表格，可选预填数据。两种数据入口（untyped `--values` / typed `--sheets`）**互斥**，按需二选一——两者都走同一条分批 `set_cell_range` 写入： +新建电子表格，可选预填数据。三种数据入口（untyped `--values` / typed `--sheets` JSON / typed `--dataframe` Arrow 二进制）**三方互斥**，按需选一——两者都走同一条分批 `set_cell_range` 写入： ```bash # 1) untyped：--values（一个二维数组，表头并入第一行；值原样写、类型由飞书自动识别， @@ -207,7 +208,7 @@ _一个或多个子表的 typed 数据，每个数组元素写入一张子表； lark-cli sheets +workbook-create --title "销售" \ --values '[["门店","销售额"],["北京",259874]]' -# 2) typed：--sheets（一步建表 + 类型保真）。date 列落成真日期（可排序/透视）、 +# 2) typed JSON：--sheets（一步建表 + 类型保真）。date 列落成真日期（可排序/透视）、 # number 不丢精度、string 列保前导零（如订单号 00123）；多子表一次建。 lark-cli sheets +workbook-create --title "交易" --sheets '{ "sheets":[ @@ -217,9 +218,16 @@ lark-cli sheets +workbook-create --title "交易" --sheets '{ "formats":{"金额":"#,##0.00"}, "data":[["2024-01-15",1234.5,"00123"]]} ]}' + +# 3) typed binary：--dataframe（pandas df.to_feather 直接出，Arrow IPC / Feather v2）。 +# 单子表（落点固定为新表的默认子表，原地复用、不残留空 Sheet1），列类型从 Arrow +# schema 自动恢复，无需手填 dtypes/formats；要多子表回到 --sheets。 +lark-cli sheets +workbook-create --title "交易" --dataframe @./in.arrow +# 或走 stdin（不落盘）： +python prepare.py | lark-cli sheets +workbook-create --title "交易" --dataframe - ``` -`--sheets` 协议与 `+table-put` 完全同构（字段含义见 lark-sheets-write-cells 的 `+table-put`，大 payload 走 stdin / `@file`）。关键差异：**新建工作簿的默认子表会被复用为第一个子表**（重命名后承载数据），不会残留空 `Sheet1`；其余子表按需新建。它把 `+table-put` 单独做不到的"建表 + typed 写入"合到一条命令，是「pandas 算完直接落地一张带真日期的新表」的首选。回读校验用 `+table-get`（与 `--sheets` 同构、可 round-trip）。 +`--sheets` 协议与 `+table-put` 完全同构（字段含义见 lark-sheets-write-cells 的 `+table-put`，大 payload 走 stdin / `@file`）；`--dataframe` 是同一份 typed 数据的二进制 wire（Arrow IPC，详见同 reference 的 `+table-put` 段落的 `--dataframe` 小节），按 producer 已有的 API 选——pandas 走 `--dataframe`，多子表 / 手拼 JSON 走 `--sheets`。关键差异：**新建工作簿的默认子表会被复用为第一个子表**（重命名后承载数据），不会残留空 `Sheet1`；其余子表按需新建。它把 `+table-put` 单独做不到的"建表 + typed 写入"合到一条命令，是「pandas 算完直接落地一张带真日期的新表」的首选。回读校验用 `+table-get`（与 `--sheets` 同构、可 round-trip；pandas 用户也可走 `--dataframe-out` 直拿 Arrow 文件）。 `--styles` 可在建表写入时同时写视觉处理。它和 `--sheets` 一样只有一种外层写法：顶层对象里放 `styles` 数组；数组每项对应一个子表，含 `name`，并按能力拆成四类可选数组： diff --git a/skills/lark-sheets/references/lark-sheets-write-cells.md b/skills/lark-sheets/references/lark-sheets-write-cells.md index 42bcb341f..d3cbc54a4 100644 --- a/skills/lark-sheets/references/lark-sheets-write-cells.md +++ b/skills/lark-sheets/references/lark-sheets-write-cells.md @@ -317,7 +317,8 @@ _公共：URL/token（无 sheet 定位） · 系统：`--dry-run`_ | Flag | Type | 必填 | 说明 | | --- | --- | --- | --- | -| `--sheets` | string + File + Stdin（复合 JSON） | required | Typed 表格协议（pandas-DataFrame-shaped）JSON：顶层 sheets 数组，每项 `{name, start_cell?, mode?, header?, allow_overwrite?, columns:["colA","colB",...], data:[[...]], dtypes?:{colA:pandasDtype, ...}, formats?:{colA:numberFormat, ...}}`。Agents 通常用 `{**json.loads(df.to_json(orient="split")), "dtypes": df.dtypes.astype(str).to_dict()}` 一行构造。`dtypes` 值是 pandas dtype 字符串（`int64`、`float64`、`Int64`、`bool`、`boolean`、`datetime64[ns]`、`object`、...），CLI 端映射成内部 string/number/date/bool —— 省略 `dtypes` 时该列按文本写入（适合原始 CSV-shaped 数据）。`formats[col]` 是 Excel number_format 字符串（如 `#,##0.00`、`0.0%`、`yyyy-mm`）；缺省时 date 列用 `yyyy-mm-dd`，string 列用文本格式 `@`。 | +| `--sheets` | string + File + Stdin（复合 JSON） | xor | Typed 表格协议（pandas-DataFrame-shaped）JSON，与 `--dataframe` 互斥：顶层 sheets 数组，每项 `{name, start_cell?, mode?, header?, allow_overwrite?, columns:["colA","colB",...], data:[[...]], dtypes?:{colA:pandasDtype, ...}, formats?:{colA:numberFormat, ...}}`。Agents 通常用 `{**json.loads(df.to_json(orient="split")), "dtypes": df.dtypes.astype(str).to_dict()}` 一行构造。`dtypes` 值是 pandas dtype 字符串（`int64`、`float64`、`Int64`、`bool`、`boolean`、`datetime64[ns]`、`object`、...），CLI 端映射成内部 string/number/date/bool —— 省略 `dtypes` 时该列按文本写入（适合原始 CSV-shaped 数据）。`formats[col]` 是 Excel number_format 字符串（如 `#,##0.00`、`0.0%`、`yyyy-mm`）；缺省时 date 列用 `yyyy-mm-dd`，string 列用文本格式 `@`。 | +| `--dataframe` | string | xor | 单 sheet 类型保真表格的二进制入口，从一个 Arrow IPC 文件（即 Feather v2，pandas `df.to_feather()` 直接写出）读入，与 `--sheets` 互斥。用 `@` 传文件或 `-` 读二进制 stdin（同其他输入 flag 的约定）。Arrow 字节按原样读 —— 不做 TrimSpace / BOM strip，IPC magic 字节完整保留（区别于文本类输入 flag）。列类型从 Arrow schema 推导（int*/uint*/float* → number，date32/date64/timestamp → date，utf8/large_utf8 → string，bool → bool）；每列的 `number_format` 可写在 Arrow Field metadata 里（`pa.field("price", pa.float64(), metadata={b"number_format": b"$#,##0.00"})`）。子表走默认落点：名为 `Sheet1`（缺则新建），从 A1 起覆盖写并带表头。要换子表名 / 起始位置 / 写入方式，或要写多子表，请改用 `--sheets`。 | ## Schemas @@ -520,6 +521,41 @@ payload = {"sheets": [{ > **dtype 速查**：`int64`/`float64`（数值）、`Int64`（含空值的整数，nullable）、`bool`/`boolean`、`datetime64[ns]`（date，默认 `yyyy-mm-dd`）、`object`（string）。pandas dtype 字符串原样塞进 dtypes 即可，CLI 端按前缀匹配（`int*`/`uint*`/`Int*`/`float*` → number 等）。未识别 dtype 兜底为 string。 +#### `--dataframe`（Arrow IPC / Feather v2 二进制入口） + +`--dataframe` 与 `--sheets` 互斥、功能等价，但走二进制 wire——pandas `df.to_feather()` 写出的 Arrow IPC 文件直接喂 CLI，类型从 Arrow schema 自动恢复，**不用再手填 dtypes/formats**，也自动绕过 NaT / NaN / `datetime64[ns, tz]` 的 JSON 序列化坑。子表落点固定为 `Sheet1`、A1 起覆盖写、带表头；要换子表名 / 起始位置 / 多子表，回到 `--sheets` JSON 协议。 + +```bash +# 文件（cwd 相对路径；受 SafePath 沙箱约束，不接受绝对路径） +lark-cli sheets +table-put --url "<表URL>" --dataframe @./in.arrow +# stdin 二进制（不落盘） +python prepare.py | lark-cli sheets +table-put --url "<表URL>" --dataframe - +``` + +```python +import io, subprocess, pandas as pd +df = pd.DataFrame({"date": pd.to_datetime(["2024-01-15"]), "amount": [1234.5], "id": ["00123"]}) + +# 1) 文件 +df.to_feather("./in.arrow") # 写到当前目录 +subprocess.run(["lark-cli","sheets","+table-put","--url",URL,"--dataframe","@./in.arrow"], check=True) + +# 2) stdin（不落盘）—— pandas 写 BytesIO，subprocess 把 buf 灌进去 +buf = io.BytesIO(); df.to_feather(buf) +subprocess.run(["lark-cli","sheets","+table-put","--url",URL,"--dataframe","-"], + input=buf.getvalue(), check=True) +``` + +> 每列的 `number_format` 写在 Arrow Field metadata 里，CLI 端自动透传到飞书显示格式（千分位 / 百分比 / 自定义日期等）： +> ```python +> import pyarrow as pa, pyarrow.feather as feather +> table = pa.Table.from_pandas(df) +> schema = table.schema.set( +> table.schema.get_field_index("amount"), +> pa.field("amount", pa.float64(), metadata={b"number_format": b"#,##0.00"})) +> feather.write_feather(table.cast(schema), "./in.arrow") +> ``` + ### Validate / DryRun / Execute 约束 - `Validate`：XOR 公共四件套；`+cells-set` 的 `--cells` 必须能解析为 JSON 二维矩阵且行列数与 `--range` 完全一致；`+cells-set-style` 的样式 flag 至少一个非空（或带 `--border-styles`）；`+cells-set-image` 的 `--range` 必须是单 cell（起止 cell 相同）；`+csv-put` 的 `--csv` 必须能按 RFC 4180 解析；防爆参数上限校验。