Work on creating tests for rsync api

tools/rsync/algorithm.go

@@ -16,8 +16,6 @@ import (
 	"fmt"
 	"hash"
 	"io"
-
-	"kitty/tools/utils"
 )
 
 // If no BlockSize is specified in the RSync instance, this value is used.
@@ -47,14 +45,14 @@ type Operation struct {
 
 var bin = binary.LittleEndian
 
-func (self Operation) Serialize() string {
+func (self Operation) Serialize() []byte {
 	ans := make([]byte, 24+len(self.Data))
 	bin.PutUint32(ans, uint32(len(self.Data)))
 	bin.PutUint32(ans[4:], uint32(self.Type))
 	bin.PutUint64(ans[8:], self.BlockIndex)
 	bin.PutUint64(ans[16:], self.BlockIndexEnd)
 	copy(ans[24:], self.Data)
-	return utils.UnsafeBytesToString(ans)
+	return ans
 }
 
 func (self *Operation) Unserialize(data []byte) (n int, err error) {
@@ -74,9 +72,9 @@ func (self *Operation) Unserialize(data []byte) (n int, err error) {
 	}
 	self.BlockIndex = bin.Uint64(data[8:])
 	self.BlockIndexEnd = bin.Uint64(data[16:])
-	self.Data = data[24 : 24+dlen]
 	n = 24 + dlen
-	return
+	self.Data = data[24:n]
+	return n, nil
 }
 
 // Signature hash item generated from target.

tools/rsync/api.go

@@ -136,18 +136,12 @@ func (self *Patcher) StartDelta(delta_output io.Writer, delta_input io.ReadSeeke
 
 // Apply a chunk of delta data
 func (self *Patcher) UpdateDelta(data []byte) (err error) {
-	if len(self.unconsumed_delta_data) > 0 {
-		data = append(self.unconsumed_delta_data, data...)
-		self.unconsumed_delta_data = nil
-	}
-	consumed, err := self.update_delta(data)
+	self.unconsumed_delta_data = append(self.unconsumed_delta_data, data...)
+	consumed, err := self.update_delta(self.unconsumed_delta_data)
 	if err != nil {
 		return err
 	}
-	data = data[consumed:]
-	if len(data) > 0 {
-		self.unconsumed_delta_data = data
-	}
+	self.unconsumed_delta_data = utils.ShiftLeft(self.unconsumed_delta_data, consumed)
 	return
 }
 
@@ -189,7 +183,7 @@ func (self *Patcher) CreateSignature(src io.Reader, callback func([]byte) error)
 }
 
 // Create a serialized delta based on the previously loaded signature
-func (self *Differ) CreateDelta(src io.Reader, output_callback func(string) error) (err error) {
+func (self *Differ) CreateDelta(src io.Reader, output_callback func([]byte) error) (err error) {
 	if err = self.finish_signature_data(); err != nil {
 		return
 	}
@@ -204,26 +198,19 @@ func (self *Differ) CreateDelta(src io.Reader, output_callback func(string) erro
 
 // Add more external signature data
 func (self *Differ) AddSignatureData(data []byte) (err error) {
-	if len(self.unconsumed_signature_data) > 0 {
-		data = append(self.unconsumed_signature_data, data...)
-		self.unconsumed_signature_data = nil
-	}
+	self.unconsumed_signature_data = append(self.unconsumed_signature_data, data...)
 	if self.rsync.UniqueHasher == nil {
-		consumed, err := self.read_signature_header(data)
+		consumed, err := self.read_signature_header(self.unconsumed_signature_data)
 		if err != nil {
 			if consumed < 0 {
-				self.unconsumed_signature_data = data
 				return nil
 			}
 			return err
 		}
-		data = data[consumed:]
-	}
-	consumed := self.read_signature_blocks(data)
-	data = data[consumed:]
-	if len(data) > 0 {
-		self.unconsumed_signature_data = data
+		self.unconsumed_signature_data = utils.ShiftLeft(self.unconsumed_signature_data, consumed)
 	}
+	consumed := self.read_signature_blocks(self.unconsumed_signature_data)
+	self.unconsumed_signature_data = utils.ShiftLeft(self.unconsumed_signature_data, consumed)
 	return nil
 }
 

tools/rsync/api_test.go

@@ -0,0 +1,77 @@
+// License: GPLv3 Copyright: 2023, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package rsync
+
+import (
+	"bytes"
+	"fmt"
+	"kitty/tools/utils"
+	"kitty/tools/utils/random"
+	"testing"
+
+	"golang.org/x/exp/slices"
+)
+
+var _ = fmt.Print
+
+func TestRsyncRoundtrip(t *testing.T) {
+	src_data := make([]byte, 4*1024*1024)
+	random.Bytes(src_data)
+
+	random_patch := func(data []byte) (size int) {
+		if offset := random.Int(len(data)); offset < len(data) {
+			max_size := utils.Min(256, len(data)-offset)
+			if size = random.Int(max_size); size > 0 {
+				random.Bytes(data[offset : offset+size])
+			}
+		}
+		return
+	}
+
+	changed := slices.Clone(src_data)
+	for i := 0; i < 8; i++ {
+		random_patch(changed)
+	}
+
+	p := NewPatcher(int64(len(src_data)))
+	sigbuf := bytes.Buffer{}
+	if err := p.CreateSignature(bytes.NewReader(changed), func(p []byte) error { _, err := sigbuf.Write(p); return err }); err != nil {
+		t.Fatal(err)
+	}
+	d := NewDiffer()
+	if err := d.AddSignatureData(sigbuf.Bytes()); err != nil {
+		t.Fatal(err)
+	}
+	deltabuf := bytes.Buffer{}
+	if err := d.CreateDelta(bytes.NewReader(src_data), func(b []byte) error { _, err := deltabuf.Write(b); return err }); err != nil {
+		t.Fatal(err)
+	}
+	outputbuf := bytes.Buffer{}
+	p.StartDelta(&outputbuf, bytes.NewReader(src_data))
+	b := make([]byte, 30*1024)
+	for {
+		n, _ := deltabuf.Read(b)
+		if n <= 0 {
+			break
+		}
+		if err := p.UpdateDelta(b[:n]); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := p.FinishDelta(); err != nil {
+		t.Fatal(err)
+	}
+
+	output := outputbuf.Bytes()
+	if !bytes.Equal(src_data, output) {
+		first_diff := utils.Min(len(src_data), len(output))
+		for i := 0; i < first_diff; i++ {
+			if src_data[i] != output[i] {
+				first_diff = i
+				break
+			}
+		}
+		t.Fatalf("Patching failed: %d extra_bytes first different byte at: %d", len(outputbuf.Bytes())-len(src_data), first_diff)
+	}
+
+}

tools/utils/misc.go

@@ -219,6 +219,14 @@ func Concat[T any](slices ...[]T) []T {
 	return result
 }
 
+func ShiftLeft[T any](s []T, amt int) []T {
+	leftover := len(s) - amt
+	if leftover > 0 {
+		copy(s, s[amt:])
+	}
+	return s[:leftover]
+}
+
 func SetStructDefaults(v reflect.Value) (err error) {
 	for _, field := range reflect.VisibleFields(v.Type()) {
 		if defval := field.Tag.Get("default"); defval != "" {

tools/utils/random/random.go

@@ -0,0 +1,35 @@
+// License: GPLv3 Copyright: 2023, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package random
+
+import (
+	"crypto/rand"
+	"fmt"
+	"math/big"
+
+	"golang.org/x/exp/constraints"
+)
+
+var _ = fmt.Print
+
+// Return a random integer in the range [0, limit). limit must be > 0
+func Int[T constraints.Integer](limit T) T {
+	b := big.NewInt(int64(limit))
+	n, err := rand.Int(rand.Reader, b)
+	if err != nil {
+		panic(err)
+	}
+	return T(n.Uint64())
+}
+
+// Return one of items randomnly
+func Choice[T any](items ...T) T {
+	return items[Int(len(items))]
+}
+
+// Write randomn bytes into the provided slice
+func Bytes(b []byte) {
+	if _, err := rand.Read(b); err != nil {
+		panic(err)
+	}
+}