diff --git a/README.md b/README.md index 6177b8b..bf91586 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ It implements the [casync](https://github.com/systemd/casync) format and interop ## Where it fits -- **Updating devices in the field.** An appliance with A/B partitions seeds from the partition it is running and writes the new image straight to the other one, downloading only the chunks that aren't already on disk. An update interrupted by a dropped connection restarts without fetching the completed chunks again. +- **Updating devices in the field.** An appliance with A/B partitions seeds from the partition it is running and writes the new image straight to the other one, downloading only the chunks that aren't already on disk. An update interrupted by a dropped connection restarts without fetching the completed chunks again. A device with a single copy of the image can be updated in place instead, rearranging the data already on it and downloading only the chunks it lacks. - **Serving a fleet from one copy.** Each client works out for itself which chunks it is missing, so one set of static files serves every client, whichever version it is starting from. Hash-named chunks never change, so a CDN can cache them indefinitely, and a `chunk-server` with a local cache at each site means a chunk crosses the WAN once rather than once per machine. - **Using an image before it has downloaded.** `mount-index` exposes an index as a file over FUSE and fetches chunks as they are read, so a VM can boot from a disk image in a remote store. With `--cor-file` it fills a local sparse copy on the way. - **Keeping every build.** Publish each build to the same store and it grows by what changed, not by another full image. `prune` removes the chunks only old versions still reference. @@ -39,7 +39,7 @@ How much you save depends on the data: on how much changed between versions, and | | desync | casync | rsync | zsync | OCI / ORAS | | --- | --- | --- | --- | --- | --- | -| Reuses local data as a seed | yes | yes | yes, the destination file | yes | no | +| Reuses local data as a seed | yes, including the destination file | yes | yes, the destination file | yes | no | | Server-side work per client | none | none | delta computed per transfer | none | none | | Server requirement | any static file host | any static file host | rsync daemon or SSH | static host with range requests | registry | | Deduplication across versions in the store | yes | yes | no | no | whole layers only | diff --git a/assemble-backfill.go b/assemble-backfill.go new file mode 100644 index 0000000..c77a28e --- /dev/null +++ b/assemble-backfill.go @@ -0,0 +1,118 @@ +package desync + +import ( + "bytes" + "fmt" + "os" + "sync" + "time" +) + +// seedFileState is what a seed file looked like when its chunks were +// validated. Comparing it with the file after assembly shows whether the file +// was written to while it was read. +type seedFileState struct { + info os.FileInfo + + // recent is set when the file was modified shortly before it was + // validated. Some filesystems keep modification times to the second or + // worse, a later write could then leave the time unchanged. + recent bool +} + +// seedTimeResolution is the coarsest resolution of modification times among +// common filesystems, FAT's two seconds. +const seedTimeResolution = 2 * time.Second + +func newSeedFileState(info os.FileInfo) seedFileState { + return seedFileState{ + info: info, + recent: time.Since(info.ModTime()) < seedTimeResolution, + } +} + +// changed reports whether the file may have been written to since its state +// was recorded. A block device has no meaningful modification time, it always +// counts as changed. +func (s seedFileState) changed(name string) bool { + if s.recent || isDevice(s.info.Mode()) { + return true + } + info, err := os.Stat(name) + if err != nil { + return true + } + return !os.SameFile(info, s.info) || + info.Size() != s.info.Size() || + !info.ModTime().Equal(s.info.ModTime()) +} + +// chunksInTarget reports for every chunk of the index whether the file holds +// its data at the chunk's position, hashing them with n goroutines. Null +// chunks are compared with zeros instead, which is much cheaper than hashing +// in images that are mostly empty space. The progress is counted in chunks. +func chunksInTarget(name string, idx Index, n int, pb ProgressBar) ([]bool, error) { + f, err := os.Open(name) + if err != nil { + return nil, err + } + defer f.Close() + + pb.SetTotal(len(idx.Chunks)) + pb.Start() + defer pb.Finish() + + null := NewNullChunk(idx.Index.ChunkSizeMax) + found := make([]bool, len(idx.Chunks)) + var wg sync.WaitGroup + work := make(chan int) + for range max(n, 1) { + wg.Go(func() { + buf := make([]byte, idx.Index.ChunkSizeMax) + for i := range work { + c := idx.Chunks[i] + if c.ID != null.ID { + found[i] = chunkInPlace(f, c, buf) + } else { + b := buf[:c.Size] + _, err := f.ReadAt(b, int64(c.Start)) + found[i] = err == nil && bytes.Equal(b, null.Data) + } + pb.Increment() + } + }) + } + for i := range idx.Chunks { + work <- i + } + close(work) + wg.Wait() + return found, nil +} + +// backfill checks every chunk of the assembled file and writes the ones that +// don't match from the store. It's used when a seed changed while it was read, +// whatever came from it may be damaged, including what the self-seed copied +// from there on. +func backfill(name string, idx Index, s Store, n int, stats *ExtractStats) error { + found, err := chunksInTarget(name, idx, n, NewProgressBar("Verifying ")) + if err != nil { + return err + } + f, err := os.OpenFile(name, os.O_RDWR, 0) + if err != nil { + return err + } + defer f.Close() + for i, ok := range found { + if ok { + continue + } + c := idx.Chunks[i] + if _, _, err := (©FromStore{store: s, chunk: c}).Execute(f); err != nil { + return fmt.Errorf("chunk %s at offset %d came from a seed that changed during assembly: %w", c.ID, c.Start, err) + } + stats.addChunksBackfilled(1) + } + return nil +} diff --git a/assemble-backfill_test.go b/assemble-backfill_test.go new file mode 100644 index 0000000..316d86e --- /dev/null +++ b/assemble-backfill_test.go @@ -0,0 +1,190 @@ +package desync + +import ( + "context" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// writeOldFile writes a file whose modification time lies well in the past, +// so only a later write can make it look changed. +func writeOldFile(t *testing.T, name string, data []byte) { + t.Helper() + require.NoError(t, os.WriteFile(name, data, 0644)) + past := time.Now().Add(-time.Hour) + require.NoError(t, os.Chtimes(name, past, past)) +} + +func TestSeedFileStateChanged(t *testing.T) { + // Record the file the way validation does, from the open file. On + // Windows, the identity of a file stat'ed by name is looked up by name + // again when it's compared, which would find the replacement. + record := func(t *testing.T, name string) seedFileState { + t.Helper() + f, err := os.Open(name) + require.NoError(t, err) + defer f.Close() + info, err := f.Stat() + require.NoError(t, err) + return newSeedFileState(info) + } + dir := t.TempDir() + + t.Run("unchanged", func(t *testing.T) { + name := filepath.Join(dir, "unchanged") + writeOldFile(t, name, []byte("seed data")) + assert.False(t, record(t, name).changed(name)) + }) + t.Run("written to", func(t *testing.T) { + name := filepath.Join(dir, "written") + writeOldFile(t, name, []byte("seed data")) + state := record(t, name) + require.NoError(t, os.WriteFile(name, []byte("SEED DATA"), 0644)) + assert.True(t, state.changed(name)) + }) + t.Run("replaced", func(t *testing.T) { + name := filepath.Join(dir, "replaced") + writeOldFile(t, name, []byte("seed data")) + state := record(t, name) + other := filepath.Join(dir, "other") + writeOldFile(t, other, []byte("seed data")) + require.NoError(t, os.Chtimes(other, state.info.ModTime(), state.info.ModTime())) + require.NoError(t, os.Rename(other, name)) + assert.True(t, state.changed(name), "same size and time, different file") + }) + t.Run("removed", func(t *testing.T) { + name := filepath.Join(dir, "removed") + writeOldFile(t, name, []byte("seed data")) + state := record(t, name) + require.NoError(t, os.Remove(name)) + assert.True(t, state.changed(name)) + }) + t.Run("modified just before", func(t *testing.T) { + // A write within the time resolution of the filesystem might not + // change the modification time + name := filepath.Join(dir, "recent") + require.NoError(t, os.WriteFile(name, []byte("seed data"), 0644)) + assert.True(t, record(t, name).changed(name)) + }) +} + +// hookStore calls a function before handing out a chunk. +type hookStore struct { + Store + hook func() +} + +func (s hookStore) GetChunk(id ChunkID) (*Chunk, error) { + s.hook() + return s.Store.GetChunk(id) +} + +// A seed that's written to during assembly is copied into the target in its +// new state. The damaged chunks are found after assembly and taken from the +// store instead. +func TestAssembleSeedChangedDuringAssembly(t *testing.T) { + chunks := randomChunks(1024, 768) + fromStore, fromSeed := chunks[0], chunks[1] + + // The store chunk comes first. With one worker it's written before the + // seed chunk, and fetching it overwrites the seed with other data of the + // same size. + setup := func(t *testing.T, store Store) (string, *ExtractStats, error) { + t.Helper() + dir := t.TempDir() + seedFile := filepath.Join(dir, "seed") + writeOldFile(t, seedFile, fromSeed.data) + seed, err := NewFileSeed(filepath.Join(dir, "target"), seedFile, chunkIndex(fromSeed), IndexRegenerationSettings{}) + require.NoError(t, err) + + // The hook runs on a worker, it can't stop the test + var once sync.Once + hooked := hookStore{Store: store, hook: func() { + once.Do(func() { + assert.NoError(t, os.WriteFile(seedFile, make([]byte, len(fromSeed.data)), 0644)) + }) + }} + target := filepath.Join(dir, "target") + stats, err := AssembleFile(context.Background(), target, chunkIndex(fromStore, fromSeed), hooked, + []Seed{seed}, AssembleOptions{N: 1, InvalidSeedAction: InvalidSeedActionBailOut}) + return target, stats, err + } + + t.Run("backfilled from the store", func(t *testing.T) { + target, stats, err := setup(t, chunkStore(fromStore, fromSeed)) + require.NoError(t, err) + assert.Equal(t, uint64(1), stats.ChunksFromSeeds) + assert.Equal(t, uint64(1), stats.ChunksBackfilled) + + got, err := os.ReadFile(target) + require.NoError(t, err) + require.Equal(t, chunkContent(fromStore, fromSeed), got) + }) + + t.Run("not in the store", func(t *testing.T) { + _, _, err := setup(t, chunkStore(fromStore)) + require.ErrorContains(t, err, "came from a seed that changed during assembly") + }) +} + +// Without a change, the output isn't checked again. +func TestAssembleSeedUnchanged(t *testing.T) { + chunks := randomChunks(1024, 768) + fromStore, fromSeed := chunks[0], chunks[1] + dir := t.TempDir() + seedFile := filepath.Join(dir, "seed") + writeOldFile(t, seedFile, fromSeed.data) + target := filepath.Join(dir, "target") + seed, err := NewFileSeed(target, seedFile, chunkIndex(fromSeed), IndexRegenerationSettings{}) + require.NoError(t, err) + + // Checking the output would fetch nothing, so count the chunks the + // store hands out instead: only the one assembly needs. + var fetched atomic.Int32 + store := hookStore{Store: chunkStore(fromStore, fromSeed), hook: func() { fetched.Add(1) }} + stats, err := AssembleFile(context.Background(), target, chunkIndex(fromStore, fromSeed), store, + []Seed{seed}, testAssembleOptions) + require.NoError(t, err) + assert.Equal(t, uint64(0), stats.ChunksBackfilled) + assert.Equal(t, int32(1), fetched.Load()) +} + +func TestChunksInTargetNullChunks(t *testing.T) { + data := randomChunks(64)[0] + null := testChunk{id: NewNullChunk(64).ID, data: make([]byte, 64)} + idx := chunkIndex(null, data, null) + + tests := map[string]struct { + content []byte + expected []bool + }{ + "zeros in place": { + content: chunkContent(null, data, null), + expected: []bool{true, true, true}, + }, + "one byte set": { + content: join(make([]byte, 63), []byte{1}, data.data, make([]byte, 64)), + expected: []bool{false, true, true}, + }, + "file too short": { + content: chunkContent(null, data, null)[:150], + expected: []bool{true, true, false}, + }, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + name := filepath.Join(t.TempDir(), "target") + require.NoError(t, os.WriteFile(name, test.content, 0644)) + found, err := chunksInTarget(name, idx, 2, NullProgressBar{}) + require.NoError(t, err) + require.Equal(t, test.expected, found) + }) + } +} diff --git a/assemble-fileseed.go b/assemble-fileseed.go new file mode 100644 index 0000000..dd6e33f --- /dev/null +++ b/assemble-fileseed.go @@ -0,0 +1,144 @@ +package desync + +import ( + "context" + "fmt" + "os" + + "github.com/folbricht/desync/pkg/chunkers" +) + +var _ Seed = &FileSeed{} + +// Controls how to regenerate invalid index. +type IndexRegenerationSettings struct { + chunkers.ChunkingSettings +} + +// FileSeed is used to copy or clone blocks from an existing index+blob during +// file extraction. +type FileSeed struct { + srcFile string + indexRegenSettings IndexRegenerationSettings + seedIndex +} + +// NewFileSeed initializes a new seed that uses an existing index and its blob +func NewFileSeed(dstFile string, srcFile string, index Index, regenSettings IndexRegenerationSettings) (*FileSeed, error) { + // Make sure chunking sizes match with provided index. + // TODO: would be nice if IndexRegenerationSettings could be restored from index entirely. + regenSettings.ChunkerParams.Min = index.Index.ChunkSizeMin + regenSettings.ChunkerParams.Avg = index.Index.ChunkSizeAvg + regenSettings.ChunkerParams.Max = index.Index.ChunkSizeMax + return &FileSeed{ + srcFile: srcFile, + indexRegenSettings: regenSettings, + seedIndex: newSeedIndex(index, CanClone(dstFile, srcFile)), + }, nil +} + +// LongestMatchFrom returns the longest sequence of chunks anywhere in the seed +// that match chunks starting at chunks[startPos]. It returns the chunk +// position of the match in the seed and the number of matching chunks, or +// (0, 0) if there is no match. +func (s *FileSeed) LongestMatchFrom(chunks []IndexChunk, startPos int) (int, int) { + return s.longestMatchFrom(chunks, startPos, 0, 0, nil) +} + +// GetSegment constructs a SeedSegment for n chunks starting at chunk position +// pos in the seed. +func (s *FileSeed) GetSegment(pos, n int) SeedSegment { + return newFileSeedSegment(s.srcFile, s.index.Chunks[pos:pos+n], s.canReflink) +} + +func (s *FileSeed) RegenerateIndex(ctx context.Context, n int, attempt int, seedNumber int) error { + chunkingPrefix := fmt.Sprintf("Attempt %d: Chunking Seed %d ", attempt, seedNumber) + index, _, err := IndexFromFile(ctx, s.srcFile, n, s.indexRegenSettings.ChunkingSettings, NewProgressBar(chunkingPrefix)) + if err != nil { + return err + } + + s.seedIndex = newSeedIndex(index, s.canReflink) + return nil +} + +type fileSeedSegment struct { + file string + chunks []IndexChunk + canReflink bool +} + +func newFileSeedSegment(file string, chunks []IndexChunk, canReflink bool) *fileSeedSegment { + return &fileSeedSegment{ + canReflink: canReflink, + file: file, + chunks: chunks, + } +} + +func (s *fileSeedSegment) FileName() string { + return s.file +} + +func (s *fileSeedSegment) Size() uint64 { + return chunkRangeLength(s.chunks) +} + +func (s *fileSeedSegment) WriteInto(dst *os.File, offset, length, blocksize uint64, isBlank bool) (uint64, uint64, error) { + if length != s.Size() { + return 0, 0, fmt.Errorf("unable to copy %d bytes from %s to %s : wrong size", length, s.file, dst.Name()) + } + src, err := os.Open(s.file) + if err != nil { + return 0, 0, err + } + defer src.Close() + + // Do a straight copy if reflinks are not supported or blocks aren't aligned + if !s.canReflink || s.chunks[0].Start%blocksize != offset%blocksize { + return copyRange(dst, src, s.chunks[0].Start, length, offset) + } + return cloneOrCopyRange(dst, src, s.chunks[0].Start, length, offset, blocksize) +} + +// Validate compares all chunks in this slice of the seed index to the underlying data +// and fails if they don't match. +func (s *fileSeedSegment) Validate(file *os.File) error { + if !chunksInPlace(file, s.chunks) { + return fmt.Errorf("seed index for %s doesn't match its data", s.file) + } + return nil +} + +type fileSeedSource struct { + segment SeedSegment + seed Seed + offset uint64 + length uint64 + blocksize uint64 + isBlank bool +} + +func (s *fileSeedSource) Execute(f *os.File) (copied uint64, cloned uint64, err error) { + return s.segment.WriteInto(f, s.offset, s.length, s.blocksize, s.isBlank) +} + +// access is only the write, the seed is another file than the target. +func (s *fileSeedSource) access() targetAccess { + return targetAccess{writes: byteRange{s.offset, s.offset + s.length}} +} + +func (s *fileSeedSource) recordStats(stats *ExtractStats, numChunks int) { + stats.addChunksFromSeed(uint64(numChunks)) +} + +func (s *fileSeedSource) Seed() Seed { return s.seed } +func (s *fileSeedSource) File() string { return s.segment.FileName() } + +func (s *fileSeedSource) Validate(file *os.File) error { + return s.segment.Validate(file) +} + +func (s *fileSeedSource) String() string { + return fmt.Sprintf("FileSeed(%s): Copy to [%d:%d]", s.segment.FileName(), s.offset, s.offset+s.length) +} diff --git a/assemble-inplacecopy.go b/assemble-inplacecopy.go new file mode 100644 index 0000000..9954be3 --- /dev/null +++ b/assemble-inplacecopy.go @@ -0,0 +1,166 @@ +package desync + +import ( + "fmt" + "os" +) + +// inPlaceCopy moves a chunk, or a run of consecutive chunks, from one position +// to another within the same file. It uses ReadAt/WriteAt (pread/pwrite) which +// are position-independent and safe for concurrent use on the same file handle. +type inPlaceCopy struct { + // chunks are the source chunks in the file, in order and without gaps + // between them. A move covers several of them when one shift moves them + // all, in which case its source and destination don't overlap. + chunks []IndexChunk + dstOffset uint64 + seed Seed + file string + + // clone is set for moves whose blocks can be cloned rather than + // copied: the file supports reflinks, the source and destination have + // the same alignment and don't overlap, which the kernel requires within + // one file. A clone keeps its data when the source is overwritten later, + // so it takes the place of reading the source without changing when the + // move has to run. Only runs of several chunks are cloned. Cloning chunk + // by chunk locks and flushes the file for every one of them, which is + // slower than copying and leaves the file in thousands of extents. + clone bool + blocksize uint64 + + // Moves that are part of a dependency cycle hold their source in a + // stash once other steps are about to overwrite it. If the stash was + // dropped for lack of memory, the chunk is taken from the store. Only + // moves of a single chunk are stashed. + stash *stash + store Store +} + +// srcOffset is where the move reads its data. +func (s *inPlaceCopy) srcOffset() uint64 { return s.chunks[0].Start } + +// size is how much data the move writes. +func (s *inPlaceCopy) size() uint64 { return chunkRangeLength(s.chunks) } + +func (s *inPlaceCopy) Execute(f *os.File) (copied uint64, cloned uint64, err error) { + // Write from the stash, the source may have been overwritten since. + if s.stash != nil { + data, err := s.stash.take(f) + if err != nil { + return 0, 0, err + } + if data == nil { + return s.fromStore().Execute(f) + } + if _, err := f.WriteAt(data, int64(s.dstOffset)); err != nil { + return 0, 0, fmt.Errorf("inPlaceCopy stash write at %d: %w", s.dstOffset, err) + } + return s.size(), 0, nil + } + + if s.clone { + copied, cloned, err := s.cloneBlocks(f) + if err == nil { + return copied, cloned, nil + } + // Not every filesystem that passes the CanClone probe can clone every + // range. Nothing was written yet, fall back to copying the data. + } + + // Normal copy — read the source into a temp buffer, then write it to the + // destination. Always buffer first to handle overlapping ranges safely. + // One chunk at a time, so a move of a long run doesn't hold all of it in + // memory; the ranges of a run don't overlap, its chunks can be moved in + // any order. + buf := getChunkBuf(maxChunkSize(s.chunks)) + defer chunkBufPool.Put(buf) + offset := s.dstOffset + for _, c := range s.chunks { + b := (*buf)[:c.Size] + if _, err := f.ReadAt(b, int64(c.Start)); err != nil { + return 0, 0, fmt.Errorf("inPlaceCopy read at %d: %w", c.Start, err) + } + if _, err := f.WriteAt(b, int64(offset)); err != nil { + return 0, 0, fmt.Errorf("inPlaceCopy write at %d: %w", offset, err) + } + offset += c.Size + } + return s.size(), 0, nil +} + +// cloneBlocks clones the whole blocks of the move and copies the unaligned +// head and tail. The blocks are cloned first, so nothing is written if that +// fails. Plain reads and writes copy the rest, the source and destination +// don't overlap and one file handle serves as both. +func (s *inPlaceCopy) cloneBlocks(f *os.File) (copied uint64, cloned uint64, err error) { + src, size, bs := s.srcOffset(), s.size(), s.blocksize + lo := (src/bs + 1) * bs // block boundary after src, as in cloneOrCopyRange + hi := (src + size) / bs * bs // last block boundary + if err := cloneRange(f, f, lo, hi-lo, s.dstOffset+(lo-src)); err != nil { + return 0, 0, err + } + buf := getChunkBuf(bs) + defer chunkBufPool.Put(buf) + for _, r := range [][2]uint64{{src, lo}, {hi, src + size}} { + b := (*buf)[:r[1]-r[0]] + if _, err := f.ReadAt(b, int64(r[0])); err != nil { + return 0, 0, fmt.Errorf("inPlaceCopy read at %d: %w", r[0], err) + } + if _, err := f.WriteAt(b, int64(s.dstOffset+(r[0]-src))); err != nil { + return 0, 0, fmt.Errorf("inPlaceCopy write at %d: %w", s.dstOffset+(r[0]-src), err) + } + copied += uint64(len(b)) + } + return copied, hi - lo, nil +} + +// access reads the content from before assembly, possibly into a stash. +func (s *inPlaceCopy) access() targetAccess { + src, size := s.srcOffset(), s.size() + return targetAccess{ + writes: byteRange{s.dstOffset, s.dstOffset + size}, + reads: byteRange{src, src + size}, + readsOld: true, + stash: s.stash, + } +} + +func (s *inPlaceCopy) recordStats(stats *ExtractStats, numChunks int) { + if s.stash != nil && s.stash.dropped() { + s.fromStore().recordStats(stats, numChunks) + return + } + stats.addChunksInPlace(uint64(numChunks)) +} + +// fromStore returns a step writing the chunk from the store, used when the +// source wasn't held in memory. Only moves of a single chunk are stashed, so +// there's just the one chunk to write. +func (s *inPlaceCopy) fromStore() *copyFromStore { + return ©FromStore{ + store: s.store, + chunk: IndexChunk{ID: s.chunks[0].ID, Start: s.dstOffset, Size: s.size()}, + } +} + +func (s *inPlaceCopy) String() string { + var from string + if s.stash != nil { + from = " from stash" + } + src, size := s.srcOffset(), s.size() + return fmt.Sprintf("InPlace: Copy [%d:%d] to [%d:%d]%s", + src, src+size, s.dstOffset, s.dstOffset+size, from) +} + +func (s *inPlaceCopy) Seed() Seed { return s.seed } +func (s *inPlaceCopy) File() string { return s.file } + +// Validate confirms that the source of the move holds the chunks the in-place +// seed index claims are there. +func (s *inPlaceCopy) Validate(file *os.File) error { + if !chunksInPlace(file, s.chunks) { + return fmt.Errorf("in-place seed index for %s doesn't match its data", s.file) + } + return nil +} diff --git a/assemble-inplacecopy_test.go b/assemble-inplacecopy_test.go new file mode 100644 index 0000000..c7a39bb --- /dev/null +++ b/assemble-inplacecopy_test.go @@ -0,0 +1,141 @@ +package desync + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Moves are cloned when the file supports it, they cover a run of several +// chunks, and their source and destination have the same alignment and don't +// overlap. +func TestInPlaceMoveClones(t *testing.T) { + a := filledChunk(256, 0x0A) + b := filledChunk(256, 0x0B) + pad := func(size int) testChunk { return filledChunk(size, 0xEE) } + + // moveClones plans the target over a file holding the in-place chunks and + // reports whether each move clones, by its string. + moveClones := func(t *testing.T, canReflink bool, inPlace []testChunk, target ...testChunk) map[string]bool { + t.Helper() + f := writeChunkFile(t, inPlace...) + seed, err := NewFileSeed(f, f, chunkIndex(inPlace...), IndexRegenerationSettings{}) + require.NoError(t, err) + seed.canReflink = canReflink + plan := newPlan(f, chunkIndex(target...), nil, + planWithInPlaceSeed(seed), planWithTargetIsBlank(false), planWithBlocksize(64)) + clones := make(map[string]bool) + for _, step := range plan.Steps() { + if c, ok := step.source.(*inPlaceCopy); ok { + clones[c.String()] = c.clone + } + } + return clones + } + + t.Run("aligned run", func(t *testing.T) { + got := moveClones(t, true, []testChunk{a, b}, pad(512), a, b) + require.Equal(t, map[string]bool{"InPlace: Copy [0:512] to [512:1024]": true}, got) + }) + t.Run("without reflink support", func(t *testing.T) { + got := moveClones(t, false, []testChunk{a, b}, pad(512), a, b) + require.Equal(t, map[string]bool{"InPlace: Copy [0:512] to [512:1024]": false}, got) + }) + t.Run("overlapping", func(t *testing.T) { + // Shifted by 64 bytes, less than a chunk, so the chunks move one by + // one and each overlaps its own source + got := moveClones(t, true, []testChunk{a, b}, pad(64), a, b) + require.Equal(t, map[string]bool{ + "InPlace: Copy [0:256] to [64:320]": false, + "InPlace: Copy [256:512] to [320:576]": false, + }, got) + }) + t.Run("different alignment", func(t *testing.T) { + got := moveClones(t, true, []testChunk{a, b}, pad(600), a, b) + require.Equal(t, map[string]bool{"InPlace: Copy [0:512] to [600:1112]": false}, got) + }) + t.Run("single chunk", func(t *testing.T) { + // Aligned and apart, but a swap moves the chunks one by one + got := moveClones(t, true, []testChunk{a, b}, b, a) + require.Equal(t, map[string]bool{ + "InPlace: Copy [256:512] to [0:256]": false, + "InPlace: Copy [0:256] to [256:512] from stash": false, + }, got) + }) +} + +// A cloned move clones its whole blocks and copies the rest, and copies all of +// it if the filesystem refuses the clone. +func TestInPlaceMoveCloneBlocks(t *testing.T) { + // 1000 bytes at 100 move to 2148, both 36 bytes past a 64 byte block + data := make([]byte, 4096) + for i := range data { + data[i] = byte(i % 251) + } + want := append([]byte(nil), data...) + copy(want[2148:], data[100:1100]) + + move := func(t *testing.T) (*inPlaceCopy, *os.File) { + t.Helper() + name := filepath.Join(t.TempDir(), "target") + require.NoError(t, os.WriteFile(name, data, 0644)) + f, err := os.OpenFile(name, os.O_RDWR, 0) + require.NoError(t, err) + t.Cleanup(func() { _ = f.Close() }) + return &inPlaceCopy{ + chunks: []IndexChunk{{Start: 100, Size: 1000}}, + dstOffset: 2148, + clone: true, + blocksize: 64, + }, f + } + contents := func(t *testing.T, f *os.File) []byte { + t.Helper() + got := make([]byte, len(data)) + _, err := f.ReadAt(got, 0) + require.NoError(t, err) + return got + } + stubClone := func(t *testing.T, clone func(dst, src *os.File, srcOffset, srcLength, dstOffset uint64) error) { + orig := cloneRange + cloneRange = clone + t.Cleanup(func() { cloneRange = orig }) + } + + t.Run("clone", func(t *testing.T) { + var calls [][3]uint64 + stubClone(t, func(dst, src *os.File, srcOffset, srcLength, dstOffset uint64) error { + calls = append(calls, [3]uint64{srcOffset, srcLength, dstOffset}) + // Stand in for the clone by copying the range + b := make([]byte, srcLength) + if _, err := src.ReadAt(b, int64(srcOffset)); err != nil { + return err + } + _, err := dst.WriteAt(b, int64(dstOffset)) + return err + }) + s, f := move(t) + copied, cloned, err := s.Execute(f) + require.NoError(t, err) + assert.Equal(t, [][3]uint64{{128, 960, 2176}}, calls, "cloned range") + assert.Equal(t, uint64(960), cloned) + assert.Equal(t, uint64(40), copied, "unaligned head and tail") + require.Equal(t, want, contents(t, f)) + }) + + t.Run("clone refused", func(t *testing.T) { + stubClone(t, func(dst, src *os.File, srcOffset, srcLength, dstOffset uint64) error { + return errors.New("simulated clone failure") + }) + s, f := move(t) + copied, cloned, err := s.Execute(f) + require.NoError(t, err) + assert.Equal(t, uint64(0), cloned) + assert.Equal(t, uint64(1000), copied) + require.Equal(t, want, contents(t, f)) + }) +} diff --git a/assemble-plan-order_test.go b/assemble-plan-order_test.go new file mode 100644 index 0000000..d8f3449 --- /dev/null +++ b/assemble-plan-order_test.go @@ -0,0 +1,183 @@ +package desync + +import ( + "fmt" + "math/rand/v2" + "os" + "path/filepath" + "slices" + "testing" + + "github.com/stretchr/testify/require" +) + +// stepAccess is what a step does to the target file: the range it writes and +// the range it reads, which holds either the content from before assembly or +// what other steps write during it. It's worked out from the sources here +// rather than taken from their access() methods, which the plan orders the +// steps by, so that a wrong declaration shows up as a wrong order. +type stepAccess struct { + writes, reads byteRange + readsOld bool + stash *stash +} + +func accessOf(t *testing.T, step *planStep) stepAccess { + t.Helper() + switch s := step.source.(type) { + case *skipInPlace: + return stepAccess{} // the data is there already + case *copyFromStore: + return stepAccess{writes: byteRange{s.chunk.Start, s.chunk.Start + s.chunk.Size}} + case *fileSeedSource: + return stepAccess{writes: byteRange{s.offset, s.offset + s.length}} + case *selfSeedSegment: + src, size := s.segment.chunks[0].Start, s.segment.Size() + return stepAccess{ + writes: byteRange{s.dstOffset, s.dstOffset + size}, + reads: byteRange{src, src + size}, + } + case *inPlaceCopy: + src, size := s.srcOffset(), s.size() + return stepAccess{ + writes: byteRange{s.dstOffset, s.dstOffset + size}, + reads: byteRange{src, src + size}, + readsOld: true, + stash: s.stash, + } + } + t.Fatalf("unknown source %T, add what it reads and writes to accessOf", step.source) + return stepAccess{} +} + +// requirePlanOrder checks that the steps of a plan get the data they expect +// whichever order the dispatcher runs them in: +// - the dependencies have no cycle, all steps can be handed out +// - no two steps write the same bytes +// - a step that reads what others write during assembly runs after them +// - a step that reads the content from before assembly runs before any +// other step overwrites it, or that step fills the reader's stash first +func requirePlanOrder(t *testing.T, steps []*planStep, msgAndArgs ...any) { + t.Helper() + fail := func(format string, args ...any) { + t.Helper() + msg := fmt.Sprintf(format, args...) + if len(msgAndArgs) > 0 { + msg += " for " + fmt.Sprintf(msgAndArgs[0].(string), msgAndArgs[1:]...) + } + require.Fail(t, msg) + } + index := make(map[*planStep]int, len(steps)) + for i, s := range steps { + index[s] = i + } + + // Sort the steps topologically. Every step comes out if there's no cycle. + inDegree := make([]int, len(steps)) + var order []int + for i, s := range steps { + inDegree[i] = len(s.dependencies) + if inDegree[i] == 0 { + order = append(order, i) + } + } + for k := 0; k < len(order); k++ { + for d := range steps[order[k]].dependents { + j := index[d] + if inDegree[j]--; inDegree[j] == 0 { + order = append(order, j) + } + } + } + if len(order) < len(steps) { + var stuck []string + for i, d := range inDegree { + if d > 0 { + stuck = append(stuck, steps[i].source.String()) + } + } + fail("dependency cycle, these steps can never run: %q", stuck) + } + + // before[i][j] is set when step j always completes before step i starts + before := make([][]bool, len(steps)) + for _, i := range order { + before[i] = make([]bool, len(steps)) + for d := range steps[i].dependencies { + j := index[d] + before[i][j] = true + for k, b := range before[j] { + before[i][k] = before[i][k] || b + } + } + } + + access := make([]stepAccess, len(steps)) + for i, s := range steps { + access[i] = accessOf(t, s) + } + for i, a := range access { + for j, b := range access { + if i == j { + continue + } + if i < j && a.writes.overlaps(b.writes) { + fail("%q and %q both write the same bytes", steps[i].source, steps[j].source) + } + if !a.reads.overlaps(b.writes) { + continue + } + switch { + case !a.readsOld && !before[i][j]: + fail("%q reads what %q writes, but can run first", steps[i].source, steps[j].source) + case a.readsOld && !before[j][i] && (a.stash == nil || !slices.Contains(steps[j].fills, a.stash)): + fail("%q can overwrite what %q reads before it's read or stashed", steps[j].source, steps[i].source) + } + } + } +} + +// Plans for random layouts, with an in-place seed, a file seed and data that +// repeats within the target, all order their steps correctly. +func TestPlanOrderRandomized(t *testing.T) { + rng := rand.New(rand.NewPCG(3, 4)) + pool := make([]testChunk, 8) + for i := range pool { + pool[i] = filledChunk(64*(rng.IntN(8)+1), byte(i+1)) + } + layout := func(maxLen int) []int { + l := make([]int, rng.IntN(maxLen)+1) + for i := range l { + l[i] = rng.IntN(len(pool)) + } + return l + } + + dir := t.TempDir() + for i := range 2000 { + oldLayout, seedLayout, newLayout := layout(12), layout(6), layout(12) + reflink := rng.IntN(2) == 0 + old := pickChunks(pool, oldLayout...) + name := filepath.Join(dir, fmt.Sprintf("target%d", i)) + require.NoError(t, os.WriteFile(name, chunkContent(old...), 0644)) + inPlace, err := NewFileSeed(name, name, chunkIndex(old...), IndexRegenerationSettings{}) + require.NoError(t, err) + + seedChunks := pickChunks(pool, seedLayout...) + seedName := filepath.Join(dir, fmt.Sprintf("seed%d", i)) + require.NoError(t, os.WriteFile(seedName, chunkContent(seedChunks...), 0644)) + seed, err := NewFileSeed(name, seedName, chunkIndex(seedChunks...), IndexRegenerationSettings{}) + require.NoError(t, err) + inPlace.canReflink, seed.canReflink = reflink, reflink + + plan := newPlan(name, chunkIndex(pickChunks(pool, newLayout...)...), nil, + planWithInPlaceSeed(inPlace), planWithSeeds([]Seed{seed}), + planWithTargetIsBlank(false), planWithBlocksize(64)) + if reflink { + plan.selfSeed.canReflink = true + plan = plan.replan() + } + requirePlanOrder(t, plan.Steps(), "old %v, seed %v, new %v, reflink %v", + oldLayout, seedLayout, newLayout, reflink) + } +} diff --git a/assemble-plan.go b/assemble-plan.go new file mode 100644 index 0000000..835a6c8 --- /dev/null +++ b/assemble-plan.go @@ -0,0 +1,888 @@ +package desync + +import ( + "errors" + "fmt" + "math" + "os" + "slices" + "sort" + "sync" + + "golang.org/x/sync/errgroup" +) + +type planOption func(*assemblePlan) + +func planWithConcurrency(n int) planOption { + return func(p *assemblePlan) { + p.concurrency = n + } +} + +// planWithSeeds sets the file seeds available to the plan. A seed reading +// from the target file itself must be set with planWithInPlaceSeed instead. +func planWithSeeds(seeds []Seed) planOption { + return func(p *assemblePlan) { + p.seeds = seeds + } +} + +// planWithInPlaceSeed sets the seed whose source is the target file itself. +// It is used to skip or rearrange data already present in the target. +func planWithInPlaceSeed(seed *FileSeed) planOption { + return func(p *assemblePlan) { + p.inPlaceSeed = seed + } +} + +// planWithBlocksize sets the blocksize of the target, used for reflinking. +// It's determined from the target if not set. +func planWithBlocksize(blocksize uint64) planOption { + return func(p *assemblePlan) { + p.blocksize = blocksize + } +} + +// planWithStashLimit sets the memory the stashes of in-place moves may hold +// at the same time. Moves whose stash doesn't fit take their chunk from the +// store instead. It's unbounded if not set. +func planWithStashLimit(budget int64) planOption { + return func(p *assemblePlan) { + p.stashLimit = budget + } +} + +func planWithTargetIsBlank(isBlank bool) planOption { + return func(p *assemblePlan) { + p.targetIsBlank = isBlank + } +} + +// assemblePlan holds a directed acyclic graph of steps. +type assemblePlan struct { + idx Index + concurrency int + target string + store Store + seeds []Seed + inPlaceSeed *FileSeed + targetIsBlank bool + blocksize uint64 + stashLimit int64 + + // Placements is an intermediate representation of the target index, + // capturing what source is used to populate each chunk. It mirrors the + // length of the index but a single step can span multiple chunks. + placements []*placement + + // unique lists every placement once, in target order. A placement + // spanning multiple chunks appears in p.placements several times but + // executes as a single step. + unique []*placement + + selfSeed *selfSeed + + // seedFiles records the state of every seed file when it was validated, + // by name, to notice files that change during assembly. + seedFiles map[string]seedFileState + + // skips holds the sources of the chunks found in place in the target, + // computed once and shared with plans for other seeds. Consecutive + // chunks share one source, which becomes a single step. + skips []*skipInPlace +} + +type assembleSource interface { + fmt.Stringer + Execute(f *os.File) (copied uint64, cloned uint64, err error) + // recordStats adds the step's chunk accounting to stats. numChunks is + // the number of index chunks the step covers. + recordStats(stats *ExtractStats, numChunks int) + // access tells what the step does to the target file, which decides + // the order it runs in relative to the other steps. + access() targetAccess +} + +// byteRange is the range [start, end) of the target file. +type byteRange struct{ start, end uint64 } + +func (r byteRange) empty() bool { return r.start >= r.end } + +func (r byteRange) overlaps(o byteRange) bool { + return !r.empty() && !o.empty() && r.start < o.end && o.start < r.end +} + +// targetAccess is what a step does to the target file: the range it writes, +// and the range it reads from the target, if any. +type targetAccess struct { + writes byteRange + reads byteRange + + // readsOld is set when the step reads the content from before assembly, + // which the steps writing there must not overwrite before it's read. + // Otherwise the step reads what other steps write, and runs after them. + readsOld bool + + // stash, if set, holds what the step reads. Instead of waiting for the + // read, the steps overwriting the range fill the stash first. + stash *stash +} + +type assembleSeedSource interface { + assembleSource + Seed() Seed + // File is the file the source reads, empty if it doesn't read one. + File() string + Validate(file *os.File) error +} + +type placement struct { + source assembleSource + step *planStep // the step executing the placement, one per unique placement +} + +// newPlan creates a fully populated assemblePlan. +func newPlan(name string, idx Index, s Store, opts ...planOption) *assemblePlan { + p := &assemblePlan{ + idx: idx, + concurrency: 1, + target: name, + store: s, + targetIsBlank: true, + stashLimit: math.MaxInt64, + placements: make([]*placement, len(idx.Chunks)), + } + for _, opt := range opts { + opt(p) + } + if p.blocksize == 0 { + p.blocksize = blocksizeOfFile(name) + } + p.selfSeed = newSelfSeed(p.target, p.idx, p.blocksize) + p.generate() + return p +} + +// replan creates a plan for different seeds. It reuses what's known about +// the target independent of the seeds: its blocksize, the self-seed and the +// chunks already in place. +func (p *assemblePlan) replan(opts ...planOption) *assemblePlan { + q := &assemblePlan{ + idx: p.idx, + concurrency: p.concurrency, + target: p.target, + store: p.store, + seeds: p.seeds, + inPlaceSeed: p.inPlaceSeed, + targetIsBlank: p.targetIsBlank, + blocksize: p.blocksize, + stashLimit: p.stashLimit, + selfSeed: p.selfSeed, + skips: p.skips, + placements: make([]*placement, len(p.idx.Chunks)), + } + for _, opt := range opts { + opt(q) + } + q.generate() + return q +} + +// Validate checks that all file seed placements still match their underlying +// data. Returns a SeedInvalid error if a seed file was modified after its +// index was created. +func (p *assemblePlan) Validate(pb ProgressBar) error { + // Phase 1 — Sequential: collect unique fileSeedSource placements, open + // their backing files, and build a list of items to validate. + type validateItem struct { + fs assembleSeedSource + file *os.File + chunks int + } + + fileMap := make(map[string]*os.File) + defer func() { + for _, f := range fileMap { + _ = f.Close() + } + }() + + invalidSeeds := make(map[Seed]error) + failedFiles := make(map[string]struct{}) + + var items []validateItem + for _, pl := range p.unique { + // Sources that don't read a file, like those of the null chunk + // seed, have no data their index could disagree with. + fs, ok := pl.source.(assembleSeedSource) + if !ok || fs.File() == "" { + continue + } + + // Skip seeds and files already known to be invalid + if _, ok := invalidSeeds[fs.Seed()]; ok { + continue + } + if _, ok := failedFiles[fs.File()]; ok { + invalidSeeds[fs.Seed()] = fmt.Errorf("seed file %s could not be opened", fs.File()) + continue + } + + if _, ok := fileMap[fs.File()]; !ok { + f, err := os.Open(fs.File()) + if err != nil { + failedFiles[fs.File()] = struct{}{} + invalidSeeds[fs.Seed()] = err + continue + } + fileMap[fs.File()] = f + + // Record the file as it is before reading its chunks, a change + // from here on is noticed after assembly. + info, err := f.Stat() + if err != nil { + invalidSeeds[fs.Seed()] = err + continue + } + if p.seedFiles == nil { + p.seedFiles = make(map[string]seedFileState) + } + p.seedFiles[fs.File()] = newSeedFileState(info) + } + + items = append(items, validateItem{fs: fs, file: fileMap[fs.File()], chunks: pl.step.numChunks}) + } + + // Phase 2 — Concurrent: validate each segment in parallel, counting the + // progress in chunks. + var total int + for _, item := range items { + total += item.chunks + } + pb.SetTotal(total) + pb.Start() + defer pb.Finish() + var mu sync.Mutex + var g errgroup.Group + g.SetLimit(p.concurrency) + for _, item := range items { + g.Go(func() error { + if err := item.fs.Validate(item.file); err != nil { + mu.Lock() + invalidSeeds[item.fs.Seed()] = err + mu.Unlock() + } + pb.Add(item.chunks) + return nil + }) + } + _ = g.Wait() // Validation failures are collected in invalidSeeds + + // Phase 3 — Sequential: build the error result. + if len(invalidSeeds) > 0 { + seeds := make([]Seed, 0, len(invalidSeeds)) + errs := make([]error, 0, len(invalidSeeds)) + for seed, err := range invalidSeeds { + seeds = append(seeds, seed) + errs = append(errs, err) + } + return SeedInvalid{Seeds: seeds, Err: errors.Join(errs...)} + } + return nil +} + +func (p *assemblePlan) generate() { + // When the target file already exists, mark chunks that are already + // correct so they can be skipped during assembly. An in-place seed + // then tells us which of the remaining chunks can be moved to their + // position from elsewhere in the target. + if !p.targetIsBlank { + if p.skips == nil { + p.skips = p.generateSkips() + p.selfSeed.useInPlace(p.skips) + } + p.placeSkips() + if p.inPlaceSeed != nil { + p.generateInPlace(p.inPlaceSeed) + } + } + + // Fill the remaining positions from the seeds, and from the target + // itself once the chunks are written elsewhere in it (the self-seed). + // Cloning is preferred over copying, and the seeds over the self-seed + // as they don't depend on other steps. + for _, cloning := range []bool{true, false} { + for _, seed := range p.seeds { + p.fillFromSeed(seed, cloning) + } + p.fillFromSelfSeed(cloning) + } + + // Fill any gaps in the file by copying from the store. + for i := range p.placements { + if p.placements[i] != nil { + continue + } + p.placements[i] = &placement{ + source: ©FromStore{ + store: p.store, + chunk: p.idx.Chunks[i], + }, + } + } + + // Every position is filled now. Create the step executing each unique + // placement, counting the index chunks it covers. + for _, pl := range p.placements { + if pl.step == nil { + pl.step = &planStep{source: pl.source} + p.unique = append(p.unique, pl) + } + pl.step.numChunks++ + } +} + +// placeSkips claims the positions of the chunks that are already in place. +// Their sources are shared between the plans of one assembly, the placements +// wrapping them are not. +func (p *assemblePlan) placeSkips() { + var run *placement + for i, src := range p.skips { + if src == nil { + run = nil + continue + } + if run == nil || run.source != src { + run = &placement{source: src} + } + p.placements[i] = run + } +} + +// fillFromSeed places the seed's matches in unfilled positions. With cloning +// set, only matches that are cloned into the target are used. +func (p *assemblePlan) fillFromSeed(seed Seed, cloning bool) { + p.fillFromMatches(seed, cloning, func(pl *placement, from, match, n int) { + pl.source = &fileSeedSource{ + segment: seed.GetSegment(match, n), + seed: seed, + offset: p.idx.Chunks[from].Start, + length: chunkRangeLength(p.idx.Chunks[from : from+n]), + blocksize: p.blocksize, + isBlank: p.targetIsBlank, + } + }) +} + +// fillFromSelfSeed places matches within the target in unfilled positions. +// Copying chunks within the target depends on the steps writing them first. +// With cloning set, only matches that are cloned are used. +func (p *assemblePlan) fillFromSelfSeed(cloning bool) { + p.fillFromMatches(p.selfSeed, cloning, func(pl *placement, from, match, n int) { + pl.source = p.selfSeed.GetSegment(match, n, p.idx.Chunks[from].Start) + }) +} + +// fillFromMatches walks the positions that aren't filled yet, looking for the +// longest run of chunks the seed can provide for each of them. place sets the +// source of a claimed run, given its placement, the target position it starts +// at, the position of the match in the seed and the number of chunks claimed. +// With cloning set, only matches that are cloned into the target are used. +func (p *assemblePlan) fillFromMatches(seed matchingSeed, cloning bool, place func(pl *placement, from, match, n int)) { + // Every position would be looked up only to find its match can't be + // cloned. With a chunk repeating many times, that's most of the planning. + cs, _ := seed.(cloningSeed) + if cloning && (cs == nil || !cs.mayClone(p.targetIsBlank)) { + return + } + for i := 0; i < len(p.idx.Chunks); i++ { + if p.placements[i] != nil { + continue + } + + match, n := seed.LongestMatchFrom(p.idx.Chunks, i) + if n < 1 { + continue + } + if cloning && !cs.clones(match, p.idx.Chunks[i:i+n], p.blocksize, p.targetIsBlank) { + continue + } + + // The same placement covers the whole matched run, though it may + // be cut short by positions that are already filled. We dedup runs + // into single steps later. + pl := &placement{} + size := p.claimRun(pl, i, n) + place(pl, i, match, size) + + i += size - 1 // the loop's i++ moves past the claimed run + } +} + +// claimRun assigns pl to up to n consecutive unfilled placements starting at +// position from. It stops at the first position that is already filled and +// returns the number of positions claimed. +func (p *assemblePlan) claimRun(pl *placement, from, n int) int { + size := 0 + for ; size < n && p.placements[from+size] == nil; size++ { + p.placements[from+size] = pl + } + return size +} + +func (p *assemblePlan) Steps() []*planStep { + // Order the steps by what they do to the target. The steps writing to it + // are in target order and don't overlap, the writers of a range are found + // by binary search. + var ( + writers []*planStep + writes []IndexChunk + ) + for _, pl := range p.unique { + if w := pl.source.access().writes; !w.empty() { + writers = append(writers, pl.step) + writes = append(writes, IndexChunk{Start: w.start, Size: w.end - w.start}) + } + } + for _, pl := range p.unique { + a := pl.source.access() + if a.reads.empty() { + continue + } + lo, hi := overlapping(writes, a.reads.start, a.reads.end) + for _, w := range writers[lo:hi] { + switch { + case w == pl.step: + // A step reads what it needs before it writes + case !a.readsOld: + w.before(pl.step) // read the data once it's written + case a.stash != nil: + if !slices.Contains(w.fills, a.stash) { + w.fills = append(w.fills, a.stash) // hold it before it's overwritten + } + default: + pl.step.before(w) // read the data before it's overwritten + } + } + } + + // List the in-place moves before the other sources so they're handed + // out first, the steps waiting for them to read their sources follow. + steps := make([]*planStep, 0, len(p.unique)) + for _, moves := range []bool{true, false} { + for _, pl := range p.unique { + if _, ok := pl.source.(*inPlaceCopy); ok == moves { + steps = append(steps, pl.step) + } + } + } + return steps +} + +// changedSeeds returns the seed files that may have been written to since they +// were validated. The target itself, the data of an in-place seed, is left +// out: assembly writes it. +func (p *assemblePlan) changedSeeds() []string { + var changed []string + for name, state := range p.seedFiles { + if name != p.target && state.changed(name) { + changed = append(changed, name) + } + } + slices.Sort(changed) + return changed +} + +// generateSkips reads the target file and returns the sources for the chunks +// that are already in the correct position so they can be skipped during +// assembly. Consecutive matching chunks share a single source so they become +// one step. +func (p *assemblePlan) generateSkips() []*skipInPlace { + found, err := chunksInTarget(p.target, p.idx, p.concurrency, NewProgressBar("Checking target ")) + if err != nil { + return nil + } + + skips := make([]*skipInPlace, len(p.idx.Chunks)) + for i, ok := range found { + if ok { + chunk := p.idx.Chunks[i] + skips[i] = &skipInPlace{start: chunk.Start, end: chunk.Start + chunk.Size} + } + } + + // Merge consecutive in-place chunks into a single source so that they + // become one step instead of one per chunk. + var run *skipInPlace + for i, sk := range skips { + if sk == nil { + run = nil + continue + } + if run == nil { + run = sk + continue + } + // Extend the existing run and share the pointer + run.end = p.idx.Chunks[i].Start + p.idx.Chunks[i].Size + skips[i] = run + } + return skips +} + +// moveOp is a chunk, or a run of consecutive chunks, that the in-place seed +// holds at a different offset in the target file. +type moveOp struct { + targetIdx int // index of the first chunk in the target index + srcIdx int // index of the first chunk in the seed index + chunks int // number of chunks the move covers + srcStart uint64 // byte offset the data is read from + dstStart uint64 // byte offset the data is written to + size uint64 // the chunks are the same on both ends +} + +// overlaps reports whether the source and the destination of the move share +// any bytes. +func (m moveOp) overlaps() bool { + return m.srcStart < m.dstStart+m.size && m.dstStart < m.srcStart+m.size +} + +// joins reports whether next continues the move without a gap, in the source +// as well as the destination. Both being contiguous means the two travel the +// same distance. The run may not grow past that distance, beyond it the source +// and the destination of the move would overlap. +func (m moveOp) joins(next moveOp) bool { + distance := max(m.srcStart, m.dstStart) - min(m.srcStart, m.dstStart) + return m.srcStart+m.size == next.srcStart && + m.dstStart+m.size == next.dstStart && + m.size+next.size <= distance +} + +// generateInPlace processes an in-place seed for the chunks that aren't +// already in place. Chunks that exist at different offsets in the seed get +// inPlaceCopy placements, ordered so that every move reads its source before +// it's overwritten. Dependency cycles between moves are broken by stashing +// sources in memory. +// +// This is in-place delta reconstruction as described by Burns and Long in +// "In-place reconstruction of delta compressed files" (1998): a graph of which +// copy has to read before another one writes, ordered topologically, with +// cycles broken by taking data from elsewhere. Android's block-based OTA +// updates break them the same way used here, by stashing sources in memory up +// to a limit. Beyond the limit, the data comes from the store. Merging the +// chunks of one shift into runs, and splitting runs that end up in a cycle, +// is added on top so that larger ranges are moved or cloned at once. +func (p *assemblePlan) generateInPlace(seed *FileSeed) { + // Stage 1: Operation list — walk target index and classify each chunk. + var moves []moveOp + + // Chunks that exist in the seed become move operations, sourced from + // their first location. The chunks already in place were skipped + // before. If the seed claims one of them is at its position anyway, + // the seed is out of date there and the position isn't a source. Zero + // chunks are left to the null seed, which clones them or doesn't write + // them at all, rather than reading one zero chunk over and over. + nullID := NewNullChunk(p.idx.Index.ChunkSizeMax).ID + for i, c := range p.idx.Chunks { + if p.placements[i] != nil || c.ID == nullID { + continue // Already placed, or zeros + } + + sources := seed.pos[c.ID] + j := slices.IndexFunc(sources, func(pos int) bool { + return seed.index.Chunks[pos].Start != c.Start + }) + if j < 0 { + continue // Not in seed; will be filled by store or file seed later + } + moves = append(moves, moveOp{ + targetIdx: i, + srcIdx: sources[j], + chunks: 1, + srcStart: seed.index.Chunks[sources[j]].Start, + dstStart: c.Start, + size: c.Size, + }) + } + + if len(moves) == 0 { + return + } + + // Stage 2: Dependency graph — edge from i to j when move i's source + // overlaps move j's destination (i must read before j writes). + succ := moveGraph(moves) + + // Stage 3: Break cycles. Moves in a dependency cycle, like two chunks + // swapping places, can't be ordered. Some of them hold their source in + // a stash instead, which removes their outgoing edges. + stashed := breakCycles(succ) + + // Stage 4: Merge the moves of one contiguous shift into a single move, + // which then runs as one step. Moves that hold their source in a stash + // stay on their own, a stash for a whole run would need far more memory. + // Merging can create dependency cycles that the moves didn't have per + // chunk, so the runs are checked and split again where it did. + if merged, mergedStashed := mergeMoves(moves, stashed); len(merged) < len(moves) { + moves, stashed = splitRunsInCycles(merged, mergedStashed, seed.index.Chunks) + } + + // Stage 6: Placements. The stashes of all moves share one budget. Steps() + // orders them by the ranges they read and write. + budget := &stashBudget{available: p.stashLimit} + for i, m := range moves { + ipc := &inPlaceCopy{ + chunks: seed.index.Chunks[m.srcIdx : m.srcIdx+m.chunks], + dstOffset: m.dstStart, + seed: seed, + file: p.target, + blocksize: p.blocksize, + store: p.store, + } + if stashed[i] { + ipc.stash = &stash{offset: m.srcStart, size: m.size, budget: budget} + } else { + ipc.clone = seed.canReflink && m.chunks > 1 && !m.overlaps() && + reflinkable(m.srcStart, m.size, m.dstStart, p.blocksize) + } + pl := &placement{source: ipc} + for j := range m.chunks { + p.placements[m.targetIdx+j] = pl + } + } +} + +// moveGraph returns the successors of every move: an edge from i to j means +// move i reads from where move j writes, so i has to run first. The moves are +// in target order, so their destinations are sorted and don't overlap. +func moveGraph(moves []moveOp) [][]int { + dst := make([]IndexChunk, len(moves)) + for i, m := range moves { + dst[i] = IndexChunk{Start: m.dstStart, Size: m.size} + } + succ := make([][]int, len(moves)) + for i, m := range moves { + lo, hi := overlapping(dst, m.srcStart, m.srcStart+m.size) + for j := lo; j < hi; j++ { + if i != j { + succ[i] = append(succ[i], j) + } + } + } + return succ +} + +// mergeMoves combines the moves of one contiguous shift into a single move. +// Consecutive chunks that all travel the same distance describe one memmove, +// which is one step instead of one per chunk: fewer dependencies to track, and +// the data can be moved in one operation. Stashed moves are left alone. +func mergeMoves(moves []moveOp, stashed []bool) ([]moveOp, []bool) { + merged := make([]moveOp, 0, len(moves)) + mergedStashed := make([]bool, 0, len(moves)) + for i, m := range moves { + last := len(merged) - 1 + if last >= 0 && !stashed[i] && !mergedStashed[last] && merged[last].joins(m) { + merged[last].size += m.size + merged[last].chunks += m.chunks + continue + } + merged = append(merged, m) + mergedStashed = append(mergedStashed, stashed[i]) + } + return merged, mergedStashed +} + +// splitRunsInCycles splits the merged runs that a dependency cycle runs through back +// into their chunks, so the moves can be ordered with the stashes they needed +// per chunk. Two regions of a file swapping places for example can't be moved +// as two runs, however the moves of their chunks interleave. Runs outside of +// any cycle stay merged. It returns the moves and which of them hold their +// source in a stash. +func splitRunsInCycles(moves []moveOp, stashed []bool, seedChunks []IndexChunk) ([]moveOp, []bool) { + graph := func() [][]int { + succ := moveGraph(moves) + for i := range succ { + if stashed[i] { + succ[i] = nil // removed to break a cycle before merging + } + } + return succ + } + succ := graph() + inCycle := movesInCycles(succ) + split := make([]bool, len(moves)) + var found bool + for i, m := range moves { + if inCycle[i] && m.chunks > 1 { + split[i], found = true, true + } + } + if found { + // Splitting runs only takes edges away, it can't put another run + // into a cycle. + moves, stashed = splitRuns(moves, stashed, split, seedChunks) + succ = graph() + } + + // A cycle through single chunks alone was there before merging and is + // broken by the stashes from then. Nothing is left to stash, but + // checking costs little and a cycle would stall assembly. + for i, b := range breakCycles(succ) { + stashed[i] = stashed[i] || b + } + return moves, stashed +} + +// movesInCycles reports the moves that take part in a dependency cycle: the +// members of strongly connected components of two moves or more. It's +// Tarjan's algorithm, iterative so that long chains of moves can't exhaust the +// stack. +func movesInCycles(succ [][]int) []bool { + n := len(succ) + var ( + index = make([]int, n) // order of discovery from 1, 0 is unvisited + low = make([]int, n) // lowest index reachable from the node's subtree + onStack = make([]bool, n) + stack []int + inCycle = make([]bool, n) + next = 1 + ) + type frame struct{ node, edge int } + visit := func(v int) { + index[v], low[v] = next, next + next++ + stack = append(stack, v) + onStack[v] = true + } + for root := range n { + if index[root] != 0 { + continue + } + visit(root) + call := []frame{{node: root}} + for len(call) > 0 { + f := &call[len(call)-1] + v := f.node + if f.edge < len(succ[v]) { + w := succ[v][f.edge] + f.edge++ + switch { + case index[w] == 0: + visit(w) + call = append(call, frame{node: w}) + case onStack[w]: + low[v] = min(low[v], index[w]) + } + continue + } + // All successors are done. If none of them reached above v, v + // and what's above it on the stack form a component. + if low[v] == index[v] { + i := len(stack) - 1 + for stack[i] != v { + i-- + } + for _, w := range stack[i:] { + onStack[w] = false + inCycle[w] = len(stack)-i > 1 + } + stack = stack[:i] + } + call = call[:len(call)-1] + if len(call) > 0 { + u := call[len(call)-1].node + low[u] = min(low[u], low[v]) + } + } + } + return inCycle +} + +// splitRuns expands the marked runs back into one move per chunk. It's used +// when merging a run created a dependency cycle, which is broken by stashing +// a move; only single chunks are stashed so the memory stays bounded. +func splitRuns(moves []moveOp, stashed, marked []bool, seedChunks []IndexChunk) ([]moveOp, []bool) { + split := make([]moveOp, 0, len(moves)) + splitStashed := make([]bool, 0, len(moves)) + for i, m := range moves { + if !marked[i] || m.chunks == 1 { + split = append(split, m) + splitStashed = append(splitStashed, stashed[i]) + continue + } + dst := m.dstStart + for k := range m.chunks { + c := seedChunks[m.srcIdx+k] + split = append(split, moveOp{ + targetIdx: m.targetIdx + k, + srcIdx: m.srcIdx + k, + chunks: 1, + srcStart: c.Start, + dstStart: dst, + size: c.Size, + }) + splitStashed = append(splitStashed, false) + dst += c.Size + } + } + return split, splitStashed +} + +// overlapping returns the range [lo, hi) of chunks overlapping the byte range +// [start, end). The chunks have to be sorted by offset without overlapping +// each other, like those of an index. +func overlapping(chunks []IndexChunk, start, end uint64) (lo, hi int) { + lo = sort.Search(len(chunks), func(i int) bool { + return chunks[i].Start+chunks[i].Size > start + }) + hi = lo + for hi < len(chunks) && chunks[hi].Start < end { + hi++ + } + return lo, hi +} + +// breakCycles picks nodes of the directed graph succ whose outgoing edges +// have to be removed for the graph to become acyclic. It runs a depth-first +// search and marks the tail of every back edge, without exploring its +// remaining edges. The edges left are those of unmarked nodes, and none of +// them is a back edge of the search, so they can't form a cycle. +func breakCycles(succ [][]int) []bool { + const ( + unvisited = iota + onStack + finished + ) + type frame struct{ node, next int } + var ( + state = make([]uint8, len(succ)) + marked = make([]bool, len(succ)) + stack []frame + ) + for root := range succ { + if state[root] != unvisited { + continue + } + state[root] = onStack + stack = append(stack, frame{node: root}) + for len(stack) > 0 { + top := &stack[len(stack)-1] + if marked[top.node] || top.next == len(succ[top.node]) { + state[top.node] = finished + stack = stack[:len(stack)-1] + continue + } + w := succ[top.node][top.next] + top.next++ + switch state[w] { + case unvisited: + state[w] = onStack + stack = append(stack, frame{node: w}) + case onStack: + marked[top.node] = true + } + } + } + return marked +} diff --git a/assemble-plan_test.go b/assemble-plan_test.go new file mode 100644 index 0000000..ac0a2fa --- /dev/null +++ b/assemble-plan_test.go @@ -0,0 +1,979 @@ +package desync + +import ( + "fmt" + "math/rand/v2" + "os" + "path/filepath" + "slices" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSelfSeedPlanSteps(t *testing.T) { + tests := map[string]struct { + index Index + expected []string + }{ + "all unique chunks": { + index: indexSequence(0x01, 0x02, 0x03), + expected: []string{ + "Store: Copy 0100000000000000000000000000000000000000000000000000000000000000 to [0:100]", + "Store: Copy 0200000000000000000000000000000000000000000000000000000000000000 to [100:200]", + "Store: Copy 0300000000000000000000000000000000000000000000000000000000000000 to [200:300]", + }, + }, + "single chunk": { + index: indexSequence(0x01), + expected: []string{ + "Store: Copy 0100000000000000000000000000000000000000000000000000000000000000 to [0:100]", + }, + }, + "repeated pair at end": { + // Sequence: 01 02 03 01 02 01 02 + // Positions 0,1 copy from 5,6; positions 3,4 copy from 5,6; + // positions 2,5,6 come from store. + index: indexSequence(0x01, 0x02, 0x03, 0x01, 0x02, 0x01, 0x02), + expected: []string{ + "SelfSeed: Copy [500:700] to [0:200]", + "Store: Copy 0300000000000000000000000000000000000000000000000000000000000000 to [200:300]", + "SelfSeed: Copy [500:700] to [300:500]", + "Store: Copy 0100000000000000000000000000000000000000000000000000000000000000 to [500:600]", + "Store: Copy 0200000000000000000000000000000000000000000000000000000000000000 to [600:700]", + }, + }, + "full duplicate sequence": { + // Sequence: 01 02 03 01 02 03 + // Positions 0-2 copy from 3-5; positions 3-5 come from store. + index: indexSequence(0x01, 0x02, 0x03, 0x01, 0x02, 0x03), + expected: []string{ + "SelfSeed: Copy [300:600] to [0:300]", + "Store: Copy 0100000000000000000000000000000000000000000000000000000000000000 to [300:400]", + "Store: Copy 0200000000000000000000000000000000000000000000000000000000000000 to [400:500]", + "Store: Copy 0300000000000000000000000000000000000000000000000000000000000000 to [500:600]", + }, + }, + "same chunk repeated": { + // Sequence: 01 01 01 + // Position 2 comes from store, then positions 0 and 1 each + // self-seed from position 2. + index: indexSequence(0x01, 0x01, 0x01), + expected: []string{ + "SelfSeed: Copy [200:300] to [0:100]", + "SelfSeed: Copy [200:300] to [100:200]", + "Store: Copy 0100000000000000000000000000000000000000000000000000000000000000 to [200:300]", + }, + }, + "single repeated chunk": { + // Sequence: 01 02 01 + // Position 0 copies from 2; positions 1,2 come from store. + index: indexSequence(0x01, 0x02, 0x01), + expected: []string{ + "SelfSeed: Copy [200:300] to [0:100]", + "Store: Copy 0200000000000000000000000000000000000000000000000000000000000000 to [100:200]", + "Store: Copy 0100000000000000000000000000000000000000000000000000000000000000 to [200:300]", + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + require.Equal(t, test.expected, planStrings("test", test.index)) + }) + } +} + +func TestInPlaceChunkDetection(t *testing.T) { + // Create chunk data and compute their IDs + data1 := make([]byte, 100) + data1[0] = 0x01 + id1 := Digest.Sum(data1) + + data2 := make([]byte, 100) + data2[0] = 0x02 + id2 := Digest.Sum(data2) + + data3 := make([]byte, 100) + data3[0] = 0x03 + id3 := Digest.Sum(data3) + + idx := Index{ + Chunks: []IndexChunk{ + {ID: id1, Start: 0, Size: 100}, + {ID: id2, Start: 100, Size: 100}, + {ID: id3, Start: 200, Size: 100}, + }, + } + + // Create a target file where chunks 0 and 2 match but chunk 1 does not + target := filepath.Join(t.TempDir(), "target") + f, err := os.Create(target) + require.NoError(t, err) + + _, err = f.Write(data1) // chunk 0: correct + require.NoError(t, err) + _, err = f.Write(make([]byte, 100)) // chunk 1: wrong data + require.NoError(t, err) + _, err = f.Write(data3) // chunk 2: correct + require.NoError(t, err) + require.NoError(t, f.Close()) + + got := planStrings(target, idx, planWithTargetIsBlank(false)) + + cid2 := ChunkID(id2) + expected := []string{ + "InPlace: Skip [0:100]", + fmt.Sprintf("Store: Copy %s to [100:200]", &cid2), + "InPlace: Skip [200:300]", + } + require.Equal(t, expected, got) + + // Subtest: all chunks match in-place — consecutive ones should merge + t.Run("consecutive merge", func(t *testing.T) { + target2 := filepath.Join(t.TempDir(), "target2") + f2, err := os.Create(target2) + require.NoError(t, err) + _, err = f2.Write(data1) + require.NoError(t, err) + _, err = f2.Write(data2) + require.NoError(t, err) + _, err = f2.Write(data3) + require.NoError(t, err) + require.NoError(t, f2.Close()) + + require.Equal(t, []string{"InPlace: Skip [0:300]"}, + planStrings(target2, idx, planWithTargetIsBlank(false))) + }) +} + +func TestFileSeedPlanSteps(t *testing.T) { + tests := map[string]struct { + target Index + seed Index + expected []string + }{ + "basic matching": { + // Target: 01, 02, 03, 04 + // Seed: 02, 03 + // Chunks 1-2 from seed, 0 and 3 from store. + target: indexSequence(0x01, 0x02, 0x03, 0x04), + seed: indexSequence(0x02, 0x03), + expected: []string{ + "Store: Copy 0100000000000000000000000000000000000000000000000000000000000000 to [0:100]", + "FileSeed(seed): Copy to [100:300]", + "Store: Copy 0400000000000000000000000000000000000000000000000000000000000000 to [300:400]", + }, + }, + "all from seed": { + // Target: 01, 02, 03 + // Seed: 01, 02, 03 + // One seed step covering all. + target: indexSequence(0x01, 0x02, 0x03), + seed: indexSequence(0x01, 0x02, 0x03), + expected: []string{ + "FileSeed(seed): Copy to [0:300]", + }, + }, + "no match": { + // Target: 01, 02 + // Seed: 05, 06 + // Both from store. + target: indexSequence(0x01, 0x02), + seed: indexSequence(0x05, 0x06), + expected: []string{ + "Store: Copy 0100000000000000000000000000000000000000000000000000000000000000 to [0:100]", + "Store: Copy 0200000000000000000000000000000000000000000000000000000000000000 to [100:200]", + }, + }, + "seed priority over self-seed": { + // Target: 01, 02, 01 + // Seed: 01, 02, 01 + // Neither can clone, so the seed fills all positions without + // the dependency a self-seed copy of position 0 would have. + target: indexSequence(0x01, 0x02, 0x01), + seed: indexSequence(0x01, 0x02, 0x01), + expected: []string{ + "FileSeed(seed): Copy to [0:300]", + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + seed, err := NewFileSeed("test", "seed", test.seed, IndexRegenerationSettings{}) + require.NoError(t, err) + + got := planStrings("test", test.target, planWithSeeds([]Seed{seed})) + require.Equal(t, test.expected, got) + }) + } +} + +func TestInPlaceSeedPlanSteps(t *testing.T) { + // Variable-size chunks, each filled with a distinct byte so its hash is + // unique. + A := filledChunk(200, 0xAA) + B := filledChunk(150, 0xBB) + C := filledChunk(100, 0xCC) + D := filledChunk(50, 0xDD) + E := filledChunk(180, 0xEE) // only appears in store + F := filledChunk(120, 0xFF) // only appears in file seed + + // storeStep formats a store-sourced step string. + storeStep := func(c testChunk, start uint64) string { + id := c.id + return fmt.Sprintf("Store: Copy %s to [%d:%d]", &id, start, start+uint64(len(c.data))) + } + + t.Run("swap two chunks", func(t *testing.T) { + // In-place: [A:200][B:150] + // Target: [B:150][A:200] + // One cycle: A↔B, broken by stashing A. The copy of B fills the + // stash before it overwrites A. + f := writeChunkFile(t, A, B) + inPlace, err := NewFileSeed(f, f, chunkIndex(A, B), IndexRegenerationSettings{}) + require.NoError(t, err) + + got := planStrings(f, chunkIndex(B, A), + planWithInPlaceSeed(inPlace), planWithTargetIsBlank(false)) + expected := []string{ + "InPlace: Copy [200:350] to [0:150]", + "InPlace: Copy [0:200] to [150:350] from stash", + } + require.Equal(t, expected, got) + }) + + t.Run("stash filled by the step overwriting it", func(t *testing.T) { + // In a swap, A is stashed. The copy of B overwrites A's source, so + // it fills the stash first rather than waiting for A. The copy of A + // still waits for B to read its source. + f := writeChunkFile(t, A, B) + inPlace, err := NewFileSeed(f, f, chunkIndex(A, B), IndexRegenerationSettings{}) + require.NoError(t, err) + + steps := newPlan(f, chunkIndex(B, A), nil, + planWithInPlaceSeed(inPlace), planWithTargetIsBlank(false)).Steps() + require.Len(t, steps, 2) + copyB, copyA := steps[0], steps[1] + require.Equal(t, "InPlace: Copy [200:350] to [0:150]", copyB.source.String()) + require.Equal(t, []*stash{copyA.source.(*inPlaceCopy).stash}, copyB.fills) + require.Empty(t, copyA.fills) + require.Empty(t, copyB.dependencies) + require.Contains(t, copyA.dependencies, copyB) + }) + + t.Run("two independent cycles", func(t *testing.T) { + // In-place: [A:200][B:150][C:100][D:50] + // Target: [B:150][A:200][D:50][C:100] + // Cycle 1: A↔B in byte range [0,350) + // Cycle 2: C↔D in byte range [350,500) + f := writeChunkFile(t, A, B, C, D) + inPlace, err := NewFileSeed(f, f, chunkIndex(A, B, C, D), IndexRegenerationSettings{}) + require.NoError(t, err) + + got := planStrings(f, chunkIndex(B, A, D, C), + planWithInPlaceSeed(inPlace), planWithTargetIsBlank(false)) + expected := []string{ + "InPlace: Copy [200:350] to [0:150]", + "InPlace: Copy [0:200] to [150:350] from stash", + "InPlace: Copy [450:500] to [350:400]", + "InPlace: Copy [350:450] to [400:500] from stash", + } + require.Equal(t, expected, got) + }) + + t.Run("rearrange with store chunks", func(t *testing.T) { + // In-place: [A:200][B:150] + // Target: [B:150][A:200][E:180] + // A↔B cycle, E from store (not in seed). + f := writeChunkFile(t, A, B) + inPlace, err := NewFileSeed(f, f, chunkIndex(A, B), IndexRegenerationSettings{}) + require.NoError(t, err) + + got := planStrings(f, chunkIndex(B, A, E), + planWithInPlaceSeed(inPlace), planWithTargetIsBlank(false)) + expected := []string{ + "InPlace: Copy [200:350] to [0:150]", + "InPlace: Copy [0:200] to [150:350] from stash", + storeStep(E, 350), + } + require.Equal(t, expected, got) + }) + + t.Run("partial rearrangement with skip", func(t *testing.T) { + // In-place: [A:200][B:150][C:100] + // Target: [A:200][C:100][B:150] + // A already at [0:200] in both indexes → skip. + // B↔C cycle: B [200:350]→[300:450], C [350:450]→[200:300]. + f := writeChunkFile(t, A, B, C) + inPlace, err := NewFileSeed(f, f, chunkIndex(A, B, C), IndexRegenerationSettings{}) + require.NoError(t, err) + + got := planStrings(f, chunkIndex(A, C, B), + planWithInPlaceSeed(inPlace), planWithTargetIsBlank(false)) + expected := []string{ + "InPlace: Copy [350:450] to [200:300]", + "InPlace: Copy [200:350] to [300:450] from stash", + "InPlace: Skip [0:200]", + } + require.Equal(t, expected, got) + }) + + t.Run("store chunk overwrites the sources of two moves", func(t *testing.T) { + // In-place: [C:100][D:50][A:200] + // Target: [E:180][D:50][C:100] + // E is written to [0:180] which holds the sources of both moves, + // so it has to wait for both of them. The two travel different + // distances, so they stay separate moves. + f := writeChunkFile(t, C, D, A) + inPlace, err := NewFileSeed(f, f, chunkIndex(C, D, A), IndexRegenerationSettings{}) + require.NoError(t, err) + + steps := newPlan(f, chunkIndex(E, D, C), nil, + planWithInPlaceSeed(inPlace), planWithTargetIsBlank(false)).Steps() + require.Equal(t, map[string][]string{ + "InPlace: Copy [100:150] to [180:230]": nil, + "InPlace: Copy [0:100] to [230:330]": nil, + storeStep(E, 0): { + "InPlace: Copy [0:100] to [230:330]", + "InPlace: Copy [100:150] to [180:230]", + }, + }, stepDependencies(steps)) + }) + + t.Run("chunks of one shift move together", func(t *testing.T) { + // In-place: [C:100][D:50][A:200] + // Target: [E:180][C:100][D:50] + // C and D stay next to each other and travel the same distance, so + // a single move covers both of them. + f := writeChunkFile(t, C, D, A) + inPlace, err := NewFileSeed(f, f, chunkIndex(C, D, A), IndexRegenerationSettings{}) + require.NoError(t, err) + + steps := newPlan(f, chunkIndex(E, C, D), nil, + planWithInPlaceSeed(inPlace), planWithTargetIsBlank(false)).Steps() + require.Equal(t, []string{ + "InPlace: Copy [0:150] to [180:330]", + storeStep(E, 0), + }, stepStrings(steps)) + require.Equal(t, 2, steps[0].numChunks, "chunks covered by the move") + }) + + t.Run("a run that would form a cycle is split again", func(t *testing.T) { + // In-place: [X:100][A:100][C:100][Z:100][B:100] + // Target: [P:100][B:100][Q:100][A:100][C:100] + // A and C travel the same distance and are adjacent, but merging + // them creates a cycle: B is written where A reads, and reads where + // C writes. The run is split again rather than held in memory. + x := filledChunk(100, 0x01) + a := filledChunk(100, 0x02) + c := filledChunk(100, 0x03) + z := filledChunk(100, 0x04) + b := filledChunk(100, 0x05) + q := filledChunk(100, 0x06) // only in the store + r := filledChunk(100, 0x07) // only in the store + + f := writeChunkFile(t, x, a, c, z, b) + inPlace, err := NewFileSeed(f, f, chunkIndex(x, a, c, z, b), IndexRegenerationSettings{}) + require.NoError(t, err) + + steps := newPlan(f, chunkIndex(r, b, q, a, c), nil, + planWithInPlaceSeed(inPlace), planWithTargetIsBlank(false)).Steps() + moveA := "InPlace: Copy [100:200] to [300:400]" + moveB := "InPlace: Copy [400:500] to [100:200]" + moveC := "InPlace: Copy [200:300] to [400:500]" + require.Equal(t, map[string][]string{ + moveA: nil, + moveB: {moveA}, + moveC: {moveB}, + storeStep(r, 0): nil, + storeStep(q, 200): {moveC}, + }, stepDependencies(steps)) + }) + + t.Run("moves outside of cycles are ordered", func(t *testing.T) { + // In-place: [A:200][B:150][C:100] + // Target: [D:50][A:200][B:150][C:100] + // Every chunk shifts by 50 bytes. There is no cycle but each move + // overwrites the source of the next one, so that has to be read + // first. The chunks all travel the same distance, but it's shorter + // than any of them, so they can't be moved as one run. + f := writeChunkFile(t, A, B, C) + inPlace, err := NewFileSeed(f, f, chunkIndex(A, B, C), IndexRegenerationSettings{}) + require.NoError(t, err) + + steps := newPlan(f, chunkIndex(D, A, B, C), nil, + planWithInPlaceSeed(inPlace), planWithTargetIsBlank(false)).Steps() + require.Equal(t, map[string][]string{ + "InPlace: Copy [0:200] to [50:250]": {"InPlace: Copy [200:350] to [250:400]"}, + "InPlace: Copy [200:350] to [250:400]": {"InPlace: Copy [350:450] to [400:500]"}, + "InPlace: Copy [350:450] to [400:500]": nil, + storeStep(D, 0): {"InPlace: Copy [0:200] to [50:250]"}, + }, stepDependencies(steps)) + }) + + t.Run("runs outside of a cycle stay merged", func(t *testing.T) { + // In-place: [F0][F1][B][F3][A][C][D][E] + // Target: [T0][A][C][T3][B][T5][T6][T7][D][E] + // A and C move together, but merged they form a cycle with B: B is + // written where A reads, and reads where C writes. Only that run is + // split. D and E are a run elsewhere, not part of any cycle, and stay + // one move. + c := func(fill byte) testChunk { return filledChunk(100, fill) } + f0, f1, b, f3, a, cc, d, e := c(0x10), c(0x11), c(0x12), c(0x13), c(0x14), c(0x15), c(0x16), c(0x17) + t0, t3, t5, t6, t7 := c(0x20), c(0x23), c(0x25), c(0x26), c(0x27) // only in the store + + f := writeChunkFile(t, f0, f1, b, f3, a, cc, d, e) + inPlace, err := NewFileSeed(f, f, chunkIndex(f0, f1, b, f3, a, cc, d, e), IndexRegenerationSettings{}) + require.NoError(t, err) + + var moves []string + for _, step := range newPlan(f, chunkIndex(t0, a, cc, t3, b, t5, t6, t7, d, e), nil, + planWithInPlaceSeed(inPlace), planWithTargetIsBlank(false)).Steps() { + if _, ok := step.source.(*inPlaceCopy); ok { + moves = append(moves, step.source.String()) + } + } + require.Equal(t, []string{ + "InPlace: Copy [400:500] to [100:200]", + "InPlace: Copy [500:600] to [200:300]", + "InPlace: Copy [200:300] to [400:500]", + "InPlace: Copy [600:800] to [800:1000]", + }, moves) + }) + + t.Run("zero chunks are left to the null seed", func(t *testing.T) { + // In-place: [zeros:256][Z:256] + // Target: [Z:256][zeros:256] + // Moving the zeros would read them from the target. The null seed + // writes them instead, once the move of Z read its source. + zeros := testChunk{id: Digest.Sum(make([]byte, 256)), data: make([]byte, 256)} + z := filledChunk(256, 0x5A) + f := writeChunkFile(t, zeros, z) + inPlace, err := NewFileSeed(f, f, chunkIndex(zeros, z), IndexRegenerationSettings{}) + require.NoError(t, err) + + steps := newPlan(f, chunkIndex(z, zeros), nil, + planWithInPlaceSeed(inPlace), planWithTargetIsBlank(false), + planWithSeeds([]Seed{&nullChunkSeed{id: zeros.id}})).Steps() + require.Equal(t, map[string][]string{ + "InPlace: Copy [256:512] to [0:256]": nil, + "FileSeed(): Copy to [256:512]": {"InPlace: Copy [256:512] to [0:256]"}, + }, stepDependencies(steps)) + }) + + t.Run("mixed in-place and file seed", func(t *testing.T) { + // In-place: [A:200][B:150] + // File seed "seedfile": [F:120] + // Target: [A:200][F:120][B:150] + // A at same offset → skip. + // B moves [200:350]→[320:470] (B must read before F writes to [200:320]). + // F from file seed at [200:320]. + f := writeChunkFile(t, A, B) + inPlaceSeed, err := NewFileSeed(f, f, chunkIndex(A, B), IndexRegenerationSettings{}) + require.NoError(t, err) + fileSeed, err := NewFileSeed(f, "seedfile", chunkIndex(F), IndexRegenerationSettings{}) + require.NoError(t, err) + + got := planStrings(f, chunkIndex(A, F, B), + planWithInPlaceSeed(inPlaceSeed), planWithSeeds([]Seed{fileSeed}), planWithTargetIsBlank(false)) + expected := []string{ + "InPlace: Copy [200:350] to [320:470]", + "InPlace: Skip [0:200]", + "FileSeed(seedfile): Copy to [200:320]", + } + require.Equal(t, expected, got) + }) +} + +// planStrings builds a plan and returns the string representation of the +// source of every step, in the order the steps are handed out. +func planStrings(name string, idx Index, opts ...planOption) []string { + return stepStrings(newPlan(name, idx, nil, opts...).Steps()) +} + +// stepStrings returns the string representation of the steps' sources. +func stepStrings(steps []*planStep) []string { + got := make([]string, len(steps)) + for i, s := range steps { + got[i] = s.source.String() + } + return got +} + +// stepDependencies returns the sorted dependencies of every step, keyed and +// identified by their string representation. +func stepDependencies(steps []*planStep) map[string][]string { + deps := make(map[string][]string, len(steps)) + for _, step := range steps { + var d []string + for dep := range step.dependencies { + d = append(d, dep.source.String()) + } + slices.Sort(d) + deps[step.source.String()] = d + } + return deps +} + +func TestBreakCycles(t *testing.T) { + rng := rand.New(rand.NewPCG(1, 2)) + for range 1000 { + // Build a random graph with up to 20 nodes, including cycles + // and nodes with several outgoing edges. + n := rng.IntN(20) + 1 + succ := make([][]int, n) + for i := range succ { + for range rng.IntN(4) { + if j := rng.IntN(n); j != i { + succ[i] = append(succ[i], j) + } + } + } + + // Removing the outgoing edges of marked nodes must leave the graph + // without cycles, i.e. all nodes can be sorted topologically. + marked := breakCycles(succ) + inDegree := make([]int, n) + for i := range succ { + if marked[i] { + continue + } + for _, j := range succ[i] { + inDegree[j]++ + } + } + var queue []int + for i, d := range inDegree { + if d == 0 { + queue = append(queue, i) + } + } + var sorted int + for len(queue) > 0 { + i := queue[0] + queue = queue[1:] + sorted++ + if marked[i] { + continue + } + for _, j := range succ[i] { + inDegree[j]-- + if inDegree[j] == 0 { + queue = append(queue, j) + } + } + } + require.Equal(t, n, sorted, "cycle left in graph %v with marked nodes %v", succ, marked) + } +} + +// movesInCycles finds exactly the nodes on a cycle: those that reach another +// node which reaches them back. +func TestMovesInCycles(t *testing.T) { + rng := rand.New(rand.NewPCG(5, 6)) + for range 1000 { + n := rng.IntN(20) + 1 + succ := make([][]int, n) + for i := range succ { + for range rng.IntN(3) { + if j := rng.IntN(n); j != i { + succ[i] = append(succ[i], j) + } + } + } + + // reach[i][j] is set when there's a path from i to j + reach := make([][]bool, n) + for i := range reach { + reach[i] = make([]bool, n) + queue := []int{i} + for len(queue) > 0 { + v := queue[0] + queue = queue[1:] + for _, w := range succ[v] { + if !reach[i][w] { + reach[i][w] = true + queue = append(queue, w) + } + } + } + } + want := make([]bool, n) + for i := range n { + for j := range n { + if i != j && reach[i][j] && reach[j][i] { + want[i] = true + } + } + } + require.Equal(t, want, movesInCycles(succ), "graph %v", succ) + } +} + +func TestFileSeedValidation(t *testing.T) { + // Create two chunks with known data and compute their IDs + data1 := make([]byte, 100) + data1[0] = 0xAA + id1 := Digest.Sum(data1) + + data2 := make([]byte, 100) + data2[0] = 0xBB + id2 := Digest.Sum(data2) + + seedIndex := Index{ + Chunks: []IndexChunk{ + {ID: id1, Start: 0, Size: 100}, + {ID: id2, Start: 100, Size: 100}, + }, + } + + // Target index matches the seed exactly + targetIndex := Index{ + Chunks: []IndexChunk{ + {ID: id1, Start: 0, Size: 100}, + {ID: id2, Start: 100, Size: 100}, + }, + } + + t.Run("valid seed", func(t *testing.T) { + seedFile := filepath.Join(t.TempDir(), "seed") + f, err := os.Create(seedFile) + require.NoError(t, err) + _, err = f.Write(data1) + require.NoError(t, err) + _, err = f.Write(data2) + require.NoError(t, err) + require.NoError(t, f.Close()) + + seed, err := NewFileSeed("target", seedFile, seedIndex, IndexRegenerationSettings{}) + require.NoError(t, err) + + plan := newPlan("target", targetIndex, nil, planWithSeeds([]Seed{seed})) + + require.NoError(t, plan.Validate(NullProgressBar{})) + }) + + t.Run("invalid seed", func(t *testing.T) { + seedFile := filepath.Join(t.TempDir(), "seed") + f, err := os.Create(seedFile) + require.NoError(t, err) + _, err = f.Write(data1) + require.NoError(t, err) + _, err = f.Write(data2) + require.NoError(t, err) + require.NoError(t, f.Close()) + + seed, err := NewFileSeed("target", seedFile, seedIndex, IndexRegenerationSettings{}) + require.NoError(t, err) + + plan := newPlan("target", targetIndex, nil, planWithSeeds([]Seed{seed})) + + // Corrupt the seed file after the plan was created + err = os.WriteFile(seedFile, make([]byte, 200), 0644) + require.NoError(t, err) + + err = plan.Validate(NullProgressBar{}) + require.Error(t, err) + + var seedErr SeedInvalid + require.ErrorAs(t, err, &seedErr) + require.Equal(t, []Seed{seed}, seedErr.Seeds) + }) + + t.Run("stale in-place seed chunk at its position", func(t *testing.T) { + // The in-place seed claims [1][2] but the file only holds chunk 1. + // Chunk 1 is skipped after hashing it. The seed is out of date for + // chunk 2, which has to come from the store without invalidating + // the seed. + f := filepath.Join(t.TempDir(), "target") + err := os.WriteFile(f, append(data1, make([]byte, 100)...), 0644) + require.NoError(t, err) + + inPlace, err := NewFileSeed(f, f, seedIndex, IndexRegenerationSettings{}) + require.NoError(t, err) + + plan := newPlan(f, targetIndex, nil, + planWithInPlaceSeed(inPlace), planWithTargetIsBlank(false)) + require.NoError(t, plan.Validate(NullProgressBar{})) + + require.Equal(t, []string{ + "InPlace: Skip [0:100]", + fmt.Sprintf("Store: Copy %s to [100:200]", ChunkID(id2)), + }, stepStrings(plan.Steps())) + }) + + t.Run("invalid in-place seed move", func(t *testing.T) { + // The in-place seed claims [1][2] but the file only holds chunk 1 + // at the correct position. Chunk 2 is moved to the front of the + // target, so Validate must check the move's source as well. + f := filepath.Join(t.TempDir(), "target") + err := os.WriteFile(f, append(data1, make([]byte, 100)...), 0644) + require.NoError(t, err) + + inPlace, err := NewFileSeed(f, f, seedIndex, IndexRegenerationSettings{}) + require.NoError(t, err) + + swapped := Index{ + Chunks: []IndexChunk{ + {ID: id2, Start: 0, Size: 100}, + {ID: id1, Start: 100, Size: 100}, + }, + } + plan := newPlan(f, swapped, nil, + planWithInPlaceSeed(inPlace), planWithTargetIsBlank(false)) + + err = plan.Validate(NullProgressBar{}) + var seedErr SeedInvalid + require.ErrorAs(t, err, &seedErr) + require.Equal(t, []Seed{inPlace}, seedErr.Seeds) + }) + + t.Run("stale chunk inside a moved run", func(t *testing.T) { + // The in-place seed claims [1][2] at the front of the file, and both + // move to the back by the same distance, so one move covers them. + // Only the first of the two is where the seed says it is, which the + // move has to notice for all of its chunks. + f := filepath.Join(t.TempDir(), "target") + err := os.WriteFile(f, append(data1, make([]byte, 100)...), 0644) + require.NoError(t, err) + + inPlace, err := NewFileSeed(f, f, seedIndex, IndexRegenerationSettings{}) + require.NoError(t, err) + + shifted := Index{ + Chunks: []IndexChunk{ + {ID: ChunkID{0xCC}, Start: 0, Size: 200}, // from the store + {ID: id1, Start: 200, Size: 100}, + {ID: id2, Start: 300, Size: 100}, + }, + } + plan := newPlan(f, shifted, nil, + planWithInPlaceSeed(inPlace), planWithTargetIsBlank(false)) + + // One move covers the whole run, and it's invalid + require.Equal(t, []string{ + "InPlace: Copy [0:200] to [200:400]", + fmt.Sprintf("Store: Copy %s to [0:200]", ChunkID{0xCC}), + }, stepStrings(plan.Steps())) + + err = plan.Validate(NullProgressBar{}) + var seedErr SeedInvalid + require.ErrorAs(t, err, &seedErr) + require.Equal(t, []Seed{inPlace}, seedErr.Seeds) + }) + + t.Run("null seed skipped", func(t *testing.T) { + // Create a null chunk index — data is all zeros + nullData := make([]byte, 100) + nullID := Digest.Sum(nullData) + + nullTargetIndex := Index{ + Chunks: []IndexChunk{ + {ID: nullID, Start: 0, Size: 100}, + }, + } + + // Use a null seed (FileName() returns "", so Validate skips it) + ns := &nullChunkSeed{id: nullID} + defer ns.close() + + plan := newPlan("target", nullTargetIndex, nil, planWithSeeds([]Seed{ns})) + + require.NoError(t, plan.Validate(NullProgressBar{})) + }) +} + +// Planning again for other seeds reuses what's known about the target +// rather than reading it again. +func TestReplanReusesTarget(t *testing.T) { + data := make([]byte, 100) + data[0] = 0xAA + idx := Index{Chunks: []IndexChunk{{ID: Digest.Sum(data), Start: 0, Size: 100}}} + + f := filepath.Join(t.TempDir(), "target") + require.NoError(t, os.WriteFile(f, data, 0644)) + plan := newPlan(f, idx, nil, planWithTargetIsBlank(false)) + + // Changing the target isn't noticed by the new plan, it's not hashed + // again + require.NoError(t, os.WriteFile(f, make([]byte, 100), 0644)) + replanned := plan.replan(planWithSeeds(nil)) + + require.Same(t, plan.selfSeed, replanned.selfSeed) + require.Equal(t, []string{"InPlace: Skip [0:100]"}, stepStrings(replanned.Steps())) +} + +func TestSelfSeedFromInPlaceChunks(t *testing.T) { + chunks := randomChunks(100, 100, 100, 100) + garbage := filledChunk(100, 0xFF) + storeCopy := func(c testChunk, start int) string { + return fmt.Sprintf("Store: Copy %s to [%d:%d]", &c.id, start, start+100) + } + + tests := map[string]struct { + target []testChunk + output []int + expected []string + }{ + "in place before the duplicate": { + target: []testChunk{chunks[0], garbage, garbage}, + output: []int{0, 1, 0}, + expected: []string{ + "InPlace: Skip [0:100]", + storeCopy(chunks[1], 100), + "SelfSeed: Copy [0:100] to [200:300]", + }, + }, + "run in place before the duplicate": { + target: []testChunk{chunks[0], chunks[1], garbage, garbage, garbage, garbage}, + output: []int{0, 1, 2, 3, 0, 1}, + expected: []string{ + "InPlace: Skip [0:200]", + storeCopy(chunks[2], 200), + storeCopy(chunks[3], 300), + "SelfSeed: Copy [0:200] to [400:600]", + }, + }, + "match limited to the run in place": { + // Positions 3 and 4 match 0 and 1, but only 0 is in place. + // 1 is copied from 4 in turn, which comes from the store. + target: []testChunk{chunks[0], garbage, garbage, garbage, garbage}, + output: []int{0, 1, 2, 0, 1}, + expected: []string{ + "InPlace: Skip [0:100]", + "SelfSeed: Copy [400:500] to [100:200]", + storeCopy(chunks[2], 200), + "SelfSeed: Copy [0:100] to [300:400]", + storeCopy(chunks[1], 400), + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + target := writeChunkFile(t, test.target...) + idx := chunkIndex(pickChunks(chunks, test.output...)...) + require.Equal(t, test.expected, planStrings(target, idx, planWithTargetIsBlank(false))) + }) + } +} + +func TestOverlapping(t *testing.T) { + // Chunks at [0:100] [100:200] [200:300] + chunks := indexSequence(1, 2, 3).Chunks + for _, test := range []struct { + start, end uint64 + lo, hi int + }{ + {0, 100, 0, 1}, // exactly the first chunk + {50, 150, 0, 2}, // across a boundary + {100, 200, 1, 2}, // touching the neighbours only + {150, 160, 1, 2}, // within a chunk + {0, 300, 0, 3}, // all chunks + {250, 400, 2, 3}, // past the end + {300, 400, 3, 3}, // beyond all chunks + } { + lo, hi := overlapping(chunks, test.start, test.end) + require.Equal(t, []int{test.lo, test.hi}, []int{lo, hi}, "range [%d:%d]", test.start, test.end) + } +} + +// Chunks that can be cloned are taken from wherever they can be, seeds first, +// before any chunks are copied. +func TestCloningPriority(t *testing.T) { + // Target: A B A with 256 byte chunks at [0:256] [256:512] [512:768]. + // With a blocksize of 64, the self-seed can clone A from 512 to 0. + a, b := ChunkID{0x0A}, ChunkID{0x0B} + target := Index{Chunks: []IndexChunk{ + {ID: a, Start: 0, Size: 256}, + {ID: b, Start: 256, Size: 256}, + {ID: a, Start: 512, Size: 256}, + }} + storeStep := func(id ChunkID, start uint64) string { + return fmt.Sprintf("Store: Copy %s to [%d:%d]", id, start, start+256) + } + + planSteps := func(t *testing.T, seedIndex Index) []string { + t.Helper() + name := filepath.Join(t.TempDir(), "target") + seed, err := NewFileSeed(name, "seed", seedIndex, IndexRegenerationSettings{}) + require.NoError(t, err) + seed.canReflink = true + + plan := newPlan(name, target, nil, planWithSeeds([]Seed{seed}), planWithBlocksize(64)) + plan.selfSeed.canReflink = true + return stepStrings(plan.replan().Steps()) + } + + t.Run("self-seed clone over seed copy", func(t *testing.T) { + // A is at an offset in the seed that can't be cloned to either + // position, so position 0 is cloned from position 2 instead, and + // only position 2 is copied from the seed. + got := planSteps(t, Index{Chunks: []IndexChunk{ + {ID: ChunkID{0xFF}, Start: 0, Size: 32}, + {ID: a, Start: 32, Size: 256}, + }}) + require.Equal(t, []string{ + "SelfSeed: Copy [512:768] to [0:256]", + storeStep(b, 256), + "FileSeed(seed): Copy to [512:768]", + }, got) + }) + + t.Run("seed clone over self-seed clone", func(t *testing.T) { + // A is aligned in the seed, both positions are cloned from there. + got := planSteps(t, Index{Chunks: []IndexChunk{ + {ID: a, Start: 0, Size: 256}, + }}) + require.Equal(t, []string{ + "FileSeed(seed): Copy to [0:256]", + storeStep(b, 256), + "FileSeed(seed): Copy to [512:768]", + }, got) + }) +} + +// countingSeed counts how often the seed is searched for a match. +type countingSeed struct { + *FileSeed + lookups int +} + +func (s *countingSeed) LongestMatchFrom(chunks []IndexChunk, startPos int) (int, int) { + s.lookups++ + return s.FileSeed.LongestMatchFrom(chunks, startPos) +} + +// A seed that can't clone isn't searched for matches to clone. Otherwise +// every position is looked up once more, only to find the match has to be +// copied. +func TestCloningPassSkipsSeedsThatCantClone(t *testing.T) { + idx := indexSequence(1, 2, 3) + seed, err := NewFileSeed("target", "seed", idx, IndexRegenerationSettings{}) + require.NoError(t, err) + seed.canReflink = false + counting := &countingSeed{FileSeed: seed} + + plan := newPlan("target", idx, nil, planWithSeeds([]Seed{counting}), planWithBlocksize(64)) + require.Equal(t, []string{"FileSeed(seed): Copy to [0:300]"}, stepStrings(plan.Steps())) + require.Equal(t, 1, counting.lookups, "one lookup finds the whole run to copy") +} + +func TestReflinkable(t *testing.T) { + for _, test := range []struct { + src, length, dst uint64 + want bool + }{ + {0, 256, 0, true}, // aligned, spans full blocks + {64, 256, 128, true}, // aligned relative to each other + {32, 256, 0, false}, // different alignment + {0, 64, 0, false}, // no full block after the first boundary + {16, 100, 16, false}, // too small for a full block + {16, 200, 1040, true}, // same alignment, full block inside + } { + require.Equal(t, test.want, reflinkable(test.src, test.length, test.dst, 64), + "src %d, length %d, dst %d", test.src, test.length, test.dst) + } +} diff --git a/assemble-selfseed.go b/assemble-selfseed.go new file mode 100644 index 0000000..7720a10 --- /dev/null +++ b/assemble-selfseed.go @@ -0,0 +1,139 @@ +package desync + +import ( + "fmt" + "os" +) + +// selfSeedMaxCandidates bounds the number of candidate positions the +// self-seed examines per match. Without a bound, chunks that repeat many +// times in the target make matching quadratic in the number of repetitions +// when reflinks are available (where run lengths aren't limited either). +const selfSeedMaxCandidates = 100 + +// selfSeed matches chunks against the file being assembled itself. Chunks +// that repeat in the target can be copied within the file once their first +// occurrence has been written. +type selfSeed struct { + file string + seedIndex + blocksize uint64 + + // inPlace looks up the chunks that are already in place in the target. + // Nothing writes over them, so unlike the rest of the target they can + // be copied from in either direction without waiting for other steps. + // inPlaceRun holds the number of consecutive chunks in place from each + // position. + inPlace seedIndex + inPlaceRun []int +} + +func newSelfSeed(file string, index Index, blocksize uint64) *selfSeed { + s := &selfSeed{ + file: file, + seedIndex: newSeedIndex(index, CanClone(file, file)), + blocksize: blocksize, + } + // Zero chunks are handled better by the null seed: writing zeros to a + // blank target is a no-op there, while a self-copy of zeros is real + // read/write I/O that also chains dependencies across the whole run. + // Remove the null chunk from the lookup so those positions fall through + // to the null seed. + delete(s.pos, NewNullChunk(index.Index.ChunkSizeMax).ID) + return s +} + +// useInPlace records the chunks found in place in the target, given the +// skips of the plan, one per chunk position, nil where the chunk isn't in +// place. +func (s *selfSeed) useInPlace(skips []*skipInPlace) { + s.inPlace = seedIndex{index: s.index, pos: make(map[ChunkID][]int), canReflink: s.canReflink} + s.inPlaceRun = make([]int, len(skips)) + for i := len(skips) - 1; i >= 0; i-- { + if skips[i] == nil { + continue + } + s.inPlaceRun[i] = 1 + if i+1 < len(skips) { + s.inPlaceRun[i] += s.inPlaceRun[i+1] + } + } + for i, sk := range skips { + id := s.index.Chunks[i].ID + // Leave out the null chunk, just like the other positions: it's + // the only chunk missing from s.pos. + if _, ok := s.pos[id]; sk != nil && ok { + s.inPlace.pos[id] = append(s.inPlace.pos[id], i) + } + } +} + +// LongestMatchFrom returns the longest sequence of matching chunks for a +// given starting position. It returns the chunk position of the match and +// the number of matching chunks, or (0, 0) if there is no match. Positions +// after startPos are considered, with matches clamped so the source and +// destination ranges can't overlap, as well as runs of chunks already in +// place anywhere in the target. +func (s *selfSeed) LongestMatchFrom(chunks []IndexChunk, startPos int) (int, int) { + pos, n := s.longestMatchFrom(chunks, startPos, startPos+1, selfSeedMaxCandidates, func(p int) int { + // The source run [p, p+n) may not overlap the destination run + // [startPos, startPos+n), which limits a usable run to the distance + // between the two. + return p - startPos + }) + // Chunks in place before startPos are only found here, if the target + // was checked for them. They're preferred over an equally long match + // elsewhere, as reading them doesn't depend on any other step. + inPos, inN := s.inPlace.longestMatchFrom(chunks, startPos, 0, selfSeedMaxCandidates, func(p int) int { + run := s.inPlaceRun[p] + if p > startPos { + run = min(run, p-startPos) + } + return run + }) + if inN > 0 && inN >= n { + return inPos, inN + } + return pos, n +} + +// GetSegment returns a segment copying n chunks starting at chunk position +// pos to dstOffset in the target. +func (s *selfSeed) GetSegment(pos, n int, dstOffset uint64) *selfSeedSegment { + return &selfSeedSegment{ + segment: newFileSeedSegment(s.file, s.index.Chunks[pos:pos+n], s.canReflink), + dstOffset: dstOffset, + blocksize: s.blocksize, + } +} + +type selfSeedSegment struct { + segment *fileSeedSegment + dstOffset uint64 + blocksize uint64 +} + +// Execute copies the chunks within the file being assembled. The source and +// destination ranges never overlap, the plan clamps matches to prevent that. +func (s *selfSeedSegment) Execute(f *os.File) (copied uint64, cloned uint64, err error) { + return s.segment.WriteInto(f, s.dstOffset, s.segment.Size(), s.blocksize, false) +} + +// access reads what other steps write to the target during assembly. +func (s *selfSeedSegment) access() targetAccess { + src, size := s.segment.chunks[0].Start, s.segment.Size() + return targetAccess{ + writes: byteRange{s.dstOffset, s.dstOffset + size}, + reads: byteRange{src, src + size}, + } +} + +func (s *selfSeedSegment) recordStats(stats *ExtractStats, numChunks int) { + stats.addChunksFromSeed(uint64(numChunks)) +} + +func (s *selfSeedSegment) String() string { + src, size := s.segment.chunks[0].Start, s.segment.Size() + return fmt.Sprintf("SelfSeed: Copy [%d:%d] to [%d:%d]", + src, src+size, s.dstOffset, s.dstOffset+size) +} diff --git a/assemble-skip.go b/assemble-skip.go new file mode 100644 index 0000000..6fefbf3 --- /dev/null +++ b/assemble-skip.go @@ -0,0 +1,27 @@ +package desync + +import ( + "fmt" + "os" +) + +// skipInPlace skips data chunks that are already in place. +type skipInPlace struct { + start uint64 + end uint64 +} + +func (s *skipInPlace) Execute(f *os.File) (copied uint64, cloned uint64, err error) { + return 0, 0, nil +} + +// access is empty, the data is in place already. +func (s *skipInPlace) access() targetAccess { return targetAccess{} } + +func (s *skipInPlace) recordStats(stats *ExtractStats, numChunks int) { + stats.addChunksInPlace(uint64(numChunks)) +} + +func (s *skipInPlace) String() string { + return fmt.Sprintf("InPlace: Skip [%d:%d]", s.start, s.end) +} diff --git a/assemble-stash.go b/assemble-stash.go new file mode 100644 index 0000000..7cbc3f2 --- /dev/null +++ b/assemble-stash.go @@ -0,0 +1,140 @@ +package desync + +import ( + "fmt" + "math" + "os" + "runtime/debug" + "runtime/metrics" + "sync" +) + +// stash holds the source of an in-place move in memory when other +// steps overwrite it before the move runs, which happens in dependency cycles +// like two chunks swapping places. The first step about to overwrite the +// source fills it, and the move releases it once it wrote the data. Filling +// on demand rather than up front keeps the memory held at any time low. If +// the stash doesn't fit into the memory budget, it's dropped instead and the +// move takes its chunk from the store. +type stash struct { + offset uint64 + size uint64 + budget *stashBudget + + mu sync.Mutex + state stashState + data []byte +} + +type stashState int + +const ( + stashEmpty stashState = iota // the source is intact and wasn't read + stashFilled // the source is held in data + stashTaken // the move has the data + stashDropped // not held, the move uses the store +) + +// fill reads the source into memory before the calling step overwrites it. +// It does nothing if that already happened, or if the move ran already. +func (b *stash) fill(f *os.File) error { + b.mu.Lock() + defer b.mu.Unlock() + if b.state != stashEmpty { + return nil + } + if !b.budget.take(b.size) { + b.state = stashDropped + return nil + } + data := make([]byte, b.size) + if _, err := f.ReadAt(data, int64(b.offset)); err != nil { + b.budget.give(b.size) + return fmt.Errorf("stash read at %d: %w", b.offset, err) + } + b.data, b.state = data, stashFilled + return nil +} + +// take returns the source data for the move and releases the stash. It +// reads the source directly if nothing overwrote it yet. It returns nil if +// the stash was dropped. +func (b *stash) take(f *os.File) ([]byte, error) { + b.mu.Lock() + defer b.mu.Unlock() + switch b.state { + case stashEmpty: + data := make([]byte, b.size) + if _, err := f.ReadAt(data, int64(b.offset)); err != nil { + return nil, fmt.Errorf("inPlaceCopy read at %d: %w", b.offset, err) + } + b.state = stashTaken + return data, nil + case stashFilled: + data := b.data + b.data, b.state = nil, stashTaken + b.budget.give(b.size) + return data, nil + default: + return nil, nil + } +} + +// pending reports whether the stash still needs to be filled. +func (b *stash) pending() bool { + b.mu.Lock() + defer b.mu.Unlock() + return b.state == stashEmpty +} + +// dropped reports whether the stash was dropped for lack of memory. +func (b *stash) dropped() bool { + b.mu.Lock() + defer b.mu.Unlock() + return b.state == stashDropped +} + +// stashBudget is the memory the stashes of in-place moves may hold at the +// same time. +type stashBudget struct { + mu sync.Mutex + available int64 +} + +func (b *stashBudget) take(n uint64) bool { + b.mu.Lock() + defer b.mu.Unlock() + if int64(n) > b.available { + return false + } + b.available -= int64(n) + return true +} + +func (b *stashBudget) give(n uint64) { + b.mu.Lock() + defer b.mu.Unlock() + b.available += int64(n) +} + +// stashMemoryLimit returns the memory in-place stashes may use. It is an +// indirection over memoryLimitBudget for tests. +var stashMemoryLimit = memoryLimitBudget + +// memoryLimitBudget returns the memory left under the Go runtime's memory +// limit, set with GOMEMLIMIT. Without a limit, the budget is unbounded. +func memoryLimitBudget() int64 { + limit := debug.SetMemoryLimit(-1) + if limit == math.MaxInt64 { + return math.MaxInt64 + } + // The runtime compares the limit against all memory it mapped, less + // what it returned to the OS. + samples := []metrics.Sample{ + {Name: "/memory/classes/total:bytes"}, + {Name: "/memory/classes/heap/released:bytes"}, + } + metrics.Read(samples) + inUse := int64(samples[0].Value.Uint64() - samples[1].Value.Uint64()) + return max(limit-inUse, 0) +} diff --git a/assemble-stash_test.go b/assemble-stash_test.go new file mode 100644 index 0000000..4e84496 --- /dev/null +++ b/assemble-stash_test.go @@ -0,0 +1,106 @@ +package desync + +import ( + "math" + "os" + "path/filepath" + "runtime/debug" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestStash(t *testing.T) { + name := filepath.Join(t.TempDir(), "target") + require.NoError(t, os.WriteFile(name, []byte("0123456789"), 0644)) + f, err := os.OpenFile(name, os.O_RDWR, 0) + require.NoError(t, err) + defer f.Close() + overwrite := func() { + _, err := f.WriteAt([]byte("xxxxx"), 0) + require.NoError(t, err) + } + reset := func() { + _, err := f.WriteAt([]byte("01234"), 0) + require.NoError(t, err) + } + + t.Run("taken without filling", func(t *testing.T) { + // Nothing overwrote the source, the move reads it directly + budget := &stashBudget{available: 0} + b := &stash{offset: 2, size: 3, budget: budget} + data, err := b.take(f) + require.NoError(t, err) + require.Equal(t, []byte("234"), data) + + // Steps overwriting the source later don't fill it anymore + require.NoError(t, b.fill(f)) + require.False(t, b.pending()) + require.False(t, b.dropped()) + }) + + t.Run("filled before being overwritten", func(t *testing.T) { + defer reset() + budget := &stashBudget{available: 3} + b := &stash{offset: 2, size: 3, budget: budget} + require.True(t, b.pending()) + require.NoError(t, b.fill(f)) + require.Equal(t, int64(0), budget.available) + overwrite() + + data, err := b.take(f) + require.NoError(t, err) + require.Equal(t, []byte("234"), data) + require.Equal(t, int64(3), budget.available, "memory returned to the budget") + }) + + t.Run("dropped without budget", func(t *testing.T) { + defer reset() + b := &stash{offset: 2, size: 3, budget: &stashBudget{available: 2}} + require.NoError(t, b.fill(f)) + require.True(t, b.dropped()) + overwrite() + + data, err := b.take(f) + require.NoError(t, err) + require.Nil(t, data, "the move has to use the store") + }) +} + +func TestStepQueue(t *testing.T) { + pending := &stash{budget: &stashBudget{}} + filled := &stash{state: stashFilled} + var ( + releasing = &planStep{source: &inPlaceCopy{stash: pending}} + filling = &planStep{source: &skipInPlace{}, fills: []*stash{pending}} + plain = &planStep{source: &skipInPlace{}} + refilling = &planStep{source: &skipInPlace{}, fills: []*stash{filled}} + ) + var q stepQueue + for _, s := range []*planStep{filling, plain, refilling, releasing} { + q.push(s) + } + var got []*planStep + for s := q.peek(); s != nil; s = q.peek() { + got = append(got, s) + q.pop() + } + // Stashes are released first and new ones filled last. Filling a stash + // that's filled already doesn't hold more memory. + require.Equal(t, []*planStep{releasing, plain, refilling, filling}, got) +} + +func TestMemoryLimitBudget(t *testing.T) { + orig := debug.SetMemoryLimit(-1) + t.Cleanup(func() { debug.SetMemoryLimit(orig) }) + + debug.SetMemoryLimit(math.MaxInt64) + require.Equal(t, int64(math.MaxInt64), memoryLimitBudget()) + + // With a limit, the budget is what's left under it + const limit = 1 << 40 + debug.SetMemoryLimit(limit) + budget := memoryLimitBudget() + require.Greater(t, budget, int64(0)) + require.Less(t, budget, int64(limit)) +} diff --git a/assemble-step.go b/assemble-step.go new file mode 100644 index 0000000..271dc7d --- /dev/null +++ b/assemble-step.go @@ -0,0 +1,108 @@ +package desync + +import "os" + +type planStep struct { + source assembleSource + + // fills are the stashes of in-place moves whose source this step + // overwrites. They are filled before the step runs. + fills []*stash + + // numChunks is the number of index chunks this step covers. + numChunks int + + // Steps that depend on this one. + dependents map[*planStep]struct{} + + // Steps that this one depends on. + dependencies map[*planStep]struct{} +} + +// before records that step "to" depends on s, setting both directions of the +// edge. +func (s *planStep) before(to *planStep) { + if s.dependents == nil { + s.dependents = make(map[*planStep]struct{}) + } + s.dependents[to] = struct{}{} + if to.dependencies == nil { + to.dependencies = make(map[*planStep]struct{}) + } + to.dependencies[s] = struct{}{} +} + +// ready returns true when all dependencies have been resolved. +func (n *planStep) ready() bool { + return len(n.dependencies) == 0 +} + +// execute fills the stashes of the in-place moves whose source the step +// overwrites, then runs it. +func (n *planStep) execute(f *os.File) (copied uint64, cloned uint64, err error) { + for _, b := range n.fills { + if err := b.fill(f); err != nil { + return 0, 0, err + } + } + return n.source.Execute(f) +} + +// releasesStash reports whether the step is an in-place move with a stash, +// which frees the stash's memory. +func (n *planStep) releasesStash() bool { + c, ok := n.source.(*inPlaceCopy) + return ok && c.stash != nil +} + +// fillsStash reports whether the step has to fill any stashes that aren't +// filled yet. +func (n *planStep) fillsStash() bool { + for _, b := range n.fills { + if b.pending() { + return true + } + } + return false +} + +// stepQueue holds the steps that are ready to run. To keep the memory held by +// in-place stashes low, it hands out steps that release stashes first, and +// those that fill new stashes last. Steps are otherwise handed out in the +// order they became ready. +type stepQueue struct { + releasing []*planStep + other []*planStep + filling []*planStep +} + +func (q *stepQueue) push(s *planStep) { + switch { + case s.releasesStash(): + q.releasing = append(q.releasing, s) + case s.fillsStash(): + q.filling = append(q.filling, s) + default: + q.other = append(q.other, s) + } +} + +// peek returns the next step without removing it, or nil if there is none. +func (q *stepQueue) peek() *planStep { + for _, l := range []*[]*planStep{&q.releasing, &q.other, &q.filling} { + if len(*l) > 0 { + return (*l)[0] + } + } + return nil +} + +// pop removes the step returned by peek. +func (q *stepQueue) pop() { + for _, l := range []*[]*planStep{&q.releasing, &q.other, &q.filling} { + if len(*l) > 0 { + *l = (*l)[1:] + return + } + } +} diff --git a/assemble-store.go b/assemble-store.go new file mode 100644 index 0000000..8aa161e --- /dev/null +++ b/assemble-store.go @@ -0,0 +1,39 @@ +package desync + +import ( + "fmt" + "os" +) + +type copyFromStore struct { + store Store + chunk IndexChunk +} + +func (s *copyFromStore) Execute(f *os.File) (copied uint64, cloned uint64, err error) { + chunk, err := s.store.GetChunk(s.chunk.ID) + if err != nil { + return 0, 0, err + } + b, err := chunk.Data() + if err != nil { + return 0, 0, err + } + if s.chunk.Size != uint64(len(b)) { + return 0, 0, fmt.Errorf("unexpected size for chunk %s", s.chunk.ID) + } + _, err = f.WriteAt(b, int64(s.chunk.Start)) + return 0, 0, err +} + +func (s *copyFromStore) access() targetAccess { + return targetAccess{writes: byteRange{s.chunk.Start, s.chunk.Start + s.chunk.Size}} +} + +func (s *copyFromStore) recordStats(stats *ExtractStats, numChunks int) { + stats.incChunksFromStore() +} + +func (s *copyFromStore) String() string { + return fmt.Sprintf("Store: Copy %v to [%d:%d]", s.chunk.ID.String(), s.chunk.Start, s.chunk.Start+s.chunk.Size) +} diff --git a/assemble.go b/assemble.go index 7066627..2cd112b 100644 --- a/assemble.go +++ b/assemble.go @@ -2,6 +2,7 @@ package desync import ( "context" + "errors" "fmt" "os" "slices" @@ -27,62 +28,6 @@ type AssembleOptions struct { InvalidSeedAction InvalidSeedAction } -// writeChunk tries to write a chunk by looking at the self seed, if it is already existing in the -// destination file or by taking it from the store. The in-place check runs first to avoid unnecessary -// writes. If the target already has the correct data, no write is performed. buf is scratch space -// for reading back the destination, reused across calls to avoid an allocation per chunk. -func writeChunk(c IndexChunk, ss *selfSeed, f *os.File, blocksize uint64, s Store, stats *ExtractStats, isBlank bool, buf []byte) error { - // If we operate on an existing file there's a good chance we already - // have the data written for this chunk. Let's read it from disk and - // compare to what is expected. This is checked first to avoid rewriting - // data that is already correct, even for chunks available in the selfSeed. - if !isBlank { - b := slices.Grow(buf[:0], int(c.Size))[:c.Size] - if _, err := f.ReadAt(b, int64(c.Start)); err != nil { - return err - } - sum := Digest.Sum(b) - if sum == c.ID { - // Record we kept this chunk in the file (when using in-place extract) - stats.incChunksInPlace() - return nil - } - } - - // If we already took this chunk from the store we can reuse it by looking - // into the selfSeed. - if segment := ss.getChunk(c.ID); segment != nil { - copied, cloned, err := segment.WriteInto(f, c.Start, c.Size, blocksize, isBlank) - if err != nil { - return err - } - stats.addBytesCopied(copied) - stats.addBytesCloned(cloned) - return nil - } - - // Record this chunk having been pulled from the store - stats.incChunksFromStore() - // Pull the (compressed) chunk from the store - chunk, err := s.GetChunk(c.ID) - if err != nil { - return err - } - b, err := chunk.Data() - if err != nil { - return err - } - // Might as well verify the chunk size while we're at it - if c.Size != uint64(len(b)) { - return fmt.Errorf("unexpected size for chunk %s", c.ID) - } - // Write the decompressed chunk into the file at the right position - if _, err = f.WriteAt(b, int64(c.Start)); err != nil { - return err - } - return nil -} - // AssembleFile re-assembles a file based on a list of index chunks. It runs n // goroutines, creating one filehandle for the file "name" per goroutine // and writes to the file simultaneously. If progress is provided, it'll be @@ -92,18 +37,11 @@ func writeChunk(c IndexChunk, ss *selfSeed, f *os.File, blocksize uint64, s Stor // differ from the expected content. This can be used to complete partly // written files. func AssembleFile(ctx context.Context, name string, idx Index, s Store, seeds []Seed, options AssembleOptions) (*ExtractStats, error) { - type Job struct { - segment IndexSegment - source SeedSegment - } var ( - attempt = 1 - in = make(chan Job) isBlank bool isBlkDevice bool - pb ProgressBar + attempt = 1 ) - g, ctx := errgroup.WithContext(ctx) // Initialize stats to be gathered during extraction stats := &ExtractStats{ @@ -131,12 +69,47 @@ func AssembleFile(ctx context.Context, name string, idx Index, s Store, seeds [] isBlank = true } + // Separate the in-place seed (if any) from the file seeds. It reads + // from the file being assembled and is handed to the plan explicitly. + // A file seed whose data file is the target can't be read from while + // the target is being written, it's used in place instead. There can + // only be one, extras are dropped. + targetInfo, err := os.Stat(name) + if err != nil { + return stats, err + } + isTarget := func(file string) bool { + info, err := os.Stat(file) + return err == nil && os.SameFile(info, targetInfo) + } + var ( + inPlaceSeed *FileSeed + fileSeeds []Seed + ) + for _, seed := range seeds { + if fs, ok := seed.(*FileSeed); ok && isTarget(fs.srcFile) { + if inPlaceSeed == nil { + inPlaceSeed = fs + } + continue + } + fileSeeds = append(fileSeeds, seed) + } + // Truncate the output file to the full expected size. Not only does this // confirm there's enough disk space, but it allows for an optimization // when dealing with the Null Chunk. On Darwin, the file is physically // pre-allocated as well since sparse files on APFS have shown to cause - // issues when written to concurrently. - if !isBlkDevice { + // issues when written to concurrently. If the file is larger than the + // output, shrinking it waits until after assembly as the data beyond the + // end may still be moved into place. A target without chunks is + // truncated right away as there is nothing to assemble. + var size int64 + if info != nil { + size = info.Size() + } + shrinkAfter := !isBlkDevice && size > idx.Length() && len(idx.Chunks) > 0 + if !isBlkDevice && !shrinkAfter { if err := preallocateFile(name, idx.Length()); err != nil { return stats, err } @@ -158,155 +131,172 @@ func AssembleFile(ctx context.Context, name string, idx Index, s Store, seeds [] return stats, err } defer ns.close() - seeds = append([]Seed{ns}, seeds...) - - // Start a self-seed which will become usable once chunks are written contiguously - // beginning at position 0. There is no need to add this to the seeds list because - // when we create a plan it will be empty. - ss, err := newSelfSeed(name, idx) - if err != nil { - return stats, err - } + fileSeeds = append([]Seed{ns}, fileSeeds...) // Record the total number of seeds and blocksize in the stats - stats.Seeds = len(seeds) + stats.Seeds = len(fileSeeds) + if inPlaceSeed != nil { + stats.Seeds++ + } stats.Blocksize = blocksize - // Start the workers, each having its own filehandle to write concurrently - for i := 0; i < options.N; i++ { - f, err := os.OpenFile(name, os.O_RDWR, 0666) - if err != nil { - return stats, fmt.Errorf("unable to open file %s, %s", name, err) + // Create the plan and validate the seed indexes. Regenerating or + // skipping invalid seeds restarts planning with the modified seeds. + plan := newPlan(name, idx, s, + planWithConcurrency(options.N), + planWithSeeds(fileSeeds), + planWithInPlaceSeed(inPlaceSeed), + planWithTargetIsBlank(isBlank), + planWithBlocksize(blocksize), + planWithStashLimit(stashMemoryLimit()), + ) + for { + err := plan.Validate(NewProgressBar(fmt.Sprintf("Attempt %d: Validating ", attempt))) + if err == nil { + break + } + var seedError SeedInvalid + if !errors.As(err, &seedError) { + return stats, err } - defer f.Close() - g.Go(func() error { - // Scratch buffer for reading chunks back from the destination, - // reused for all chunks this worker processes. Grown on demand - // if the index reports no (or a too small) max chunk size. - buf := make([]byte, idx.Index.ChunkSizeMax) - for job := range in { - pb.Add(job.segment.lengthChunks()) - if job.source != nil { - // If we have a seedSegment we expect 1 or more chunks between - // the start and the end of this segment. - stats.addChunksFromSeed(uint64(job.segment.lengthChunks())) - offset := job.segment.start() - length := job.segment.lengthBytes() - copied, cloned, err := job.source.WriteInto(f, offset, length, blocksize, isBlank) - if err != nil { - return err - } - // Null segments (identified by an empty filename, like in - // Plan.Validate) skip the read-back validation below: they write - // deterministic zeros with no external source that could change, - // and reading back and hashing large null sections is expensive. - if job.source.FileName() == "" { - stats.addBytesCopied(copied) - stats.addBytesCloned(cloned) - ss.add(job.segment) - continue - } + switch options.InvalidSeedAction { + case InvalidSeedActionBailOut: + return stats, err + case InvalidSeedActionRegenerate: + Log.WithError(err).Info("Unable to use one or more seeds, regenerating them") + for i, s := range seedError.Seeds { + if err := s.RegenerateIndex(ctx, options.N, attempt, i+1); err != nil { + return stats, err + } + } + attempt++ + case InvalidSeedActionSkip: + Log.WithError(err).Infof("Unable to use one or more seeds, skipping them") + if inPlaceSeed != nil && slices.Contains(seedError.Seeds, Seed(inPlaceSeed)) { + inPlaceSeed = nil + } + fileSeeds = slices.DeleteFunc(fileSeeds, func(s Seed) bool { + return slices.Contains(seedError.Seeds, s) + }) + default: + panic("Unhandled InvalidSeedAction") + } + plan = plan.replan(planWithSeeds(fileSeeds), planWithInPlaceSeed(inPlaceSeed)) + } - // Validate that the written chunks are exactly what we were expecting. - // Because the seed might point to a RW location, if the data changed - // while we were extracting an index, we might end up writing to the - // destination some unexpected values. - for _, c := range job.segment.chunks() { - buf = slices.Grow(buf[:0], int(c.Size))[:c.Size] - if _, err := f.ReadAt(buf, int64(c.Start)); err != nil { - return err - } - sum := Digest.Sum(buf) - if sum != c.ID { - if options.InvalidSeedAction == InvalidSeedActionRegenerate { - // Try harder before giving up and aborting - Log.WithField("ID", c.ID).Info("The seed may have changed during processing, trying to take the chunk from the self seed or the store") - if err := writeChunk(c, ss, f, blocksize, s, stats, isBlank, buf); err != nil { - return err - } - } else { - return fmt.Errorf("written data in %s doesn't match its expected hash value, seed may have changed during processing", name) - } - } - } + // Generate the plan steps necessary to build the target + steps := plan.Steps() + if len(steps) == 0 { + return stats, nil + } - stats.addBytesCopied(copied) - stats.addBytesCloned(cloned) - // Record this segment's been written in the self-seed to make it - // available going forward - ss.add(job.segment) - continue - } + // Set up progress bar + pb := NewProgressBar(fmt.Sprintf("Attempt %d: Assembling ", attempt)) + pb.SetTotal(len(idx.Chunks)) + pb.Start() + defer pb.Finish() - // If we don't have a seedSegment we expect an IndexSegment with just - // a single chunk, that we can take from either the selfSeed, from the - // destination file, or from the store. - if len(job.segment.chunks()) != 1 { - panic("Received an unexpected segment that doesn't contain just a single chunk") - } - c := job.segment.chunks()[0] + // Workers take steps that can run from one channel and report them on + // another once complete. + var ( + work = make(chan *planStep) + completed = make(chan *planStep) + ) + + g, ctx := errgroup.WithContext(ctx) - if err := writeChunk(c, ss, f, blocksize, s, stats, isBlank, buf); err != nil { + // Bring up the workers + for range options.N { + g.Go(func() error { + f, err := os.OpenFile(name, os.O_RDWR, 0666) + if err != nil { + return fmt.Errorf("unable to open file %s, %s", name, err) + } + defer f.Close() + for step := range work { + copied, cloned, err := step.execute(f) + if err != nil { return err } - - // Record this chunk's been written in the self-seed. - // Even if we already confirmed that this chunk is present in the - // self-seed, we still need to record it as being written, otherwise - // the self-seed position pointer doesn't advance as we expect. - ss.add(job.segment) + // Update byte-level stats + stats.addBytesCopied(copied) + stats.addBytesCloned(cloned) + // Update chunk-level stats + step.source.recordStats(stats, step.numChunks) + select { + case completed <- step: + case <-ctx.Done(): + return ctx.Err() + } } return nil }) } - // Let the sequencer break up the index into segments, create and validate a plan, - // feed the workers, and stop if there are any errors - seq := NewSeedSequencer(idx, seeds...) - plan := seq.Plan() - for { - validatingPrefix := fmt.Sprintf("Attempt %d: Validating ", attempt) - if err := plan.Validate(ctx, options.N, NewProgressBar(validatingPrefix)); err != nil { - // This plan has at least one invalid seed - switch options.InvalidSeedAction { - case InvalidSeedActionBailOut: - return stats, err - case InvalidSeedActionRegenerate: - Log.WithError(err).Info("Unable to use one of the chosen seeds, regenerating it") - if err := seq.RegenerateInvalidSeeds(ctx, options.N, attempt); err != nil { - return stats, err + // Hand out the steps as they become ready, until all are complete or + // a worker failed. + var queue stepQueue + for _, step := range steps { + if step.ready() { + queue.push(step) + } + } + remaining := len(steps) +dispatch: + for remaining > 0 { + next := queue.peek() + var out chan<- *planStep + if next != nil { + out = work + } + select { + case out <- next: + queue.pop() + case step := <-completed: + remaining-- + pb.Add(step.numChunks) + + // Remove the dependency from the steps blocked by this one, + // queueing those that have no more dependencies. + for b := range step.dependents { + delete(b.dependencies, step) + if b.ready() { + queue.push(b) } - case InvalidSeedActionSkip: - // Recreate the plan. This time the seed marked as invalid will be skipped - Log.WithError(err).Info("Unable to use one of the chosen seeds, skipping it") - default: - panic("Unhandled InvalidSeedAction") } - - attempt += 1 - seq.Rewind() - plan = seq.Plan() - continue + case <-ctx.Done(): + break dispatch } - // Found a valid plan - break } + close(work) - pb = NewProgressBar(fmt.Sprintf("Attempt %d: Assembling ", attempt)) - pb.SetTotal(len(idx.Chunks)) - pb.Start() - defer pb.Finish() + // Wait for the workers to complete. They exit without error when the + // context is cancelled while they wait for a step, so the assembly is + // only complete if all steps are. + err = g.Wait() + if err == nil && remaining > 0 { + err = ctx.Err() + } -loop: - for _, segment := range plan { - select { - case <-ctx.Done(): - break loop - case in <- Job{segment.indexSegment, segment.source}: + // A seed file that was written to while it was read may have been copied + // into the target in its new state. Check the output and take the chunks + // that don't match from the store. + if err == nil { + if changed := plan.changedSeeds(); len(changed) > 0 { + Log.WithField("seeds", changed).Warn("Seeds changed during assembly, verifying the output") + if err := backfill(name, idx, s, options.N, stats); err != nil { + return stats, err + } + } + } + + // Shrink the file now that all in-place reads are complete. + if err == nil && shrinkAfter { + if err := os.Truncate(name, idx.Length()); err != nil { + return stats, err } } - close(in) - return stats, g.Wait() + return stats, err } diff --git a/assemble_test.go b/assemble_test.go index 0653d4a..a10c714 100644 --- a/assemble_test.go +++ b/assemble_test.go @@ -1,24 +1,41 @@ package desync import ( + "bytes" "context" "crypto/md5" "crypto/rand" "fmt" + "math" + mrand "math/rand/v2" "os" "path/filepath" + "slices" "testing" "github.com/folbricht/desync/pkg/chunkers" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +// Build an index with a pre-determined set of (potentially repeated) chunks +func indexSequence(ids ...uint8) Index { + var ( + chunks = make([]IndexChunk, len(ids)) + start uint64 = 0 + size uint64 = 100 + ) + for i, id := range ids { + chunks[i] = IndexChunk{Start: start, Size: size, ID: ChunkID{id}} + start += size + } + return Index{Chunks: chunks} +} + func TestExtract(t *testing.T) { // Make a test file that's guaranteed to have duplicate chunks. b, err := os.ReadFile("testdata/chunker.input") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) for range 4 { // Replicate it a few times to make sure we get dupes b = append(b, b...) } @@ -38,27 +55,21 @@ func TestExtract(t *testing.T) { chunkers.DefaultChunkingSettings(), NewProgressBar(""), ) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) // Chop up the input file into a (temporary) local store store := t.TempDir() s, err := NewLocalStore(store, StoreOptions{}) - if err != nil { - t.Fatal(err) - } - if err := ChopFile(context.Background(), in, index.Chunks, s, 10, NewProgressBar("")); err != nil { - t.Fatal(err) - } + require.NoError(t, err) + + err = ChopFile(context.Background(), in, index.Chunks, s, 10, NewProgressBar("")) + require.NoError(t, err) // Make a blank store - used to test a case where no chunk *should* be requested blankstore := t.TempDir() bs, err := NewLocalStore(blankstore, StoreOptions{}) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) // Prepare output files for each test - first a non-existing one out1 := filepath.Join(tmp, "out1") @@ -74,20 +85,9 @@ func TestExtract(t *testing.T) { out3 := filepath.Join(tmp, "out3") require.NoError(t, os.WriteFile(out3, b, 0644)) - // At this point we have the data needed for the test setup - // in - Temp file that represents the original input file - // inSub - MD5 of the input file - // index - Index file for the input file - // s - Local store containing the chunks needed to rebuild the input file - // bs - A blank local store, all GetChunk fail on it - // out1 - Just a non-existing file that gets assembled - // out2 - The output file already fully complete, no GetChunk should be needed - // out3 - Partial/damaged file with most, but not all data correct - // seedIndex + seedFile - Seed file to help assemble the input tests := map[string]struct { outfile string store Store - seed []Seed }{ "extract to new file": {outfile: out1, store: s}, "extract to complete file": {outfile: out2, store: bs}, @@ -96,19 +96,16 @@ func TestExtract(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { - if _, err := AssembleFile(context.Background(), test.outfile, index, test.store, nil, + _, err := AssembleFile(context.Background(), test.outfile, index, test.store, nil, AssembleOptions{10, InvalidSeedActionBailOut}, - ); err != nil { - t.Fatal(err) - } - b, err := os.ReadFile(test.outfile) - if err != nil { - t.Fatal(err) - } - outSum := md5.Sum(b) - if inSum != outSum { - t.Fatal("checksum of extracted file doesn't match expected") - } + ) + require.NoError(t, err) + + outBytes, err := os.ReadFile(test.outfile) + require.NoError(t, err) + + outSum := md5.Sum(outBytes) + assert.Equal(t, inSum, outSum, "checksum of extracted file doesn't match expected") }) } } @@ -117,9 +114,7 @@ func TestSeed(t *testing.T) { // Prepare different types of data slices that'll be used to assemble target // and seed files with varying amount of duplication data1, err := os.ReadFile("testdata/chunker.input") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) null := make([]byte, 4*chunkers.DefaultChunkSizeMax) rand1 := make([]byte, 4*chunkers.DefaultChunkSizeMax) rand.Read(rand1) @@ -130,9 +125,7 @@ func TestSeed(t *testing.T) { store := t.TempDir() s, err := NewLocalStore(store, StoreOptions{}) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) // Define tests with files with different content, by building files out // of sets of byte slices to create duplication or not between the target and @@ -191,14 +184,11 @@ func TestSeed(t *testing.T) { chunkers.DefaultChunkingSettings(), NewProgressBar(""), ) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) // Chop up the input file into the store - if err := ChopFile(context.Background(), dst, dstIndex.Chunks, s, 10, NewProgressBar("")); err != nil { - t.Fatal(err) - } + err = ChopFile(context.Background(), dst, dstIndex.Chunks, s, 10, NewProgressBar("")) + require.NoError(t, err) // Build the seed files and indexes then populate the array of seeds var seeds []Seed @@ -212,29 +202,20 @@ func TestSeed(t *testing.T) { chunkers.DefaultChunkingSettings(), NewProgressBar(""), ) - if err != nil { - t.Fatal(err) - } - seed, err := NewIndexSeed(dst, seedFile, seedIndex, IndexRegenerationSettings{}) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) + seed, err := NewFileSeed(dst, seedFile, seedIndex, IndexRegenerationSettings{}) + require.NoError(t, err) seeds = append(seeds, seed) } - if _, err := AssembleFile(context.Background(), dst, dstIndex, s, seeds, + _, err = AssembleFile(context.Background(), dst, dstIndex, s, seeds, AssembleOptions{10, InvalidSeedActionBailOut}, - ); err != nil { - t.Fatal(err) - } + ) + require.NoError(t, err) b, err := os.ReadFile(dst) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) outSum := md5.Sum(b) - if dstSum != outSum { - t.Fatal("checksum of extracted file doesn't match expected") - } + assert.Equal(t, dstSum, outSum, "checksum of extracted file doesn't match expected") }) } @@ -248,9 +229,7 @@ func TestSelfSeedInPlace(t *testing.T) { store := t.TempDir() s, err := NewLocalStore(store, StoreOptions{}) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) // Build a number of fake chunks that can then be used in the test in any order type rawChunk struct { @@ -265,9 +244,7 @@ func TestSelfSeedInPlace(t *testing.T) { b := make([]byte, size) rand.Read(b) chunk := NewChunk(b) - if err = s.StoreChunk(chunk); err != nil { - t.Fatal(err) - } + require.NoError(t, s.StoreChunk(chunk)) chunks[i] = rawChunk{chunk.ID(), b} } @@ -322,26 +299,18 @@ func TestSelfSeedInPlace(t *testing.T) { stats, err := AssembleFile(context.Background(), dst, idx, s, nil, AssembleOptions{1, InvalidSeedActionBailOut}, ) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) // Compare the checksums to that of the input data b, err = os.ReadFile(dst) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) outSum := md5.Sum(b) - if sum != outSum { - t.Fatal("checksum of extracted file doesn't match expected") - } + assert.Equal(t, sum, outSum, "checksum of extracted file doesn't match expected") - // All chunks must be in-place. The in-place check in writeChunk - // runs before the self-seed lookup, so repeated chunks are not - // re-written from the self-seed. - if stats.ChunksInPlace != uint64(len(test.index)) { - t.Fatalf("expected all %d chunks in-place, got %d", len(test.index), stats.ChunksInPlace) - } + // All chunks must be in-place. The plan generator creates skip + // placements before matching the self-seed, so repeated chunks + // are not re-written from the self-seed. + assert.Equal(t, uint64(len(test.index)), stats.ChunksInPlace, "expected all chunks in-place") }) } @@ -399,14 +368,520 @@ func join(slices ...[]byte) []byte { return out } -func readCaibxFile(t *testing.T, indexLocation string) (idx Index) { - is, err := NewLocalIndexStore(filepath.Dir(indexLocation)) +// testChunk is a chunk of known content, used to build indexes and file +// content from the same set of chunks. +type testChunk struct { + id ChunkID + data []byte +} + +// randomChunks returns chunks of the given sizes filled with random data. +func randomChunks(sizes ...int) []testChunk { + chunks := make([]testChunk, len(sizes)) + for i, size := range sizes { + b := make([]byte, size) + rand.Read(b) + chunks[i] = testChunk{id: Digest.Sum(b), data: b} + } + return chunks +} + +// filledChunk returns a chunk of size bytes, all of them set to fill. +func filledChunk(size int, fill byte) testChunk { + b := bytes.Repeat([]byte{fill}, size) + return testChunk{id: Digest.Sum(b), data: b} +} + +// pickChunks returns the chunks at the given positions, the layout of a file +// in a test scenario. +func pickChunks(chunks []testChunk, positions ...int) []testChunk { + out := make([]testChunk, len(positions)) + for i, pos := range positions { + out[i] = chunks[pos] + } + return out +} + +// chunkIndex lays the chunks out contiguously in the order given, with the +// maximum chunk size AssembleFile needs for its null chunk seed. +func chunkIndex(chunks ...testChunk) Index { + idx := Index{Chunks: make([]IndexChunk, len(chunks))} + var start uint64 + for i, c := range chunks { + size := uint64(len(c.data)) + idx.Chunks[i] = IndexChunk{ID: c.id, Start: start, Size: size} + idx.Index.ChunkSizeMax = max(idx.Index.ChunkSizeMax, size) + start += size + } + return idx +} + +// chunkContent returns the data of the chunks, concatenated. It's both the +// content of a file and the expected output of an assembly. +func chunkContent(chunks ...testChunk) []byte { + var out []byte + for _, c := range chunks { + out = append(out, c.data...) + } + return out +} + +// chunkStore returns a store holding only the given chunks, so that a chunk +// the plan routes to the store unexpectedly fails with ChunkMissing rather +// than succeeding silently. +func chunkStore(chunks ...testChunk) *TestStore { + s := &TestStore{Chunks: make(map[ChunkID][]byte, len(chunks))} + for _, c := range chunks { + s.Chunks[c.id] = c.data + } + return s +} + +// writeChunkFile writes the chunks' content to a file in a new temporary +// directory and returns its path. +func writeChunkFile(t *testing.T, chunks ...testChunk) string { + t.Helper() + path := filepath.Join(t.TempDir(), "target") + require.NoError(t, os.WriteFile(path, chunkContent(chunks...), 0644)) + return path +} + +// testAssembleOptions are the options the assembly tests run with: several +// workers, and no tolerance for seeds that don't match their data. +var testAssembleOptions = AssembleOptions{N: 4, InvalidSeedAction: InvalidSeedActionBailOut} + +// TestAssembleIntegration exercises the full assembly pipeline end-to-end, +// combining all source types in a single reconstruction: in-place skips, +// in-place copies (including cycles broken by stashing), self-seed, +// file seeds, and store fetches. It uses variable-size chunks so that +// byte-offset calculations, overlap detection, and stash sizing are tested +// with non-uniform boundaries. +// +// Each scenario writes an "old" file (the in-place seed), then calls +// AssembleFile to reconstruct a different target layout. The test verifies +// both the output content (md5 checksum + file size) and the per-source +// chunk statistics reported by ExtractStats. +func TestAssembleIntegration(t *testing.T) { + // Chunks of different sizes make sure the offset math in overlap + // detection, the stash sizing of in-place moves and cycle breaking + // are exercised with non-trivial byte boundaries. + chunks := randomChunks(1024, 768, 512, 896, 640, 1152, 384, 1280, 576, 704) + + // Named constants for chunk indices to make scenario definitions readable. + const ( + A = 0 // 1024 bytes + B = 1 // 768 bytes + C = 2 // 512 bytes + D = 3 // 896 bytes + E = 4 // 640 bytes + F = 5 // 1152 bytes + G = 6 // 384 bytes + H = 7 // 1280 bytes + X = 8 // 576 bytes + Y = 9 // 704 bytes + ) + + // The scenarios below refer to the chunks by position. + buildIndex := func(positions ...int) Index { + return chunkIndex(pickChunks(chunks, positions...)...) + } + buildContent := func(positions ...int) []byte { + return chunkContent(pickChunks(chunks, positions...)...) + } + buildStore := func(positions ...int) *TestStore { + return chunkStore(pickChunks(chunks, positions...)...) + } + + type scenario struct { + name string + inPlaceIndices []int // Chunks written to target file before assembly (the "old" content) + targetIndices []int // Desired output layout + fileSeedIndices []int // External file seed content (nil = no file seed) + storeIndices []int // Chunks available in the store + wantInPlace uint64 + wantFromSeeds uint64 + wantFromStore uint64 + } + + scenarios := []scenario{ + // Scenario 1: exercises every source type in one assembly. + // + // In-place seed (old file): [A][B][C][D][E] = 3840 bytes + // Target: [B][A][C][F][G][G][D][H] = 6400 bytes + // File seed: [F][X][X] + // Store: G, H + // + // After truncation to 6400 bytes the file is: + // [A:1024][B:768][C:512][D:896][E:640][zeros:2560] + // + // Source analysis per target position: + // Pos 0 (B): in-place copy — B exists at seed offset 1024, target offset 0. + // Part of A↔B cycle (asymmetric sizes: 1024 vs 768). + // Pos 1 (A): in-place copy — A exists at seed offset 0, target offset 768. + // Part of A↔B cycle, broken by stashing A. + // Pos 2 (C): skip in-place — C is at offset 1792 in both seed and target. + // Pos 3 (F): file seed — F is not in the in-place seed, found in file seed. + // D's in-place read [2304:3200] overlaps F's write [2304:3456], + // so D's read must complete first (enforced by inPlaceReads). + // Pos 4 (G): self-seed — G appears at both pos 4 and 5. Self-seed copies + // from pos 5 (requires source position > target position). + // Pos 5 (G): store — self-seed can't source from itself (p <= startPos). + // Pos 6 (D): in-place copy — D at seed offset 2304, target offset 4224. + // Independent move, no cycle. + // Pos 7 (H): store — H is not in any seed. + { + name: "all source types combined", + inPlaceIndices: []int{A, B, C, D, E}, + targetIndices: []int{B, A, C, F, G, G, D, H}, + fileSeedIndices: []int{F, X, X}, + storeIndices: []int{G, H}, + wantInPlace: 4, // B (cycle), A (cycle), C (skip), D (independent move) + wantFromSeeds: 2, // F (file seed), G at pos 4 (self-seed) + wantFromStore: 2, // G at pos 5, H + }, + + // Scenario 2: in-place seed is larger than the target. + // + // In-place seed: [A][B][C][D] = 3200 bytes + // Target: [B][A] = 1792 bytes + // + // Since the seed (3200) is larger than the target (1792), truncation + // is deferred until after assembly so that in-place reads can access + // the full seed data. A↔B form a swap cycle. After assembly, the + // file is truncated to 1792 bytes. + { + name: "in-place seed larger than target", + inPlaceIndices: []int{A, B, C, D}, + targetIndices: []int{B, A}, + storeIndices: nil, + wantInPlace: 2, // A↔B swap cycle + wantFromSeeds: 0, + wantFromStore: 0, + }, + + // Scenario 3: in-place seed is smaller than the target. + // + // In-place seed: [A][B] = 1792 bytes + // Target: [A][B][C][D] = 3200 bytes + // + // The file is extended (truncated up) to 3200 bytes. A and B are + // already at the correct offsets and detected by the initial scan. + // C and D are beyond the seed data and must come from the store. + { + name: "in-place seed smaller than target", + inPlaceIndices: []int{A, B}, + targetIndices: []int{A, B, C, D}, + storeIndices: []int{C, D}, + wantInPlace: 2, // A, B detected in-place by initial scan + wantFromSeeds: 0, + wantFromStore: 2, // C, D fetched from store + }, + + // Scenario 4: data inserted at the front. + // + // In-place seed: [A][B][C] = 2304 bytes + // Target: [H][A][B][C] = 3584 bytes + // + // There is no cycle, but A is written over the sources of B and C, + // so those have to be read first. + { + name: "insert at the front", + inPlaceIndices: []int{A, B, C}, + targetIndices: []int{H, A, B, C}, + storeIndices: []int{H}, + wantInPlace: 3, // A, B, C moved + wantFromSeeds: 0, + wantFromStore: 1, // H + }, + + // Scenario 5: one chunk overwrites the sources of two moves. + // + // In-place seed: [C][G][A] = 1920 bytes + // Target: [H][C][G] = 2176 bytes + // + // H from the store is written to [0:1280] which holds the sources + // of both C and G. It has to wait for both to be read. + { + name: "store chunk over two move sources", + inPlaceIndices: []int{C, G, A}, + targetIndices: []int{H, C, G}, + storeIndices: []int{H}, + wantInPlace: 2, // C, G moved + wantFromSeeds: 0, + wantFromStore: 1, // H + }, + + // Scenario 6: nested cycles. + // + // In-place seed: [H][G][X] = 2240 bytes + // Target: [G][X][C][H] = 2752 bytes + // + // H is written over the sources of G and X, and G and X are written + // over the source of H. That's two cycles sharing H, which can't + // be broken by stashing a single chunk per cycle blindly. + { + name: "nested cycles", + inPlaceIndices: []int{H, G, X}, + targetIndices: []int{G, X, C, H}, + storeIndices: []int{C}, + wantInPlace: 3, // G, X, H moved + wantFromSeeds: 0, + wantFromStore: 1, // C + }, + } + + for _, sc := range scenarios { + t.Run(sc.name, func(t *testing.T) { + dir := t.TempDir() + targetPath := filepath.Join(dir, "target") + + // Write the "old" file content — this is what the in-place seed + // describes. AssembleFile will detect it as non-empty, run the + // initial scan, and use the in-place seed to rearrange chunks. + inPlaceContent := buildContent(sc.inPlaceIndices...) + require.NoError(t, os.WriteFile(targetPath, inPlaceContent, 0644)) + + // Create the in-place seed. This wraps a FileSeed where source + // and destination are the same file. + inPlaceIdx := buildIndex(sc.inPlaceIndices...) + inPlaceSeed, err := NewFileSeed(targetPath, targetPath, inPlaceIdx, IndexRegenerationSettings{}) + require.NoError(t, err) + seeds := []Seed{inPlaceSeed} + + // If the scenario includes a file seed, write it to a separate + // file and create a FileSeed that maps its chunks by ID. + if sc.fileSeedIndices != nil { + seedPath := filepath.Join(dir, "fileseed") + seedContent := buildContent(sc.fileSeedIndices...) + require.NoError(t, os.WriteFile(seedPath, seedContent, 0644)) + seedIdx := buildIndex(sc.fileSeedIndices...) + fs, err := NewFileSeed(targetPath, seedPath, seedIdx, IndexRegenerationSettings{}) + require.NoError(t, err) + seeds = append(seeds, fs) + } + + // Build the target index (desired output layout) and compute + // the expected content for verification. + targetIdx := buildIndex(sc.targetIndices...) + expected := buildContent(sc.targetIndices...) + expectedSum := md5.Sum(expected) + + // Build the store with only the chunks that should be fetched + // from it. Any chunk incorrectly routed here will succeed; + // any chunk missing from here will fail with ChunkMissing. + store := buildStore(sc.storeIndices...) + + // Run the full assembly pipeline with 4 concurrent workers. + stats, err := AssembleFile( + context.Background(), targetPath, targetIdx, store, seeds, + testAssembleOptions, + ) + require.NoError(t, err) + + // Verify the output file matches the expected content. + output, err := os.ReadFile(targetPath) + require.NoError(t, err) + assert.Equal(t, int64(len(expected)), int64(len(output)), "output file size mismatch") + outSum := md5.Sum(output) + assert.Equal(t, expectedSum, outSum, "output checksum mismatch") + + // Verify that chunks were sourced from the expected places. + // This catches planner bugs where the output is correct but + // chunks were fetched from the wrong source (e.g. store + // instead of in-place copy). + assert.Equal(t, len(sc.targetIndices), stats.ChunksTotal, "ChunksTotal") + assert.Equal(t, sc.wantInPlace, stats.ChunksInPlace, "ChunksInPlace") + assert.Equal(t, sc.wantFromSeeds, stats.ChunksFromSeeds, "ChunksFromSeeds") + assert.Equal(t, sc.wantFromStore, stats.ChunksFromStore, "ChunksFromStore") + }) + } +} + +// Whether a seed is used in place depends on its data file being the target +// of the assembly, not on how the seed was created. +func TestAssembleSeedOnTarget(t *testing.T) { + chunks := randomChunks(1024, 768) + a, b := chunks[0], chunks[1] + oldIdx, oldContent := chunkIndex(a, b), chunkContent(a, b) + newIdx, expected := chunkIndex(b, a), chunkContent(b, a) + + // The store is empty, all data has to come from the seed + store := chunkStore() + + t.Run("file seed on the target", func(t *testing.T) { + // Reading from the target while writing to it would overwrite + // chunks before they're copied. The seed has to be used in place. + target := filepath.Join(t.TempDir(), "target") + require.NoError(t, os.WriteFile(target, oldContent, 0644)) + seed, err := NewFileSeed(target, target, oldIdx, IndexRegenerationSettings{}) + require.NoError(t, err) + + stats, err := AssembleFile(context.Background(), target, newIdx, store, []Seed{seed}, testAssembleOptions) + require.NoError(t, err) + assert.Equal(t, uint64(2), stats.ChunksInPlace) + + got, err := os.ReadFile(target) + require.NoError(t, err) + require.Equal(t, expected, got) + }) + + t.Run("file seed of another file", func(t *testing.T) { + // Like extracting into a temporary file that replaces the seed's + // file later. The seed is read from, not rearranged. + dir := t.TempDir() + old := filepath.Join(dir, "old") + require.NoError(t, os.WriteFile(old, oldContent, 0644)) + target := filepath.Join(dir, "target") + seed, err := NewFileSeed(target, old, oldIdx, IndexRegenerationSettings{}) + require.NoError(t, err) + + stats, err := AssembleFile(context.Background(), target, newIdx, store, []Seed{seed}, testAssembleOptions) + require.NoError(t, err) + assert.Equal(t, uint64(2), stats.ChunksFromSeeds) + + got, err := os.ReadFile(target) + require.NoError(t, err) + require.Equal(t, expected, got) + + // The seed's own file is left alone + got, err = os.ReadFile(old) + require.NoError(t, err) + require.Equal(t, oldContent, got) + }) +} + +// TestAssembleDuplicateOfChunkInPlace extracts over a file that holds a chunk +// in place which the output repeats later. Without a seed, the later copy +// still comes from the file rather than the store. +func TestAssembleDuplicateOfChunkInPlace(t *testing.T) { + chunks := randomChunks(1024, 768) + a, b := chunks[0], chunks[1] + target := writeChunkFile(t, a, filledChunk(768, 0xFF), filledChunk(1024, 0xFF)) + idx := chunkIndex(a, b, a) + + // The store doesn't have the chunk that's in place + stats, err := AssembleFile(context.Background(), target, idx, chunkStore(b), nil, testAssembleOptions) require.NoError(t, err) - defer is.Close() - indexName := filepath.Base(indexLocation) - idx, err = is.GetIndex(indexName) + assert.Equal(t, uint64(1), stats.ChunksInPlace) + assert.Equal(t, uint64(1), stats.ChunksFromSeeds) + assert.Equal(t, uint64(1), stats.ChunksFromStore) + + got, err := os.ReadFile(target) require.NoError(t, err) - return idx + require.Equal(t, chunkContent(a, b, a), got) +} + +// TestAssembleInPlaceRandomized rearranges random in-place layouts into random +// targets. Every chunk available in the old file has to be taken from there, +// and the output has to match the target regardless of how the moves overlap. +func TestAssembleInPlaceRandomized(t *testing.T) { + rng := mrand.New(mrand.NewPCG(1, 2)) + + // A pool of chunks with different sizes and random content + sizes := make([]int, 8) + for i := range sizes { + sizes[i] = 64 * (rng.IntN(8) + 1) + } + pool := randomChunks(sizes...) + store := chunkStore(pool...) + + randomLayout := func() []int { + layout := make([]int, rng.IntN(10)+1) + for i := range layout { + layout[i] = rng.IntN(len(pool)) + } + return layout + } + + // Without a memory budget for stashes, cycles are broken with the store + // instead. The chunks taken from there aren't in place. + for _, budget := range []int64{math.MaxInt64, 0, 512} { + t.Run(fmt.Sprintf("budget %d", budget), func(t *testing.T) { + setStashLimit(t, budget) + dir := t.TempDir() + for i := range 300 { + oldLayout, newLayout := randomLayout(), randomLayout() + oldChunks := pickChunks(pool, oldLayout...) + newChunks := pickChunks(pool, newLayout...) + oldIdx, oldContent := chunkIndex(oldChunks...), chunkContent(oldChunks...) + newIdx, expected := chunkIndex(newChunks...), chunkContent(newChunks...) + + target := filepath.Join(dir, fmt.Sprintf("target%d", i)) + require.NoError(t, os.WriteFile(target, oldContent, 0644)) + seed, err := NewFileSeed(target, target, oldIdx, IndexRegenerationSettings{}) + require.NoError(t, err) + + stats, err := AssembleFile(context.Background(), target, newIdx, store, []Seed{seed}, + testAssembleOptions) + require.NoError(t, err, "old layout %v, new layout %v", oldLayout, newLayout) + + output, err := os.ReadFile(target) + require.NoError(t, err) + require.Equal(t, expected, output, "old layout %v, new layout %v", oldLayout, newLayout) + + var wantInPlace uint64 + for _, c := range newLayout { + if slices.Contains(oldLayout, c) { + wantInPlace++ + } + } + if budget == math.MaxInt64 { + require.Equal(t, wantInPlace, stats.ChunksInPlace, "old layout %v, new layout %v", oldLayout, newLayout) + } else { + require.LessOrEqual(t, stats.ChunksInPlace, wantInPlace, "old layout %v, new layout %v", oldLayout, newLayout) + } + } + }) + } +} + +// setStashLimit sets the memory in-place stashes may hold for the test. +func setStashLimit(t *testing.T, budget int64) { + t.Helper() + orig := stashMemoryLimit + stashMemoryLimit = func() int64 { return budget } + t.Cleanup(func() { stashMemoryLimit = orig }) +} + +// A swap needs one of the chunks held in memory. Without any memory for it, +// that chunk comes from the store. +func TestAssembleSwapWithoutStashLimit(t *testing.T) { + setStashLimit(t, 0) + + chunks := randomChunks(1024, 768) + a, b := chunks[0], chunks[1] + + target := writeChunkFile(t, a, b) + seed, err := NewFileSeed(target, target, chunkIndex(a, b), IndexRegenerationSettings{}) + require.NoError(t, err) + + stats, err := AssembleFile(context.Background(), target, chunkIndex(b, a), chunkStore(a, b), + []Seed{seed}, testAssembleOptions) + require.NoError(t, err) + assert.Equal(t, uint64(1), stats.ChunksInPlace) + assert.Equal(t, uint64(1), stats.ChunksFromStore) + + got, err := os.ReadFile(target) + require.NoError(t, err) + require.Equal(t, chunkContent(b, a), got) +} + +// A target that's larger than the output is shrunk to the output's size, +// keeping the chunks that are already in place. +func TestExtractOverLargerFile(t *testing.T) { + chunks := randomChunks(1024, 768) + a, b := chunks[0], chunks[1] + target := writeChunkFile(t, a, b) + + // The store is empty, the chunk has to be found in place + stats, err := AssembleFile(context.Background(), target, chunkIndex(a), chunkStore(), nil, + testAssembleOptions) + require.NoError(t, err) + assert.Equal(t, uint64(1), stats.ChunksInPlace) + + got, err := os.ReadFile(target) + require.NoError(t, err) + require.Equal(t, a.data, got) } // An index of a zero-length file has no chunks. Extracting it should produce @@ -437,39 +912,19 @@ func TestExtractEmptyIndex(t *testing.T) { require.Empty(t, b) } -func TestExtractWithNonStaticSeeds(t *testing.T) { - n := 10 - outDir := t.TempDir() - out := filepath.Join(outDir, "out") - - // Test a seed that is initially valid, but becomes corrupted halfway through - // the extraction operation - MockValidate = true - - store, err := NewLocalStore("testdata/blob2.store", StoreOptions{}) - require.NoError(t, err) - defer store.Close() - - index := readCaibxFile(t, "testdata/blob2.caibx") - - var seeds []Seed - srcIndex := readCaibxFile(t, "testdata/blob2_corrupted.caibx") - indexRegenSettings := IndexRegenerationSettings{chunkers.DefaultChunkingSettings()} - seed, err := NewIndexSeed(out, "testdata/blob2_corrupted", srcIndex, indexRegenSettings) - require.NoError(t, err) - seeds = append(seeds, seed) - - // Test that the MockValidate works as expected - seq := NewSeedSequencer(index, seeds...) - plan := seq.Plan() - err = plan.Validate(context.Background(), n, NullProgressBar{}) - require.NoError(t, err) - - options := AssembleOptions{n, InvalidSeedActionRegenerate} - _, err = AssembleFile(context.Background(), out, index, store, seeds, options) - require.NoError(t, err) - - //Test the output - err = VerifyIndex(context.Background(), out, index, n, NullProgressBar{}) - require.NoError(t, err) +// Cancelling the context fails the assembly, even when the dispatcher notices +// before a worker does and the workers exit without error. +func TestAssembleCancelled(t *testing.T) { + chunks := randomChunks(64, 64, 64, 64) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + // Which of the dispatcher and the worker notices first is random, try + // often enough to see both. + for range 50 { + target := filepath.Join(t.TempDir(), "target") + _, err := AssembleFile(ctx, target, chunkIndex(chunks...), chunkStore(chunks...), nil, + AssembleOptions{N: 1, InvalidSeedAction: InvalidSeedActionBailOut}) + require.ErrorIs(t, err, context.Canceled) + } } diff --git a/cmd/desync/extract.go b/cmd/desync/extract.go index 2ba8555..206d403 100644 --- a/cmd/desync/extract.go +++ b/cmd/desync/extract.go @@ -52,8 +52,9 @@ If a seed is invalid, the extract operation is aborted by default. With --skip-invalid-seeds, invalid seeds are discarded and the extraction continues without them. Alternatively, --regenerate-invalid-seeds regenerates invalid seed indexes in memory from the available data; neither data nor indexes are -changed on disk. Also, if a seed changes while processing, its invalid chunks -will be taken from the self seed, or the store, instead of aborting.`, +changed on disk. Seeds are validated once before extraction begins. If a seed +file is modified while the extraction is running, the target is verified +afterwards and chunks that don't match are taken from the store.`, Example: ` desync extract -s http://192.168.1.1/ -c /path/to/local file.caibx largefile.bin desync extract -s /mnt/store -s /tmp/other/store file.tar.caibx file.tar desync extract -s /mnt/store --seed /mnt/v1.caibx v2.caibx v2.vmdk @@ -219,7 +220,7 @@ func readSeeds(dstFile string, seedsInfo []string, opts cmdStoreOptions, indexRe return nil, err } - seed, err := desync.NewIndexSeed(dstFile, srcFile, srcIndex, indexRegenSettings) + seed, err := desync.NewFileSeed(dstFile, srcFile, srcIndex, indexRegenSettings) if err != nil { return nil, err } @@ -263,7 +264,7 @@ func readSeedDirs(dstFile, dstIdxFile string, dirs []string, opts cmdStoreOption if err != nil { return err } - seed, err := desync.NewIndexSeed(dstFile, srcFile, srcIndex, indexRegenSettings) + seed, err := desync.NewFileSeed(dstFile, srcFile, srcIndex, indexRegenSettings) if err != nil { return err } diff --git a/cmd/desync/extract_test.go b/cmd/desync/extract_test.go index 62bc3a0..b80b2df 100644 --- a/cmd/desync/extract_test.go +++ b/cmd/desync/extract_test.go @@ -74,9 +74,10 @@ func TestExtractCommand(t *testing.T) { // Explicitly set blob1 seed because seed-dir skips a seed if it's the same index file we gave in input. {"extract with seed directory without skipping invalid seeds", []string{"-s", "testdata/blob1.store", "--seed-dir", "testdata", "--seed", "testdata/blob1.caibx", "testdata/blob1.caibx"}, out1}, - // Same as above, no need for `--skip-invalid-seeds` + // The plan generator processes seeds in order, so the corrupted seed + // may get placements that fail validation. Use --skip-invalid-seeds. {"extract with multiple corrupted seeds", - []string{"--store", "testdata/empty.store", "--seed", "testdata/blob2_corrupted.caibx", "--seed", "testdata/blob1.caibx", "testdata/blob1.caibx"}, out1}, + []string{"--store", "testdata/empty.store", "--seed", "testdata/blob2_corrupted.caibx", "--seed", "testdata/blob1.caibx", "--skip-invalid-seeds", "testdata/blob1.caibx"}, out1}, {"extract with single seed that has all the expected chunks", []string{"--store", "testdata/empty.store", "--seed", "testdata/blob1.caibx", "testdata/blob1.caibx"}, out1}, // blob2_corrupted is a corrupted blob that doesn't match its seed index. We regenerate the seed index to match @@ -189,6 +190,53 @@ func TestExtractAdaptiveNextToFixedStore(t *testing.T) { assert.LessOrEqual(t, peak.Load(), int64(2)) } +// The output file can be a seed for its own extraction. With --in-place, its +// data is rearranged within the file. Otherwise it's an ordinary seed for the +// temporary file that replaces it at the end. The store is empty, so all data +// has to come from the seed either way. +func TestExtractWithOutputAsSeed(t *testing.T) { + expected, err := os.ReadFile("testdata/blob1") + require.NoError(t, err) + index, err := os.ReadFile("testdata/blob1.caibx") + require.NoError(t, err) + + for _, test := range []struct { + name string + args func(out string) []string + }{ + {"seed without in-place", + func(out string) []string { return []string{"--seed", out + ".caibx"} }}, + {"seed with in-place", + func(out string) []string { return []string{"--in-place", "--seed", out + ".caibx"} }}, + {"seed directory without in-place", + func(out string) []string { return []string{"--seed-dir", filepath.Dir(out)} }}, + {"seed directory with in-place", + func(out string) []string { return []string{"--in-place", "--seed-dir", filepath.Dir(out)} }}, + } { + t.Run(test.name, func(t *testing.T) { + // The output and its index next to it. Seeds are given by index + // only, "index:data" can't express Windows paths with a drive. + dir := t.TempDir() + out := filepath.Join(dir, "out") + require.NoError(t, os.WriteFile(out, expected, 0644)) + require.NoError(t, os.WriteFile(out+".caibx", index, 0644)) + + cmd := newExtractCommand(context.Background()) + args := append([]string{"--store", "testdata/empty.store"}, test.args(out)...) + cmd.SetArgs(append(args, "testdata/blob1.caibx", out)) + stderr = io.Discard + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + _, err := cmd.ExecuteC() + require.NoError(t, err) + + got, err := os.ReadFile(out) + require.NoError(t, err) + require.Equal(t, expected, got) + }) + } +} + func TestExtractWithFailover(t *testing.T) { outDir := t.TempDir() out := filepath.Join(outDir, "out") diff --git a/docs/cli/desync_extract.md b/docs/cli/desync_extract.md index d965f77..7c87961 100644 --- a/docs/cli/desync_extract.md +++ b/docs/cli/desync_extract.md @@ -23,8 +23,9 @@ If a seed is invalid, the extract operation is aborted by default. With --skip-invalid-seeds, invalid seeds are discarded and the extraction continues without them. Alternatively, --regenerate-invalid-seeds regenerates invalid seed indexes in memory from the available data; neither data nor indexes are -changed on disk. Also, if a seed changes while processing, its invalid chunks -will be taken from the self seed, or the store, instead of aborting. +changed on disk. Seeds are validated once before extraction begins. If a seed +file is modified while the extraction is running, the target is verified +afterwards and chunks that don't match are taken from the store. ``` desync extract [flags] diff --git a/docs/concepts.md b/docs/concepts.md index 20c60d0..116a13a 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -45,12 +45,18 @@ Copy-on-write filesystems such as Btrfs and XFS support cloning of blocks betwee - **Null Seed** — a built-in seed for chunks of max size containing only 0 bytes. This can significantly reduce disk usage of files with large 0-byte ranges, such as VM images, effectively turning an eager-zeroed VM disk into a sparse disk. - **Self Seed** — as chunks are written to the destination file, the file itself becomes a seed. If a chunk or series of chunks appears again later in the file, it is cloned from the position written previously, saving storage for files with repetitive sections. - **File Seeds** — seed files and their indexes can be provided when extracting. For example, `image-v1.vmdk` and `image-v1.vmdk.caibx` can be used as seed for extracting `image-v2.vmdk`. The additional disk space required will be only the delta between the two versions. +- **In-place Seed** — with `--in-place` (`-k`), the file or block device being written can be a seed for its own update, for example `--seed image-v1.caibx:/dev/sdc` when extracting `image-v2.caibx` onto `/dev/sdc`. Chunks already at their position aren't written at all. Chunks that are elsewhere in the target are moved to their position, cloned where the filesystem supports it, in an order that reads each one before anything overwrites it. When chunks swap places, some of them are held in memory until they're written. `GOMEMLIMIT` bounds that memory, and chunks that don't fit are fetched from the store. Without `--in-place`, a seed pointing at the output is an ordinary file seed for the temporary file that replaces the output at the end. + +Chunks already in place are used first, then chunks elsewhere in the target. The rest come from the seeds: the null seed, the file seeds in the order they're given, then the self seed, first where sections can be cloned and then where they have to be copied. Whatever is left is fetched from the store. + +Seeds are validated before extraction begins. If a seed file changes while the extraction runs, detected by its size and modification time, the target is verified afterwards and chunks that don't match are fetched from the store. A seed on a block device other than the target has no modification time to go by, so the target is always verified when one is used. ```mermaid graph LR subgraph "External Seeds" S1["Seed 1
(file + index)"] S2["Seed 2
(file + index)"] + IP["In-place Seed
(target + index, -k)"] end subgraph "Built-in Seeds" @@ -66,12 +72,14 @@ graph LR S2 -- "clone/copy
matching chunks" --> Result NS -- "clone/copy
zero regions" --> Result SS -- "clone/copy
repeated sections" --> Result + IP -- "move
chunks within target" --> Result CS -. "fetch
remaining chunks" .-> Result style S1 fill:#4a90d9,stroke:#2a6cb0,color:#fff style S2 fill:#4a90d9,stroke:#2a6cb0,color:#fff style NS fill:#6ab04c,stroke:#4a8a2c,color:#fff style SS fill:#6ab04c,stroke:#4a8a2c,color:#fff + style IP fill:#4a90d9,stroke:#2a6cb0,color:#fff style CS fill:#e17055,stroke:#c0392b,color:#fff style Result fill:#f6b93b,stroke:#d4951a,color:#fff ``` diff --git a/docs/cookbook.md b/docs/cookbook.md index b333395..e02d86b 100644 --- a/docs/cookbook.md +++ b/docs/cookbook.md @@ -54,6 +54,14 @@ Extract an image directly onto a block device. The `-k` or `--in-place` option i desync extract -k -s /mnt/store image.caibx /dev/sdc ``` +Update an image on a block device in place, reusing the data already on it. The index of the current image is a seed whose data is the device itself. Chunks that moved are rearranged on the device, and only chunks it doesn't contain yet are read from the store. + +```text +desync extract -k -s /mnt/store --seed image-v1.caibx:/dev/sdc image-v2.caibx /dev/sdc +``` + +When chunks swap places, some of them are held in memory until they are written. Set `GOMEMLIMIT` to bound the memory used for that, for example `GOMEMLIMIT=256MiB`. Chunks that don't fit are read from the store instead. + Extract a file using a remote index stored in an HTTP index store. ```text diff --git a/errors.go b/errors.go index 98f025b..2b143da 100644 --- a/errors.go +++ b/errors.go @@ -56,3 +56,17 @@ func (e InvalidFormat) Error() string { type Interrupted struct{} func (e Interrupted) Error() string { return "interrupted" } + +// SeedInvalid is returned when a seed's data doesn't match its index. +type SeedInvalid struct { + Seeds []Seed + Err error +} + +func (e SeedInvalid) Error() string { + return fmt.Sprintf("invalid seed: %s", e.Err) +} + +func (e SeedInvalid) Unwrap() error { + return e.Err +} diff --git a/extractstats.go b/extractstats.go index 9deefcf..9f06c49 100644 --- a/extractstats.go +++ b/extractstats.go @@ -10,20 +10,23 @@ type ExtractStats struct { ChunksFromSeeds uint64 `json:"chunks-from-seeds"` ChunksFromStore uint64 `json:"chunks-from-store"` ChunksInPlace uint64 `json:"chunks-in-place"` - BytesCopied uint64 `json:"bytes-copied-from-seeds"` - BytesCloned uint64 `json:"bytes-cloned-from-seeds"` - Blocksize uint64 `json:"blocksize"` - BytesTotal int64 `json:"bytes-total"` - ChunksTotal int `json:"chunks-total"` - Seeds int `json:"seeds"` + // ChunksBackfilled counts the chunks written from the store again after + // assembly, because a seed they came from changed while it was read. + ChunksBackfilled uint64 `json:"chunks-backfilled"` + BytesCopied uint64 `json:"bytes-copied-from-seeds"` + BytesCloned uint64 `json:"bytes-cloned-from-seeds"` + Blocksize uint64 `json:"blocksize"` + BytesTotal int64 `json:"bytes-total"` + ChunksTotal int `json:"chunks-total"` + Seeds int `json:"seeds"` } func (s *ExtractStats) incChunksFromStore() { atomic.AddUint64(&s.ChunksFromStore, 1) } -func (s *ExtractStats) incChunksInPlace() { - atomic.AddUint64(&s.ChunksInPlace, 1) +func (s *ExtractStats) addChunksInPlace(n uint64) { + atomic.AddUint64(&s.ChunksInPlace, n) } func (s *ExtractStats) addChunksFromSeed(n uint64) { @@ -34,6 +37,10 @@ func (s *ExtractStats) addBytesCopied(n uint64) { atomic.AddUint64(&s.BytesCopied, n) } +func (s *ExtractStats) addChunksBackfilled(n uint64) { + atomic.AddUint64(&s.ChunksBackfilled, n) +} + func (s *ExtractStats) addBytesCloned(n uint64) { atomic.AddUint64(&s.BytesCloned, n) } diff --git a/fileseed.go b/fileseed.go deleted file mode 100644 index 49ed45a..0000000 --- a/fileseed.go +++ /dev/null @@ -1,278 +0,0 @@ -package desync - -import ( - "context" - "fmt" - "io" - "os" - "slices" - "sync" - - "github.com/folbricht/desync/pkg/chunkers" -) - -var _ Seed = &FileSeed{} - -// Controls how to regenerate invalid index. -type IndexRegenerationSettings struct { - chunkers.ChunkingSettings -} - -// FileSeed is used to copy or clone blocks from an existing index+blob during -// file extraction. -type FileSeed struct { - srcFile string - index Index - indexRegenSettings IndexRegenerationSettings - pos map[ChunkID][]int - canReflink bool - isInvalid bool - mu sync.RWMutex -} - -// NewIndexSeed initializes a new seed that uses an existing index and its blob -func NewIndexSeed(dstFile string, srcFile string, index Index, regenSettings IndexRegenerationSettings) (*FileSeed, error) { - s := FileSeed{ - srcFile: srcFile, - pos: make(map[ChunkID][]int), - index: index, - indexRegenSettings: regenSettings, - canReflink: CanClone(dstFile, srcFile), - isInvalid: false, - } - for i, c := range s.index.Chunks { - s.pos[c.ID] = append(s.pos[c.ID], i) - } - // Make sure chunking sizes match with provided index. - // TODO: would be nice if IndexRegenerationSettings could be restored from index entirely. - s.indexRegenSettings.ChunkingSettings.ChunkerParams.Min = s.index.Index.ChunkSizeMin - s.indexRegenSettings.ChunkingSettings.ChunkerParams.Avg = s.index.Index.ChunkSizeAvg - s.indexRegenSettings.ChunkingSettings.ChunkerParams.Max = s.index.Index.ChunkSizeMax - return &s, nil -} - -// LongestMatchWith returns the longest sequence of chunks anywhere in Source -// that match `chunks` starting at chunks[0], limiting the maximum number of chunks -// if reflinks are not supported. If there is no match, it returns a length of zero -// and a nil SeedSegment. -func (s *FileSeed) LongestMatchWith(chunks []IndexChunk) (int, SeedSegment) { - s.mu.RLock() - isInvalid := s.isInvalid - s.mu.RUnlock() - - // isInvalid can be concurrently read or written. Use a mutex to avoid a race - if len(chunks) == 0 || len(s.index.Chunks) == 0 || isInvalid { - return 0, nil - } - pos, ok := s.pos[chunks[0].ID] - if !ok { - return 0, nil - } - // From every position of chunks[0] in the source, find a slice of - // matching chunks. Then return the longest of those slices. - var ( - match []IndexChunk - max int - limit int - ) - if !s.canReflink { - // Limit the maximum number of chunks, in a single sequence, to avoid - // having jobs that are too unbalanced. - // However, if reflinks are supported, we don't limit it to make it faster and - // take less space. - limit = 100 - } - for _, p := range pos { - m := s.maxMatchFrom(chunks, p, limit) - if len(m) > max { - match = m - max = len(m) - } - if limit != 0 && limit == max { - break - } - } - return max, newFileSeedSegment(s.srcFile, match, s.canReflink) -} - -func (s *FileSeed) RegenerateIndex(ctx context.Context, n int, attempt int, seedNumber int) error { - chunkingPrefix := fmt.Sprintf("Attempt %d: Chunking Seed %d ", attempt, seedNumber) - - index, _, err := IndexFromFile(ctx, s.srcFile, n, s.indexRegenSettings.ChunkingSettings, NewProgressBar(chunkingPrefix)) - if err != nil { - return err - } - - s.index = index - s.SetInvalid(false) - s.pos = make(map[ChunkID][]int, len(s.index.Chunks)) - for i, c := range s.index.Chunks { - s.pos[c.ID] = append(s.pos[c.ID], i) - } - - return nil -} - -func (s *FileSeed) SetInvalid(value bool) { - s.mu.Lock() - defer s.mu.Unlock() - s.isInvalid = value -} - -func (s *FileSeed) IsInvalid() bool { - s.mu.Lock() - defer s.mu.Unlock() - return s.isInvalid -} - -// Returns a slice of chunks from the seed. Compares chunks from position 0 -// with seed chunks starting at p. A "limit" value of zero means that there is no limit. -func (s *FileSeed) maxMatchFrom(chunks []IndexChunk, p int, limit int) []IndexChunk { - if len(chunks) == 0 { - return nil - } - var ( - sp int - dp = p - ) - for { - if limit != 0 && sp == limit { - break - } - if dp >= len(s.index.Chunks) || sp >= len(chunks) { - break - } - if chunks[sp].ID != s.index.Chunks[dp].ID { - break - } - dp++ - sp++ - } - return s.index.Chunks[p:dp] -} - -type fileSeedSegment struct { - file string - chunks []IndexChunk - canReflink bool -} - -func newFileSeedSegment(file string, chunks []IndexChunk, canReflink bool) *fileSeedSegment { - return &fileSeedSegment{ - canReflink: canReflink, - file: file, - chunks: chunks, - } -} - -func (s *fileSeedSegment) FileName() string { - return s.file -} - -func (s *fileSeedSegment) Size() uint64 { - if len(s.chunks) == 0 { - return 0 - } - last := s.chunks[len(s.chunks)-1] - return last.Start + last.Size - s.chunks[0].Start -} - -func (s *fileSeedSegment) WriteInto(dst *os.File, offset, length, blocksize uint64, isBlank bool) (uint64, uint64, error) { - if length != s.Size() { - return 0, 0, fmt.Errorf("unable to copy %d bytes from %s to %s : wrong size", length, s.file, dst.Name()) - } - src, err := os.Open(s.file) - if err != nil { - return 0, 0, err - } - defer src.Close() - - // Do a straight copy if reflinks are not supported or blocks aren't aligned - if !s.canReflink || s.chunks[0].Start%blocksize != offset%blocksize { - return s.copy(dst, src, s.chunks[0].Start, length, offset) - } - return s.clone(dst, src, s.chunks[0].Start, length, offset, blocksize) -} - -// Validate compares all chunks in this slice of the seed index to the underlying data -// and fails if they don't match. -func (s *fileSeedSegment) Validate(file *os.File) error { - var buf []byte - for _, c := range s.chunks { - buf = slices.Grow(buf[:0], int(c.Size))[:c.Size] - if _, err := file.ReadAt(buf, int64(c.Start)); err != nil { - return err - } - sum := Digest.Sum(buf) - if sum != c.ID { - return fmt.Errorf("seed index for %s doesn't match its data", s.file) - } - } - return nil -} - -// Performs a plain copy of everything in the seed to the target, not cloning -// of blocks. -func (s *fileSeedSegment) copy(dst, src *os.File, srcOffset, length, dstOffset uint64) (uint64, uint64, error) { - if _, err := dst.Seek(int64(dstOffset), io.SeekStart); err != nil { - return 0, 0, err - } - if _, err := src.Seek(int64(srcOffset), io.SeekStart); err != nil { - return 0, 0, err - } - - // Copy using a fixed buffer. Using io.Copy() with a LimitReader will make it - // create a buffer matching N of the LimitReader which can be too large - copied, err := io.CopyBuffer(dst, io.LimitReader(src, int64(length)), make([]byte, 64*1024)) - return uint64(copied), 0, err -} - -// Reflink the overlapping blocks in the two ranges and copy the bit before and -// after the blocks. -func (s *fileSeedSegment) clone(dst, src *os.File, srcOffset, srcLength, dstOffset, blocksize uint64) (uint64, uint64, error) { - if srcOffset%blocksize != dstOffset%blocksize { - return 0, 0, fmt.Errorf("reflink ranges not aligned between %s and %s", src.Name(), dst.Name()) - } - - srcAlignStart := (srcOffset/blocksize + 1) * blocksize - srcAlignEnd := (srcOffset + srcLength) / blocksize * blocksize - - // If the range is too small to contain a full aligned block, there is - // nothing that can be cloned. Copy the whole range instead. This also - // guards against srcAlignEnd-srcAlignStart underflowing, and against - // calling CloneRange with a zero length, which FICLONERANGE interprets - // as "clone to the end of the source file". Filesystems with large - // blocks, like ZFS with the default 128k recordsize, hit this case - // frequently. - if srcAlignEnd <= srcAlignStart { - return s.copy(dst, src, srcOffset, srcLength, dstOffset) - } - - dstAlignStart := (dstOffset/blocksize + 1) * blocksize - alignLength := srcAlignEnd - srcAlignStart - dstAlignEnd := dstAlignStart + alignLength - - // fill the area before the first aligned block - var copied uint64 - c1, _, err := s.copy(dst, src, srcOffset, srcAlignStart-srcOffset, dstOffset) - if err != nil { - return c1, 0, err - } - copied += c1 - // fill the area after the last aligned block - c2, _, err := s.copy(dst, src, srcAlignEnd, srcOffset+srcLength-srcAlignEnd, dstAlignEnd) - if err != nil { - return copied + c2, 0, err - } - copied += c2 - // close the aligned blocks - if err := cloneRange(dst, src, srcAlignStart, alignLength, dstAlignStart); err != nil { - // Not every filesystem that passes the CanClone probe can clone every - // range. ZFS for example requires alignment to its record size and - // refuses to clone data that hasn't been committed to disk yet. Fall - // back to copying the blocks. - c3, _, err := s.copy(dst, src, srcAlignStart, alignLength, dstAlignStart) - return copied + c3, 0, err - } - return copied, alignLength, nil -} diff --git a/fileseed_test.go b/fileseed_test.go index 801cd5b..270e323 100644 --- a/fileseed_test.go +++ b/fileseed_test.go @@ -89,3 +89,32 @@ func TestFileSeedSegmentSmallerThanBlock(t *testing.T) { assert.Equal(t, data, got[:size]) assert.Equal(t, bytes.Repeat([]byte{0xff}, 3*blocksize-int(size)), got[size:]) } + +// Validating a seed segment reuses its read buffer rather than allocating +// one per chunk. +func TestFileSeedSegmentValidateAllocations(t *testing.T) { + if raceEnabled { + t.Skip("the race detector makes sync.Pool drop buffers at random") + } + var ( + chunks []IndexChunk + data []byte + ) + for i := range 100 { + b := make([]byte, 4096) + b[0] = byte(i) + chunks = append(chunks, IndexChunk{ID: Digest.Sum(b), Start: uint64(len(data)), Size: 4096}) + data = append(data, b...) + } + name := filepath.Join(t.TempDir(), "seed") + require.NoError(t, os.WriteFile(name, data, 0644)) + f, err := os.Open(name) + require.NoError(t, err) + defer f.Close() + + segment := newFileSeedSegment(name, chunks, false) + allocs := testing.AllocsPerRun(10, func() { + require.NoError(t, segment.Validate(f)) + }) + require.Less(t, allocs, float64(len(chunks)/10)) +} diff --git a/go.mod b/go.mod index 76f80b3..c6f8fde 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/folbricht/desync -go 1.25.0 +go 1.26.0 require ( cloud.google.com/go/storage v1.60.0 @@ -20,10 +20,10 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.11.1 - golang.org/x/crypto v0.55.0 - golang.org/x/sync v0.22.0 - golang.org/x/sys v0.47.0 - golang.org/x/term v0.45.0 + golang.org/x/crypto v0.57.0 + golang.org/x/sync v0.23.0 + golang.org/x/sys v0.48.0 + golang.org/x/term v0.46.0 google.golang.org/api v0.267.0 gopkg.in/cheggaaa/pb.v1 v1.0.28 oras.land/oras-go/v2 v2.6.2 @@ -81,9 +81,9 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect go.opentelemetry.io/otel/trace v1.45.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/net v0.58.0 // indirect + golang.org/x/net v0.60.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/text v0.41.0 // indirect + golang.org/x/text v0.42.0 // indirect golang.org/x/time v0.14.0 // indirect google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect diff --git a/go.sum b/go.sum index 7e7fa19..64261c8 100644 --- a/go.sum +++ b/go.sum @@ -190,28 +190,28 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= -golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= -golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= -golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= -golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= -golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= -golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= +golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= -golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/index.go b/index.go index 47723e9..ab24b9c 100644 --- a/index.go +++ b/index.go @@ -32,6 +32,16 @@ type IndexChunk struct { Size uint64 } +// chunkRangeLength returns the number of bytes covered by a consecutive run +// of index chunks. +func chunkRangeLength(chunks []IndexChunk) uint64 { + if len(chunks) == 0 { + return 0 + } + last := chunks[len(chunks)-1] + return last.Start + last.Size - chunks[0].Start +} + // IndexFromReader parses a caibx structure (from a reader) and returns a populated Caibx // object func IndexFromReader(r io.Reader) (c Index, err error) { diff --git a/nullseed.go b/nullseed.go index 6e2c579..a9d4dba 100644 --- a/nullseed.go +++ b/nullseed.go @@ -2,7 +2,6 @@ package desync import ( "context" - "fmt" "io" "os" "path/filepath" @@ -52,9 +51,9 @@ func (s *nullChunkSeed) close() error { return nil } -func (s *nullChunkSeed) LongestMatchWith(chunks []IndexChunk) (int, SeedSegment) { - if len(chunks) == 0 { - return 0, nil +func (s *nullChunkSeed) LongestMatchFrom(chunks []IndexChunk, startPos int) (int, int) { + if startPos >= len(chunks) { + return 0, 0 } var ( n int @@ -63,7 +62,7 @@ func (s *nullChunkSeed) LongestMatchWith(chunks []IndexChunk) (int, SeedSegment) if !s.canReflink { limit = 100 } - for _, c := range chunks { + for _, c := range chunks[startPos:] { if limit != 0 && limit == n { break } @@ -72,12 +71,25 @@ func (s *nullChunkSeed) LongestMatchWith(chunks []IndexChunk) (int, SeedSegment) } n++ } - if n == 0 { - return 0, nil + return 0, n +} + +// mayClone reports whether null chunks can be cloned, or don't need to be +// written at all as the target is still blank. +func (s *nullChunkSeed) mayClone(isBlank bool) bool { return isBlank || s.canReflink } + +// clones reports whether null chunks written over the target chunks dst are +// cloned, or not written at all as the target is still blank. +func (s *nullChunkSeed) clones(pos int, dst []IndexChunk, blocksize uint64, isBlank bool) bool { + if isBlank { + return true } - return n, &nullChunkSection{ - from: chunks[0].Start, - to: chunks[n-1].Start + chunks[n-1].Size, + offset := dst[0].Start + return s.canReflink && reflinkable(offset, chunkRangeLength(dst), offset, blocksize) +} + +func (s *nullChunkSeed) GetSegment(pos, n int) SeedSegment { + return &nullChunkSection{ blockfile: s.blockfile, blockLen: s.blockLen, canReflink: s.canReflink, @@ -88,17 +100,7 @@ func (s *nullChunkSeed) RegenerateIndex(ctx context.Context, n int, attempt int, panic("A nullseed can't be regenerated") } -func (s *nullChunkSeed) SetInvalid(value bool) { - panic("A nullseed is never expected to be invalid") -} - -func (s *nullChunkSeed) IsInvalid() bool { - // A nullseed is never expected to be invalid - return false -} - type nullChunkSection struct { - from, to uint64 blockfile *os.File blockLen uint64 // Length of the blockfile, a multiple of the blocksize canReflink bool @@ -113,13 +115,7 @@ func (s *nullChunkSection) FileName() string { return "" } -func (s *nullChunkSection) Size() uint64 { return s.to - s.from } - func (s *nullChunkSection) WriteInto(dst *os.File, offset, length, blocksize uint64, isBlank bool) (uint64, uint64, error) { - if length != s.Size() { - return 0, 0, fmt.Errorf("unable to copy %d bytes to %s : wrong size", length, dst.Name()) - } - // A blank target, a new or truncated file, reads as zeros already. Cloning // or copying zeros into it would only fill in what's a hole now, block by // block when cloning. @@ -127,7 +123,7 @@ func (s *nullChunkSection) WriteInto(dst *os.File, offset, length, blocksize uin return 0, 0, nil } if !s.canReflink { - return s.copy(dst, offset, s.Size()) + return s.copy(dst, offset, length) } return s.clone(dst, offset, length, blocksize) } diff --git a/nullseed_test.go b/nullseed_test.go index b5f6c0d..727903b 100644 --- a/nullseed_test.go +++ b/nullseed_test.go @@ -29,7 +29,7 @@ func TestNullChunkSectionCloneFallback(t *testing.T) { require.NoError(t, err) newSection := func() *nullChunkSection { - return &nullChunkSection{from: 0, to: length, blockfile: blockfile, blockLen: blocksize, canReflink: true} + return &nullChunkSection{blockfile: blockfile, blockLen: blocksize, canReflink: true} } t.Run("copies zeros when cloning fails", func(t *testing.T) { @@ -92,7 +92,7 @@ func TestNullChunkSectionCloneFallback(t *testing.T) { require.NoError(t, err) defer dst.Close() - section := &nullChunkSection{from: from, to: from + sectionLen, blockfile: blockfile, blockLen: blocksize, canReflink: true} + section := &nullChunkSection{blockfile: blockfile, blockLen: blocksize, canReflink: true} copied, cloned, err := section.WriteInto(dst, from, sectionLen, bigBlock, false) require.NoError(t, err) assert.Equal(t, uint64(0), cloned) @@ -167,7 +167,7 @@ func TestNullChunkSectionClonesLargePieces(t *testing.T) { require.NoError(t, err) defer dst.Close() - section := &nullChunkSection{from: from, to: from + length, blockfile: blockfile, blockLen: blockLen, canReflink: true} + section := &nullChunkSection{blockfile: blockfile, blockLen: blockLen, canReflink: true} copied, cloned, err := section.WriteInto(dst, from, length, blocksize, false) require.NoError(t, err) assert.Equal(t, uint64(9*blocksize), cloned) diff --git a/race_disabled_test.go b/race_disabled_test.go new file mode 100644 index 0000000..ba91712 --- /dev/null +++ b/race_disabled_test.go @@ -0,0 +1,7 @@ +//go:build !race + +package desync + +// raceEnabled reports whether the tests run with the race detector, which +// changes allocation behavior, e.g. sync.Pool drops items at random. +const raceEnabled = false diff --git a/race_enabled_test.go b/race_enabled_test.go new file mode 100644 index 0000000..ff3fac8 --- /dev/null +++ b/race_enabled_test.go @@ -0,0 +1,7 @@ +//go:build race + +package desync + +// raceEnabled reports whether the tests run with the race detector, which +// changes allocation behavior, e.g. sync.Pool drops items at random. +const raceEnabled = true diff --git a/seed.go b/seed.go index aa541a2..9ab533a 100644 --- a/seed.go +++ b/seed.go @@ -2,7 +2,11 @@ package desync import ( "context" + "fmt" + "io" "os" + "sort" + "sync" ) // DefaultBlockSize is used when the actual filesystem block size cannot be determined automatically @@ -16,10 +20,137 @@ var cloneRange = CloneRange // another index+blob that present on disk already and is used to copy or clone // existing chunks or blocks into the target from. type Seed interface { - LongestMatchWith(chunks []IndexChunk) (int, SeedSegment) + // LongestMatchFrom returns the longest sequence of chunks anywhere in the + // seed that match chunks starting at chunks[startPos]. It returns the + // chunk position of the match in the seed and the number of matching + // chunks, or (0, 0) if there is no match. + LongestMatchFrom(chunks []IndexChunk, startPos int) (pos int, n int) + // GetSegment returns a SeedSegment covering n chunks starting at chunk + // position pos in the seed. pos must be a position returned by + // LongestMatchFrom; n may be smaller than the reported match length. + GetSegment(pos, n int) SeedSegment RegenerateIndex(ctx context.Context, n int, attempt int, seedNumber int) error - SetInvalid(value bool) - IsInvalid() bool +} + +// seedIndex holds a seed's index along with a lookup table of chunk positions +// by ID. It implements the matching kernel shared by file seeds and the +// self-seed. +type seedIndex struct { + index Index + pos map[ChunkID][]int + canReflink bool +} + +func newSeedIndex(index Index, canReflink bool) seedIndex { + s := seedIndex{ + index: index, + pos: make(map[ChunkID][]int, len(index.Chunks)), + canReflink: canReflink, + } + for i, c := range index.Chunks { + s.pos[c.ID] = append(s.pos[c.ID], i) + } + return s +} + +// longestMatchFrom finds, among the seed positions holding the same chunk as +// chunks[startPos], the position with the longest run of consecutive matches. +// Only positions from minPos on are candidates; they're found by binary +// search, so a chunk repeated many times before minPos costs nothing. clamp, +// if not nil, returns the maximum run length allowed for a candidate seed +// position; returning 0 discards the candidate. maxCandidates, when not zero, +// bounds how many candidate positions are examined. Later candidates win ties. +// Returns the winning position and run length, or (0, 0) if there is no match. +func (s *seedIndex) longestMatchFrom(chunks []IndexChunk, startPos, minPos, maxCandidates int, clamp func(p int) int) (int, int) { + if startPos >= len(chunks) || len(s.index.Chunks) == 0 { + return 0, 0 + } + pos, ok := s.pos[chunks[startPos].ID] + if !ok { + return 0, 0 + } + pos = pos[sort.SearchInts(pos, minPos):] + var ( + bestPos int + maxLen int + limit int + examined int + ) + if !s.canReflink { + // Limit the maximum number of chunks in a single sequence to avoid + // having jobs that are too unbalanced. However, if reflinks are + // supported, we don't limit it to make it faster and take less space. + limit = 100 + } + for _, p := range pos { + // Applying the clamp as a match limit also bounds the work done per + // candidate. + lim := limit + if clamp != nil { + c := clamp(p) + if c == 0 { + continue + } + if lim == 0 || c < lim { + lim = c + } + } + n := maxMatchFrom(chunks[startPos:], s.index.Chunks, p, lim) + if n > 0 && n >= maxLen { + bestPos = p + maxLen = n + } + if limit != 0 && limit == maxLen { + break + } + examined++ + if maxCandidates != 0 && examined == maxCandidates { + break + } + } + return bestPos, maxLen +} + +func (s *seedIndex) mayClone(isBlank bool) bool { return s.canReflink } + +// clones reports whether the chunks from position pos in the seed would be +// cloned rather than copied over the target chunks dst. +func (s *seedIndex) clones(pos int, dst []IndexChunk, blocksize uint64, isBlank bool) bool { + return s.canReflink && s.reflinkable(pos, dst, blocksize) +} + +// reflinkable reports whether the chunks from position pos in the seed would +// be cloned rather than copied over the target chunks dst, if the seed +// supports reflinks. +func (s *seedIndex) reflinkable(pos int, dst []IndexChunk, blocksize uint64) bool { + return reflinkable(s.index.Chunks[pos].Start, chunkRangeLength(dst), dst[0].Start, blocksize) +} + +// maxMatchFrom compares chunks starting at position 0 with seedChunks starting +// at position p. Returns the number of consecutive matching chunks. A limit of +// zero means no limit. +func maxMatchFrom(chunks, seedChunks []IndexChunk, p, limit int) int { + if len(chunks) == 0 { + return 0 + } + var ( + sp int + dp = p + ) + for { + if limit != 0 && sp == limit { + break + } + if dp >= len(seedChunks) || sp >= len(chunks) { + break + } + if chunks[sp].ID != seedChunks[dp].ID { + break + } + dp++ + sp++ + } + return dp - p } // SeedSegment represents a matching range between a Seed and a file being @@ -27,34 +158,160 @@ type Seed interface { // a target file during an extract operation. type SeedSegment interface { FileName() string - Size() uint64 Validate(file *os.File) error WriteInto(dst *os.File, offset, end, blocksize uint64, isBlank bool) (copied uint64, cloned uint64, err error) } -// IndexSegment represents a contiguous section of an index which is used when -// assembling a file from seeds. first/last are positions in the index. -type IndexSegment struct { - index Index - first, last int +// chunkBufPool holds read buffers for chunkInPlace callers that don't bring +// their own, so validating seeds doesn't allocate a buffer per chunk. +var chunkBufPool sync.Pool + +// getChunkBuf returns a buffer of at least size bytes from the pool. It has +// to be returned to the pool once it's no longer used. +func getChunkBuf(size uint64) *[]byte { + p, _ := chunkBufPool.Get().(*[]byte) + if p == nil || uint64(len(*p)) < size { + b := make([]byte, size) + p = &b + } + return p +} + +// chunkInPlace reports whether f already contains the chunk's data at the +// chunk's position. buf is used for reading when it is large enough, which +// allows callers checking many chunks to reuse one allocation. If it's nil, +// a buffer from a pool is used. +func chunkInPlace(f *os.File, c IndexChunk, buf []byte) bool { + if uint64(len(buf)) < c.Size { + p := getChunkBuf(c.Size) + defer chunkBufPool.Put(p) + buf = *p + } + b := buf[:c.Size] + if _, err := f.ReadAt(b, int64(c.Start)); err != nil { + return false + } + return Digest.Sum(b) == c.ID +} + +// maxChunkSize is the size of the largest chunk, the buffer a reader of all of +// them needs. +func maxChunkSize(chunks []IndexChunk) uint64 { + var size uint64 + for _, c := range chunks { + size = max(size, c.Size) + } + return size } -func (s IndexSegment) lengthChunks() int { - return s.last - s.first + 1 +// chunksInPlace reports whether f contains the data of every chunk at the +// chunk's position, reading them all into one buffer. +func chunksInPlace(f *os.File, chunks []IndexChunk) bool { + p := getChunkBuf(maxChunkSize(chunks)) + defer chunkBufPool.Put(p) + for _, c := range chunks { + if !chunkInPlace(f, c, *p) { + return false + } + } + return true } -func (s IndexSegment) lengthBytes() uint64 { - return s.end() - s.start() +// copyRange copies length bytes from src at srcOffset to dst at dstOffset. +// src and dst must be distinct file handles since both handles' offsets are +// moved, but they may refer to the same file as long as the ranges don't +// overlap. +func copyRange(dst, src *os.File, srcOffset, length, dstOffset uint64) (uint64, uint64, error) { + if _, err := dst.Seek(int64(dstOffset), io.SeekStart); err != nil { + return 0, 0, err + } + if _, err := src.Seek(int64(srcOffset), io.SeekStart); err != nil { + return 0, 0, err + } + // Copy using a fixed buffer. Using io.Copy() with a LimitReader will make it + // create a buffer matching N of the LimitReader which can be too large + copied, err := io.CopyBuffer(dst, io.LimitReader(src, int64(length)), make([]byte, 64*1024)) + return uint64(copied), 0, err } -func (s IndexSegment) start() uint64 { - return s.index.Chunks[s.first].Start +// matchingSeed is the part of a seed the plan uses to find the chunks it can +// provide. The self-seed implements it without being a full Seed, its +// segments need the target offset they're written to. +type matchingSeed interface { + LongestMatchFrom(chunks []IndexChunk, startPos int) (pos int, n int) } -func (s IndexSegment) end() uint64 { - return s.index.Chunks[s.last].Start + s.index.Chunks[s.last].Size +// cloningSeed is implemented by seeds that can tell whether a match would be +// cloned into the target, or not need to be written at all, rather than be +// copied. Seeds that don't implement it are assumed to copy. +type cloningSeed interface { + // mayClone reports whether the seed can clone any chunks at all. Looking + // for matches to clone is skipped for seeds that can't. + mayClone(isBlank bool) bool + // clones reports whether the seed's chunks from chunk position pos would + // be cloned when written over the target chunks dst. + clones(pos int, dst []IndexChunk, blocksize uint64, isBlank bool) bool } -func (s IndexSegment) chunks() []IndexChunk { - return s.index.Chunks[s.first : s.last+1] +// reflinkable reports whether cloneOrCopyRange clones at least one block of +// a range rather than only copying it. The ranges need the same alignment +// relative to blocksize and have to span a full aligned block. +func reflinkable(srcOffset, length, dstOffset, blocksize uint64) bool { + if srcOffset%blocksize != dstOffset%blocksize { + return false + } + srcAlignStart := (srcOffset/blocksize + 1) * blocksize + srcAlignEnd := (srcOffset + length) / blocksize * blocksize + return srcAlignEnd > srcAlignStart +} + +// cloneOrCopyRange reflinks the aligned blocks the two ranges have in common +// and copies the unaligned head and tail. Both ranges must have the same +// alignment relative to blocksize. Falls back to copying whenever cloning +// isn't possible. +func cloneOrCopyRange(dst, src *os.File, srcOffset, length, dstOffset, blocksize uint64) (uint64, uint64, error) { + if srcOffset%blocksize != dstOffset%blocksize { + return 0, 0, fmt.Errorf("reflink ranges not aligned between %s and %s", src.Name(), dst.Name()) + } + + // If the range is too small to contain a full aligned block, there is + // nothing that can be cloned. Copy the whole range instead. This also + // guards against srcAlignEnd-srcAlignStart underflowing, and against + // calling CloneRange with a zero length, which FICLONERANGE interprets + // as "clone to the end of the source file". Filesystems with large + // blocks, like ZFS with the default 128k recordsize, hit this case + // frequently. + if !reflinkable(srcOffset, length, dstOffset, blocksize) { + return copyRange(dst, src, srcOffset, length, dstOffset) + } + srcAlignStart := (srcOffset/blocksize + 1) * blocksize + srcAlignEnd := (srcOffset + length) / blocksize * blocksize + + dstAlignStart := (dstOffset/blocksize + 1) * blocksize + alignLength := srcAlignEnd - srcAlignStart + dstAlignEnd := dstAlignStart + alignLength + + // fill the area before the first aligned block + var copied uint64 + c1, _, err := copyRange(dst, src, srcOffset, srcAlignStart-srcOffset, dstOffset) + if err != nil { + return c1, 0, err + } + copied += c1 + // fill the area after the last aligned block + c2, _, err := copyRange(dst, src, srcAlignEnd, srcOffset+length-srcAlignEnd, dstAlignEnd) + if err != nil { + return copied + c2, 0, err + } + copied += c2 + // close the aligned blocks + if err := cloneRange(dst, src, srcAlignStart, alignLength, dstAlignStart); err != nil { + // Not every filesystem that passes the CanClone probe can clone every + // range. ZFS for example requires alignment to its record size and + // refuses to clone data that hasn't been committed to disk yet. Fall + // back to copying the blocks. + c3, _, err := copyRange(dst, src, srcAlignStart, alignLength, dstAlignStart) + return copied + c3, 0, err + } + return copied, alignLength, nil } diff --git a/seed_test.go b/seed_test.go new file mode 100644 index 0000000..67c699c --- /dev/null +++ b/seed_test.go @@ -0,0 +1,28 @@ +package desync + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// Positions before minPos aren't looked at, however often the chunk repeats +// there. Otherwise every lookup in a long stretch of one repeated chunk would +// walk all its earlier occurrences, making planning quadratic. +func TestLongestMatchFromSkipsEarlierPositions(t *testing.T) { + const n = 10000 + var idx Index + for i := range n { + idx.Chunks = append(idx.Chunks, IndexChunk{ID: ChunkID{1}, Start: uint64(i) * 100, Size: 100}) + } + s := newSeedIndex(idx, false) + + var considered int + pos, length := s.longestMatchFrom(idx.Chunks, 9000, 9001, 100, func(p int) int { + considered++ + return p - 9000 + }) + require.Equal(t, 9100, pos) + require.Equal(t, 100, length) + require.Equal(t, 100, considered, "candidates looked at") +} diff --git a/selfseed.go b/selfseed.go deleted file mode 100644 index 2d1e53c..0000000 --- a/selfseed.go +++ /dev/null @@ -1,96 +0,0 @@ -package desync - -import ( - "context" - "sync" -) - -// TODO: selfSeed doesn't have a LongestMatchWith! -// var _ Seed = &selfSeed{} - -// selfSeed is used to populate a contiguous seed during extraction in order -// to copy/clone ranges that were written to the output file earlier. This is -// to potentially dedup/reflink duplicate chunks or ranges of chunks within the -// same file. -type selfSeed struct { - file string - index Index - pos map[ChunkID][]int - canReflink bool - written int - mu sync.RWMutex - cache map[int]int -} - -// newSelfSeed initializes a new seed based on the file being extracted -func newSelfSeed(file string, index Index) (*selfSeed, error) { - s := selfSeed{ - file: file, - pos: make(map[ChunkID][]int), - index: index, - canReflink: CanClone(file, file), - cache: make(map[int]int), - } - return &s, nil -} - -// add records a new segment that's been written to the file. Since only contiguous -// ranges of chunks are considered and writing happens concurrently, the segment -// written here will not be usable until all earlier chunks have been written as -// well. -func (s *selfSeed) add(segment IndexSegment) { - s.mu.Lock() - defer s.mu.Unlock() - - // Make a record of this segment in the cache since those could come in - // out-of-order - s.cache[segment.first] = segment.last + 1 - - // Advance pos until we find a chunk we don't yet have recorded while recording - // the chunk positions we do have in the position map used to find seed matches. - // Since it's guaranteed that the numbers are only increasing, we drop old numbers - // from the cache map to keep its size to a minimum and only store out-of-sequence - // numbers - for { - // See if we can advance the write pointer in the self-seed which requires - // consecutive chunks. If we don't have the next segment yet, just keep it - // in the cache until we do. - next, ok := s.cache[s.written] - if !ok { - break - } - // Record all chunks in this segment as written by adding them to the position map - for i := s.written; i < next; i++ { - chunk := s.index.Chunks[i] - s.pos[chunk.ID] = append(s.pos[chunk.ID], i) - } - delete(s.cache, s.written) - s.written = next - } -} - -// getChunk returns a segment with the requested chunk ID. If selfSeed doesn't -// have the requested chunk, nil will be returned. -func (s *selfSeed) getChunk(id ChunkID) SeedSegment { - s.mu.RLock() - pos, ok := s.pos[id] - s.mu.RUnlock() - if !ok { - return nil - } - first := pos[0] - return newFileSeedSegment(s.file, s.index.Chunks[first:first+1], s.canReflink) -} - -func (s *selfSeed) RegenerateIndex(ctx context.Context, n int, attempt int, seedNumber int) error { - panic("A selfSeed can't be regenerated") -} - -func (s *selfSeed) SetInvalid(value bool) { - panic("A selfSeed is never expected to be invalid") -} - -func (s *selfSeed) IsInvalid() bool { - // A selfSeed is never expected to be invalid - return false -} diff --git a/selfseed_test.go b/selfseed_test.go deleted file mode 100644 index 9207a4f..0000000 --- a/selfseed_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package desync - -import ( - "context" - "crypto/md5" - "crypto/rand" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestSelfSeed(t *testing.T) { - // Setup a temporary store - store := t.TempDir() - - s, err := NewLocalStore(store, StoreOptions{}) - require.NoError(t, err) - - // Build a number of fake chunks that can then be used in the test in any order - type rawChunk struct { - id ChunkID - data []byte - } - size := 1024 - numChunks := 10 - chunks := make([]rawChunk, numChunks) - - for i := range numChunks { - b := make([]byte, size) - rand.Read(b) - chunk := NewChunk(b) - require.NoError(t, s.StoreChunk(chunk)) - chunks[i] = rawChunk{chunk.ID(), b} - } - - // Define tests with files with different content, by building files out - // of sets of byte slices to create duplication or not between the target and - // its seeds. Also define a min/max of bytes that should be cloned (from the - // self-seed). That number can vary since even with 1 worker goroutine there - // another feeder goroutine which can influence timings/results a little. - tests := map[string]struct { - index []int - minCloned int - maxCloned int - }{ - "single chunk": { - index: []int{0}, - minCloned: 0, - maxCloned: 0, - }, - "repeating single chunk": { - index: []int{0, 0, 0, 0, 0}, - minCloned: 3 * size, - maxCloned: 4 * size, - }, - "repeating chunk sequence": { - index: []int{0, 1, 2, 0, 1, 2, 2}, - minCloned: 4 * size, - maxCloned: 4 * size, - }, - "repeating chunk sequence mid file": { - index: []int{1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3}, - minCloned: 7 * size, - maxCloned: 7 * size, - }, - "repeating chunk sequence reversed": { - index: []int{0, 1, 2, 2, 1, 0}, - minCloned: 2 * size, - maxCloned: 3 * size, - }, - "non-repeating chunks": { - index: []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, - minCloned: 0, - maxCloned: 0, - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - // Build an index from the target chunks - var idx Index - var b []byte - for i, p := range test.index { - chunk := IndexChunk{ - ID: chunks[p].id, - Start: uint64(i * size), - Size: uint64(size), - } - b = append(b, chunks[p].data...) - idx.Chunks = append(idx.Chunks, chunk) - } - - // Calculate the expected checksum - sum := md5.Sum(b) - - // Build a temp target file to extract into - dst := filepath.Join(t.TempDir(), "dst") - - // Extract the file - stats, err := AssembleFile(context.Background(), dst, idx, s, nil, - AssembleOptions{1, InvalidSeedActionBailOut}, - ) - require.NoError(t, err) - - // Compare the checksums to that of the input data - b, err = os.ReadFile(dst) - require.NoError(t, err) - require.Equal(t, sum, md5.Sum(b), "checksum of extracted file doesn't match expected") - - // Compare to the expected number of bytes copied or cloned from the seed - fromSeed := int(stats.BytesCopied + stats.BytesCloned) - require.GreaterOrEqual(t, fromSeed, test.minCloned, "bytes copied/cloned from self-seed") - require.LessOrEqual(t, fromSeed, test.maxCloned, "bytes copied/cloned from self-seed") - }) - } - -} diff --git a/sequencer.go b/sequencer.go deleted file mode 100644 index bcf515d..0000000 --- a/sequencer.go +++ /dev/null @@ -1,177 +0,0 @@ -package desync - -import ( - "context" - "os" - - "golang.org/x/sync/errgroup" -) - -// SeedSequencer is used to find sequences of chunks from seed files when assembling -// a file from an index. Using seeds reduces the need to download and decompress chunks -// from chunk stores. It also enables the use of reflinking/cloning of sections of -// files from a seed file where supported to reduce disk usage. -type SeedSequencer struct { - seeds []Seed - index Index - current int -} - -// SeedSegmentCandidate represent a single segment that we expect to use -// in a Plan -type SeedSegmentCandidate struct { - seed Seed - source SeedSegment - indexSegment IndexSegment -} - -type Plan []SeedSegmentCandidate - -var MockValidate = false - -// NewSeedSequencer initializes a new sequencer from a number of seeds. -func NewSeedSequencer(idx Index, src ...Seed) *SeedSequencer { - return &SeedSequencer{ - seeds: src, - index: idx, - } -} - -// Plan returns a new possible plan, representing an ordered list of -// segments that can be used to re-assemble the requested file -func (r *SeedSequencer) Plan() (plan Plan) { - for { - seed, segment, source, done := r.Next() - plan = append(plan, SeedSegmentCandidate{seed, source, segment}) - if done { - break - } - } - return plan -} - -// Next returns a sequence of index chunks (from the target index) and the -// longest matching segment from one of the seeds. If source is nil, no -// match was found in the seeds and the chunk needs to be retrieved from a -// store. If done is true, the sequencer is complete. -func (r *SeedSequencer) Next() (seed Seed, segment IndexSegment, source SeedSegment, done bool) { - var ( - max uint64 - advance = 1 - ) - for _, s := range r.seeds { - n, m := s.LongestMatchWith(r.index.Chunks[r.current:]) - if n > 0 && m.Size() > max { - seed = s - source = m - advance = n - max = m.Size() - } - } - - segment = IndexSegment{index: r.index, first: r.current, last: r.current + advance - 1} - r.current += advance - return seed, segment, source, r.current >= len(r.index.Chunks) -} - -// Rewind resets the current target index to the beginning. -func (r *SeedSequencer) Rewind() { - r.current = 0 -} - -// isFileSeed returns true if this segment is pointing to a fileSeed -func (s SeedSegmentCandidate) isFileSeed() bool { - // We expect an empty filename when using nullSeeds - return s.source != nil && s.source.FileName() != "" -} - -// RegenerateInvalidSeeds regenerates the index to match the unexpected seed content -func (r *SeedSequencer) RegenerateInvalidSeeds(ctx context.Context, n int, attempt int) error { - seedNumber := 1 - for _, s := range r.seeds { - if s.IsInvalid() { - if err := s.RegenerateIndex(ctx, n, attempt, seedNumber); err != nil { - return err - } - seedNumber += 1 - } - } - return nil -} - -// Validate validates a proposed plan by checking if all the chosen chunks -// are correctly provided from the seeds. In case a seed has invalid chunks, the -// entire seed is marked as invalid and an error is returned. -func (p Plan) Validate(ctx context.Context, n int, pb ProgressBar) (err error) { - type Job struct { - candidate SeedSegmentCandidate - file *os.File - } - var ( - in = make(chan Job) - fileMap = make(map[string]*os.File) - ) - if MockValidate { - // This is used in the automated tests to mock a plan that is valid - return nil - } - length := 0 - for _, s := range p { - if !s.isFileSeed() { - continue - } - length += s.indexSegment.lengthChunks() - } - pb.SetTotal(length) - pb.Start() - defer pb.Finish() - // Share a single file descriptor per seed for all the goroutines - for _, s := range p { - if !s.isFileSeed() { - continue - } - name := s.source.FileName() - if _, present := fileMap[name]; present { - continue - } else { - file, err := os.Open(name) - if err != nil { - // We were not able to open the seed. Mark it as invalid and return - s.seed.SetInvalid(true) - return err - } - fileMap[name] = file - defer file.Close() - } - } - g, ctx := errgroup.WithContext(ctx) - // Concurrently validate all the chunks in this plan - for range n { - g.Go(func() error { - for job := range in { - if err := job.candidate.source.Validate(job.file); err != nil { - job.candidate.seed.SetInvalid(true) - return err - } - pb.Add(job.candidate.indexSegment.lengthChunks()) - } - return nil - }) - } - -loop: - for _, s := range p { - if !s.isFileSeed() { - // This is not a fileSeed, we have nothing to validate - continue - } - select { - case <-ctx.Done(): - break loop - case in <- Job{s, fileMap[s.source.FileName()]}: - } - } - close(in) - - return g.Wait() -} diff --git a/store.go b/store.go index dc52920..906e080 100644 --- a/store.go +++ b/store.go @@ -206,6 +206,14 @@ func (o StoreOptions) tlsClientConfig() (*tls.Config, error) { return nil, fmt.Errorf("failed to load client certificate from %s: %w", o.ClientCert, err) } tlsConfig.Certificates = []tls.Certificate{certificate} + // Always present the configured certificate, regardless of the CAs the + // server lists in its CertificateRequest. Go's default selection sends + // no certificate when the list doesn't include the issuer, which breaks + // TLS terminators that advertise a fixed set of public CAs and forward + // the client certificate to the backend for verification. + tlsConfig.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + return &certificate, nil + } } // Load custom CA set if provided diff --git a/store_tls_test.go b/store_tls_test.go new file mode 100644 index 0000000..890a841 --- /dev/null +++ b/store_tls_test.go @@ -0,0 +1,98 @@ +package desync + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// newTestCA returns a self-signed CA certificate and its key. +func newTestCA(t *testing.T, cn string) (*x509.Certificate, *ecdsa.PrivateKey) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: cn}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageCertSign, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + require.NoError(t, err) + cert, err := x509.ParseCertificate(der) + require.NoError(t, err) + return cert, key +} + +// TestTLSClientCertSentRegardlessOfAcceptableCAs checks that the configured +// client certificate is sent even when the server's CertificateRequest lists +// CAs that did not issue it. Some TLS terminators (for example managed +// ingresses that forward the client certificate to the backend for +// verification) advertise a fixed list of public CAs, and Go's default +// selection would then send no certificate at all. +func TestTLSClientCertSentRegardlessOfAcceptableCAs(t *testing.T) { + clientCA, clientCAKey := newTestCA(t, "private client CA") + unrelatedCA, _ := newTestCA(t, "unrelated public CA") + + // Client certificate issued by the private CA. + clientKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + clientTmpl := &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{CommonName: "client"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + } + clientDER, err := x509.CreateCertificate(rand.Reader, clientTmpl, clientCA, &clientKey.PublicKey, clientCAKey) + require.NoError(t, err) + + dir := t.TempDir() + certFile := filepath.Join(dir, "client.crt") + keyFile := filepath.Join(dir, "client.key") + keyDER, err := x509.MarshalECPrivateKey(clientKey) + require.NoError(t, err) + require.NoError(t, os.WriteFile(certFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: clientDER}), 0o600)) + require.NoError(t, os.WriteFile(keyFile, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600)) + + // The server advertises only the unrelated CA in its CertificateRequest + // and requires a certificate without verifying it, like an ingress that + // leaves verification to the backend. + advertised := x509.NewCertPool() + advertised.AddCert(unrelatedCA) + var gotCN string + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if len(r.TLS.PeerCertificates) > 0 { + gotCN = r.TLS.PeerCertificates[0].Subject.CommonName + } + })) + srv.TLS = &tls.Config{ClientAuth: tls.RequireAnyClientCert, ClientCAs: advertised} + srv.StartTLS() + defer srv.Close() + + opt := StoreOptions{ClientCert: certFile, ClientKey: keyFile, TrustInsecure: true} + tlsConfig, err := opt.tlsClientConfig() + require.NoError(t, err) + client := &http.Client{Transport: &http.Transport{TLSClientConfig: tlsConfig}} + resp, err := client.Get(srv.URL) + require.NoError(t, err, "client certificate was likely not sent") + require.NoError(t, resp.Body.Close()) + require.Equal(t, "client", gotCN) +}