diff --git a/cmd/stindex/accounting.go b/cmd/stindex/accounting.go new file mode 100644 index 000000000..fbe73fde9 --- /dev/null +++ b/cmd/stindex/accounting.go @@ -0,0 +1,58 @@ +// Copyright (C) 2020 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package main + +import ( + "fmt" + "log" + "os" + "text/tabwriter" + + "github.com/syncthing/syncthing/lib/db/backend" +) + +// account prints key and data size statistics per class +func account(ldb backend.Backend) { + it, err := ldb.NewPrefixIterator(nil) + if err != nil { + log.Fatal(err) + } + + var ksizes [256]int + var dsizes [256]int + var counts [256]int + var max [256]int + + for it.Next() { + key := it.Key() + t := key[0] + ds := len(it.Value()) + ks := len(key) + s := ks + ds + + counts[t]++ + ksizes[t] += ks + dsizes[t] += ds + if s > max[t] { + max[t] = s + } + } + + tw := tabwriter.NewWriter(os.Stdout, 1, 1, 1, ' ', tabwriter.AlignRight) + toti, totds, totks := 0, 0, 0 + for t := range ksizes { + if ksizes[t] > 0 { + // yes metric kilobytes 🤘 + fmt.Fprintf(tw, "0x%02x:\t%d items,\t%d KB keys +\t%d KB data,\t%d B +\t%d B avg,\t%d B max\t\n", t, counts[t], ksizes[t]/1000, dsizes[t]/1000, ksizes[t]/counts[t], dsizes[t]/counts[t], max[t]) + toti += counts[t] + totds += dsizes[t] + totks += ksizes[t] + } + } + fmt.Fprintf(tw, "Total\t%d items,\t%d KB keys +\t%d KB data.\t\n", toti, totks/1000, totds/1000) + tw.Flush() +} diff --git a/cmd/stindex/main.go b/cmd/stindex/main.go index ed48c35d0..e5cd54ee2 100644 --- a/cmd/stindex/main.go +++ b/cmd/stindex/main.go @@ -35,15 +35,18 @@ func main() { log.Fatal(err) } - if mode == "dump" { + switch mode { + case "dump": dump(ldb) - } else if mode == "dumpsize" { + case "dumpsize": dumpsize(ldb) - } else if mode == "idxck" { + case "idxck": if !idxck(ldb) { os.Exit(1) } - } else { + case "account": + account(ldb) + default: fmt.Println("Unknown mode") } } diff --git a/go.mod b/go.mod index e07ae08d3..e9382ac0b 100644 --- a/go.mod +++ b/go.mod @@ -35,11 +35,14 @@ require ( github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 github.com/sasha-s/go-deadlock v0.2.0 github.com/shirou/gopsutil v0.0.0-20190714054239-47ef3260b6bf + github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/syncthing/notify v0.0.0-20190709140112-69c7a957d3e2 github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d github.com/thejerf/suture v3.0.2+incompatible github.com/urfave/cli v1.22.2 github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0 + github.com/willf/bitset v1.1.10 // indirect + github.com/willf/bloom v2.0.3+incompatible golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 golang.org/x/text v0.3.2 diff --git a/go.sum b/go.sum index 16ad30245..2d11c4eb8 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,10 @@ github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrU github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -178,7 +180,10 @@ github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= @@ -199,6 +204,10 @@ github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0 h1:okhMind4q9H1OxF44gNegWkiP4H/gsTFLalHFa4OOUI= github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0/go.mod h1:TTbGUfE+cXXceWtbTHq6lqcTvYPBKLNejBEbnUsQJtU= +github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc= +github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/willf/bloom v2.0.3+incompatible h1:QDacWdqcAUI1MPOwIQZRy9kOR7yxfyEmxX8Wdm2/JPA= +github.com/willf/bloom v2.0.3+incompatible/go.mod h1:MmAltL9pDMNTrvUkxdg0k0q5I0suxmuwp3KbyrZLOZ8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= @@ -246,6 +255,7 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= diff --git a/lib/db/backend/backend.go b/lib/db/backend/backend.go index e5f6251b7..682e64dd7 100644 --- a/lib/db/backend/backend.go +++ b/lib/db/backend/backend.go @@ -100,6 +100,7 @@ type Backend interface { NewReadTransaction() (ReadTransaction, error) NewWriteTransaction() (WriteTransaction, error) Close() error + Compact() error } type Tuning int diff --git a/lib/db/backend/leveldb_backend.go b/lib/db/backend/leveldb_backend.go index 7ac5c88a0..b714accda 100644 --- a/lib/db/backend/leveldb_backend.go +++ b/lib/db/backend/leveldb_backend.go @@ -81,6 +81,10 @@ func (b *leveldbBackend) Delete(key []byte) error { return wrapLeveldbErr(b.ldb.Delete(key, nil)) } +func (b *leveldbBackend) Compact() error { + return wrapLeveldbErr(b.ldb.CompactRange(util.Range{})) +} + // leveldbSnapshot implements backend.ReadTransaction type leveldbSnapshot struct { snap *leveldb.Snapshot diff --git a/lib/db/db_test.go b/lib/db/db_test.go index ed7b0310e..37c929883 100644 --- a/lib/db/db_test.go +++ b/lib/db/db_test.go @@ -165,7 +165,7 @@ func TestUpdate0to3(t *testing.T) { folder := []byte(update0to3Folder) - if err := updater.updateSchema0to1(); err != nil { + if err := updater.updateSchema0to1(0); err != nil { t.Fatal(err) } @@ -187,7 +187,7 @@ func TestUpdate0to3(t *testing.T) { t.Error("Invalid file wasn't added to global list") } - if err := updater.updateSchema1to2(); err != nil { + if err := updater.updateSchema1to2(1); err != nil { t.Fatal(err) } @@ -214,7 +214,7 @@ func TestUpdate0to3(t *testing.T) { t.Error("Local file wasn't added to sequence bucket", err) } - if err := updater.updateSchema2to3(); err != nil { + if err := updater.updateSchema2to3(2); err != nil { t.Fatal(err) } diff --git a/lib/db/keyer.go b/lib/db/keyer.go index 1f88d5d65..d9e7811a8 100644 --- a/lib/db/keyer.go +++ b/lib/db/keyer.go @@ -59,6 +59,9 @@ const ( // KeyTypeNeed = KeyTypeNeed = 12 + + // KeyTypeBlockList = BlockList + KeyTypeBlockList = 13 ) type keyer interface { @@ -93,6 +96,9 @@ type keyer interface { // Folder metadata GenerateFolderMetaKey(key, folder []byte) (folderMetaKey, error) + + // Block lists + GenerateBlockListKey(key []byte, hash []byte) blockListKey } // defaultKeyer implements our key scheme. It needs folder and device @@ -281,6 +287,19 @@ func (k defaultKeyer) GenerateFolderMetaKey(key, folder []byte) (folderMetaKey, return key, nil } +type blockListKey []byte + +func (k defaultKeyer) GenerateBlockListKey(key []byte, hash []byte) blockListKey { + key = resize(key, keyPrefixLen+len(hash)) + key[0] = KeyTypeBlockList + copy(key[keyPrefixLen:], hash) + return key +} + +func (k blockListKey) BlocksHash() []byte { + return k[keyPrefixLen:] +} + // resize returns a byte slice of the specified size, reusing bs if possible func resize(bs []byte, size int) []byte { if cap(bs) < size { diff --git a/lib/db/lowlevel.go b/lib/db/lowlevel.go index c08848d61..ef3290bfa 100644 --- a/lib/db/lowlevel.go +++ b/lib/db/lowlevel.go @@ -9,9 +9,23 @@ package db import ( "bytes" "encoding/binary" + "time" "github.com/syncthing/syncthing/lib/db/backend" "github.com/syncthing/syncthing/lib/protocol" + "github.com/syncthing/syncthing/lib/sync" + "github.com/willf/bloom" +) + +const ( + // We set the bloom filter capacity to handle 100k individual block lists + // with a false positive probability of 1% for the first pass. Once we know + // how many block lists we have we will use that number instead, if it's + // more than 100k. For fewer than 100k block lists we will just get better + // false positive rate instead. + blockGCBloomCapacity = 100000 + blockGCBloomFalsePositiveRate = 0.01 // 1% + blockGCInterval = 13 * time.Hour ) // Lowlevel is the lowest level database interface. It has a very simple @@ -21,9 +35,12 @@ import ( // any given backend. type Lowlevel struct { backend.Backend - folderIdx *smallIndex - deviceIdx *smallIndex - keyer keyer + folderIdx *smallIndex + deviceIdx *smallIndex + keyer keyer + gcMut sync.RWMutex + gcKeyCount int + gcStop chan struct{} } func NewLowlevel(backend backend.Backend) *Lowlevel { @@ -31,11 +48,19 @@ func NewLowlevel(backend backend.Backend) *Lowlevel { Backend: backend, folderIdx: newSmallIndex(backend, []byte{KeyTypeFolderIdx}), deviceIdx: newSmallIndex(backend, []byte{KeyTypeDeviceIdx}), + gcMut: sync.NewRWMutex(), + gcStop: make(chan struct{}), } db.keyer = newDefaultKeyer(db.folderIdx, db.deviceIdx) + go db.gcRunner() return db } +func (db *Lowlevel) Close() error { + close(db.gcStop) + return db.Backend.Close() +} + // ListFolders returns the list of folders currently in the database func (db *Lowlevel) ListFolders() []string { return db.folderIdx.Values() @@ -44,6 +69,9 @@ func (db *Lowlevel) ListFolders() []string { // updateRemoteFiles adds a list of fileinfos to the database and updates the // global versionlist and metadata. func (db *Lowlevel) updateRemoteFiles(folder, device []byte, fs []protocol.FileInfo, meta *metadataTracker) error { + db.gcMut.RLock() + defer db.gcMut.RUnlock() + t, err := db.newReadWriteTransaction() if err != nil { return err @@ -73,7 +101,7 @@ func (db *Lowlevel) updateRemoteFiles(folder, device []byte, fs []protocol.FileI meta.addFile(devID, f) l.Debugf("insert; folder=%q device=%v %v", folder, devID, f) - if err := t.Put(dk, mustMarshal(&f)); err != nil { + if err := t.putFile(dk, f); err != nil { return err } @@ -97,6 +125,9 @@ func (db *Lowlevel) updateRemoteFiles(folder, device []byte, fs []protocol.FileI // updateLocalFiles adds fileinfos to the db, and updates the global versionlist, // metadata, sequence and blockmap buckets. func (db *Lowlevel) updateLocalFiles(folder []byte, fs []protocol.FileInfo, meta *metadataTracker) error { + db.gcMut.RLock() + defer db.gcMut.RUnlock() + t, err := db.newReadWriteTransaction() if err != nil { return err @@ -151,7 +182,7 @@ func (db *Lowlevel) updateLocalFiles(folder []byte, fs []protocol.FileInfo, meta meta.addFile(protocol.LocalDeviceID, f) l.Debugf("insert (local); folder=%q %v", folder, f) - if err := t.Put(dk, mustMarshal(&f)); err != nil { + if err := t.putFile(dk, f); err != nil { return err } @@ -195,6 +226,9 @@ func (db *Lowlevel) updateLocalFiles(folder []byte, fs []protocol.FileInfo, meta } func (db *Lowlevel) dropFolder(folder []byte) error { + db.gcMut.RLock() + defer db.gcMut.RUnlock() + t, err := db.newReadWriteTransaction() if err != nil { return err @@ -250,6 +284,9 @@ func (db *Lowlevel) dropFolder(folder []byte) error { } func (db *Lowlevel) dropDeviceFolder(device, folder []byte, meta *metadataTracker) error { + db.gcMut.RLock() + defer db.gcMut.RUnlock() + t, err := db.newReadWriteTransaction() if err != nil { return err @@ -425,16 +462,100 @@ func (db *Lowlevel) dropPrefix(prefix []byte) error { return t.commit() } -func unmarshalTrunc(bs []byte, truncate bool) (FileIntf, error) { - if truncate { - var tf FileInfoTruncated - err := tf.Unmarshal(bs) - return tf, err +func (db *Lowlevel) gcRunner() { + t := time.NewTicker(blockGCInterval) + defer t.Stop() + for { + select { + case <-db.gcStop: + return + case <-t.C: + if err := db.gcBlocks(); err != nil { + l.Warnln("Database block GC failed:", err) + } + } + } +} + +func (db *Lowlevel) gcBlocks() error { + // The block GC uses a bloom filter to track used block lists. This means + // iterating over all items, adding their block lists to the filter, then + // iterating over the block lists and removing those that don't match the + // filter. The filter will give false positives so we will keep around one + // percent of block lists that we don't really need (at most). + // + // Block GC needs to run when there are no modifications to the FileInfos or + // block lists. + + db.gcMut.Lock() + defer db.gcMut.Unlock() + + t, err := db.newReadWriteTransaction() + if err != nil { + return err + } + defer t.Release() + + // Set up the bloom filter with the initial capacity and false positive + // rate, or higher capacity if we've done this before and seen lots of block + // lists. + + capacity := blockGCBloomCapacity + if db.gcKeyCount > capacity { + capacity = db.gcKeyCount + } + filter := bloom.NewWithEstimates(uint(capacity), blockGCBloomFalsePositiveRate) + + // Iterate the FileInfos, unmarshal the blocks hashes and add them to + // the filter. + + it, err := db.NewPrefixIterator([]byte{KeyTypeDevice}) + if err != nil { + return err + } + for it.Next() { + var bl BlocksHashOnly + if err := bl.Unmarshal(it.Value()); err != nil { + return err + } + filter.Add(bl.BlocksHash) + } + it.Release() + if err := it.Error(); err != nil { + return err } - var tf protocol.FileInfo - err := tf.Unmarshal(bs) - return tf, err + // Iterate over block lists, removing keys with hashes that don't match + // the filter. + + it, err = db.NewPrefixIterator([]byte{KeyTypeBlockList}) + if err != nil { + return err + } + matched := 0 + for it.Next() { + key := blockListKey(it.Key()) + if filter.Test(key.BlocksHash()) { + matched++ + continue + } + if err := t.Delete(key); err != nil { + return err + } + } + it.Release() + if err := it.Error(); err != nil { + return err + } + + // Remember the number of unique keys we kept until the next pass. + db.gcKeyCount = matched + + if err := t.Commit(); err != nil { + return err + } + + return db.Compact() } func unmarshalVersionList(data []byte) (VersionList, bool) { diff --git a/lib/db/schemaupdater.go b/lib/db/schemaupdater.go index 7f6c8b1b3..803394f31 100644 --- a/lib/db/schemaupdater.go +++ b/lib/db/schemaupdater.go @@ -22,9 +22,10 @@ import ( // 5: v0.14.49 // 6: v0.14.50 // 7: v0.14.53 +// 8: v1.4.0 const ( - dbVersion = 7 - dbMinSyncthingVersion = "v0.14.53" + dbVersion = 8 + dbMinSyncthingVersion = "v1.4.0" ) type databaseDowngradeError struct { @@ -68,35 +69,26 @@ func (db *schemaUpdater) updateSchema() error { return nil } - if prevVersion < 1 { - if err := db.updateSchema0to1(); err != nil { - return err - } + type migration struct { + schemaVersion int64 + migration func(prevVersion int) error } - if prevVersion < 2 { - if err := db.updateSchema1to2(); err != nil { - return err - } + var migrations = []migration{ + {1, db.updateSchema0to1}, + {2, db.updateSchema1to2}, + {3, db.updateSchema2to3}, + {5, db.updateSchemaTo5}, + {6, db.updateSchema5to6}, + {7, db.updateSchema6to7}, + {8, db.updateSchema7to8}, } - if prevVersion < 3 { - if err := db.updateSchema2to3(); err != nil { - return err - } - } - // This update fixes problems existing in versions 3 and 4 - if prevVersion == 3 || prevVersion == 4 { - if err := db.updateSchemaTo5(); err != nil { - return err - } - } - if prevVersion < 6 { - if err := db.updateSchema5to6(); err != nil { - return err - } - } - if prevVersion < 7 { - if err := db.updateSchema6to7(); err != nil { - return err + + for _, m := range migrations { + if prevVersion < m.schemaVersion { + l.Infof("Migrating database to schema version %d...", m.schemaVersion) + if err := m.migration(int(prevVersion)); err != nil { + return err + } } } @@ -107,10 +99,11 @@ func (db *schemaUpdater) updateSchema() error { return err } - return nil + l.Infoln("Compacting database after migration...") + return db.Compact() } -func (db *schemaUpdater) updateSchema0to1() error { +func (db *schemaUpdater) updateSchema0to1(_ int) error { t, err := db.newReadWriteTransaction() if err != nil { return err @@ -216,7 +209,7 @@ func (db *schemaUpdater) updateSchema0to1() error { // updateSchema1to2 introduces a sequenceKey->deviceKey bucket for local items // to allow iteration in sequence order (simplifies sending indexes). -func (db *schemaUpdater) updateSchema1to2() error { +func (db *schemaUpdater) updateSchema1to2(_ int) error { t, err := db.newReadWriteTransaction() if err != nil { return err @@ -251,7 +244,7 @@ func (db *schemaUpdater) updateSchema1to2() error { } // updateSchema2to3 introduces a needKey->nil bucket for locally needed files. -func (db *schemaUpdater) updateSchema2to3() error { +func (db *schemaUpdater) updateSchema2to3(_ int) error { t, err := db.newReadWriteTransaction() if err != nil { return err @@ -302,7 +295,11 @@ func (db *schemaUpdater) updateSchema2to3() error { // release candidates (dbVersion 3 and 4) // https://github.com/syncthing/syncthing/issues/5007 // https://github.com/syncthing/syncthing/issues/5053 -func (db *schemaUpdater) updateSchemaTo5() error { +func (db *schemaUpdater) updateSchemaTo5(prevVersion int) error { + if prevVersion != 3 && prevVersion != 4 { + return nil + } + t, err := db.newReadWriteTransaction() if err != nil { return err @@ -321,10 +318,10 @@ func (db *schemaUpdater) updateSchemaTo5() error { return err } - return db.updateSchema2to3() + return db.updateSchema2to3(2) } -func (db *schemaUpdater) updateSchema5to6() error { +func (db *schemaUpdater) updateSchema5to6(_ int) error { // For every local file with the Invalid bit set, clear the Invalid bit and // set LocalFlags = FlagLocalIgnored. @@ -369,7 +366,7 @@ func (db *schemaUpdater) updateSchema5to6() error { // updateSchema6to7 checks whether all currently locally needed files are really // needed and removes them if not. -func (db *schemaUpdater) updateSchema6to7() error { +func (db *schemaUpdater) updateSchema6to7(_ int) error { t, err := db.newReadWriteTransaction() if err != nil { return err @@ -423,3 +420,36 @@ func (db *schemaUpdater) updateSchema6to7() error { } return t.commit() } + +func (db *schemaUpdater) updateSchema7to8(_ int) error { + // Loads and rewrites all files with blocks, to deduplicate block lists. + + t, err := db.newReadWriteTransaction() + if err != nil { + return err + } + defer t.close() + + it, err := t.NewPrefixIterator([]byte{KeyTypeDevice}) + if err != nil { + return err + } + for it.Next() { + var fi protocol.FileInfo + if err := fi.Unmarshal(it.Value()); err != nil { + return err + } + if fi.Blocks == nil { + continue + } + if err := t.putFile(it.Key(), fi); err != nil { + return err + } + } + it.Release() + if err := it.Error(); err != nil { + return err + } + + return t.commit() +} diff --git a/lib/db/structs.pb.go b/lib/db/structs.pb.go index 8151e6749..312856cff 100644 --- a/lib/db/structs.pb.go +++ b/lib/db/structs.pb.go @@ -110,6 +110,7 @@ type FileInfoTruncated struct { Sequence int64 `protobuf:"varint,10,opt,name=sequence,proto3" json:"sequence,omitempty"` // repeated BlockInfo Blocks = 16 SymlinkTarget string `protobuf:"bytes,17,opt,name=symlink_target,json=symlinkTarget,proto3" json:"symlink_target,omitempty"` + BlocksHash []byte `protobuf:"bytes,18,opt,name=blocks_hash,json=blocksHash,proto3" json:"blocks_hash,omitempty"` Type protocol.FileInfoType `protobuf:"varint,2,opt,name=type,proto3,enum=protocol.FileInfoType" json:"type,omitempty"` Permissions uint32 `protobuf:"varint,4,opt,name=permissions,proto3" json:"permissions,omitempty"` ModifiedNs int32 `protobuf:"varint,11,opt,name=modified_ns,json=modifiedNs,proto3" json:"modified_ns,omitempty"` @@ -153,6 +154,82 @@ func (m *FileInfoTruncated) XXX_DiscardUnknown() { var xxx_messageInfo_FileInfoTruncated proto.InternalMessageInfo +// BlockList is the structure used to store block lists +type BlockList struct { + Blocks []protocol.BlockInfo `protobuf:"bytes,1,rep,name=Blocks,proto3" json:"Blocks"` +} + +func (m *BlockList) Reset() { *m = BlockList{} } +func (m *BlockList) String() string { return proto.CompactTextString(m) } +func (*BlockList) ProtoMessage() {} +func (*BlockList) Descriptor() ([]byte, []int) { + return fileDescriptor_e774e8f5f348d14d, []int{3} +} +func (m *BlockList) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *BlockList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_BlockList.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *BlockList) XXX_Merge(src proto.Message) { + xxx_messageInfo_BlockList.Merge(m, src) +} +func (m *BlockList) XXX_Size() int { + return m.ProtoSize() +} +func (m *BlockList) XXX_DiscardUnknown() { + xxx_messageInfo_BlockList.DiscardUnknown(m) +} + +var xxx_messageInfo_BlockList proto.InternalMessageInfo + +// BlocksHashOnly is used to only unmarshal the block list hash from a FileInfo +type BlocksHashOnly struct { + BlocksHash []byte `protobuf:"bytes,18,opt,name=blocks_hash,json=blocksHash,proto3" json:"blocks_hash,omitempty"` +} + +func (m *BlocksHashOnly) Reset() { *m = BlocksHashOnly{} } +func (m *BlocksHashOnly) String() string { return proto.CompactTextString(m) } +func (*BlocksHashOnly) ProtoMessage() {} +func (*BlocksHashOnly) Descriptor() ([]byte, []int) { + return fileDescriptor_e774e8f5f348d14d, []int{4} +} +func (m *BlocksHashOnly) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *BlocksHashOnly) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_BlocksHashOnly.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *BlocksHashOnly) XXX_Merge(src proto.Message) { + xxx_messageInfo_BlocksHashOnly.Merge(m, src) +} +func (m *BlocksHashOnly) XXX_Size() int { + return m.ProtoSize() +} +func (m *BlocksHashOnly) XXX_DiscardUnknown() { + xxx_messageInfo_BlocksHashOnly.DiscardUnknown(m) +} + +var xxx_messageInfo_BlocksHashOnly proto.InternalMessageInfo + // For each folder and device we keep one of these to track the current // counts and sequence. We also keep one for the global state of the folder. type Counts struct { @@ -170,7 +247,7 @@ func (m *Counts) Reset() { *m = Counts{} } func (m *Counts) String() string { return proto.CompactTextString(m) } func (*Counts) ProtoMessage() {} func (*Counts) Descriptor() ([]byte, []int) { - return fileDescriptor_e774e8f5f348d14d, []int{3} + return fileDescriptor_e774e8f5f348d14d, []int{5} } func (m *Counts) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -208,7 +285,7 @@ func (m *CountsSet) Reset() { *m = CountsSet{} } func (m *CountsSet) String() string { return proto.CompactTextString(m) } func (*CountsSet) ProtoMessage() {} func (*CountsSet) Descriptor() ([]byte, []int) { - return fileDescriptor_e774e8f5f348d14d, []int{4} + return fileDescriptor_e774e8f5f348d14d, []int{6} } func (m *CountsSet) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -241,6 +318,8 @@ func init() { proto.RegisterType((*FileVersion)(nil), "db.FileVersion") proto.RegisterType((*VersionList)(nil), "db.VersionList") proto.RegisterType((*FileInfoTruncated)(nil), "db.FileInfoTruncated") + proto.RegisterType((*BlockList)(nil), "db.BlockList") + proto.RegisterType((*BlocksHashOnly)(nil), "db.BlocksHashOnly") proto.RegisterType((*Counts)(nil), "db.Counts") proto.RegisterType((*CountsSet)(nil), "db.CountsSet") } @@ -248,50 +327,54 @@ func init() { func init() { proto.RegisterFile("structs.proto", fileDescriptor_e774e8f5f348d14d) } var fileDescriptor_e774e8f5f348d14d = []byte{ - // 683 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0x4f, 0x6f, 0xda, 0x4e, - 0x10, 0xc5, 0xc1, 0x10, 0x18, 0x20, 0xbf, 0x64, 0x15, 0x45, 0x16, 0xd2, 0xcf, 0x58, 0x54, 0x95, - 0xac, 0x1e, 0xa0, 0x4d, 0x6e, 0xed, 0x8d, 0x46, 0x91, 0x90, 0xaa, 0xb6, 0x5a, 0xa2, 0x9c, 0x2a, - 0x21, 0xff, 0x59, 0xc8, 0x2a, 0xc6, 0x4b, 0xbc, 0x4b, 0x22, 0xf2, 0x29, 0x7a, 0xec, 0x31, 0x1f, - 0x27, 0xc7, 0x1c, 0xab, 0x1e, 0x50, 0x0a, 0x3d, 0xf4, 0x63, 0x54, 0xbb, 0x6b, 0x1b, 0x37, 0xa7, - 0xde, 0xe6, 0xbd, 0x19, 0x7b, 0x66, 0xde, 0xbc, 0x85, 0x16, 0x17, 0xc9, 0x22, 0x10, 0xbc, 0x37, - 0x4f, 0x98, 0x60, 0x68, 0x27, 0xf4, 0xdb, 0x2f, 0x12, 0x32, 0x67, 0xbc, 0xaf, 0x08, 0x7f, 0x31, - 0xe9, 0x4f, 0xd9, 0x94, 0x29, 0xa0, 0x22, 0x5d, 0xd8, 0x3e, 0x8a, 0xa8, 0xaf, 0x4b, 0x02, 0x16, - 0xf5, 0x7d, 0x32, 0xd7, 0x7c, 0xf7, 0x1a, 0x1a, 0x67, 0x34, 0x22, 0x17, 0x24, 0xe1, 0x94, 0xc5, - 0xe8, 0x35, 0xec, 0xde, 0xe8, 0xd0, 0x32, 0x1c, 0xc3, 0x6d, 0x1c, 0xef, 0xf7, 0xb2, 0x8f, 0x7a, - 0x17, 0x24, 0x10, 0x2c, 0x19, 0x98, 0x0f, 0xab, 0x4e, 0x09, 0x67, 0x65, 0xe8, 0x08, 0xaa, 0x21, - 0xb9, 0xa1, 0x01, 0xb1, 0x76, 0x1c, 0xc3, 0x6d, 0xe2, 0x14, 0x21, 0x0b, 0x76, 0x69, 0x7c, 0xe3, - 0x45, 0x34, 0xb4, 0xca, 0x8e, 0xe1, 0xd6, 0x70, 0x06, 0xbb, 0x67, 0xd0, 0x48, 0xdb, 0x7d, 0xa0, - 0x5c, 0xa0, 0x37, 0x50, 0x4b, 0xff, 0xc5, 0x2d, 0xc3, 0x29, 0xbb, 0x8d, 0xe3, 0xff, 0x7a, 0xa1, - 0xdf, 0x2b, 0x4c, 0x95, 0xb6, 0xcc, 0xcb, 0xde, 0x9a, 0xdf, 0xee, 0x3b, 0xa5, 0xee, 0x93, 0x09, - 0x07, 0xb2, 0x6a, 0x18, 0x4f, 0xd8, 0x79, 0xb2, 0x88, 0x03, 0x4f, 0x90, 0x10, 0x21, 0x30, 0x63, - 0x6f, 0x46, 0xd4, 0xf8, 0x75, 0xac, 0x62, 0xc9, 0x71, 0x7a, 0x47, 0xd4, 0x20, 0x65, 0xac, 0x62, - 0xf4, 0x3f, 0xc0, 0x8c, 0x85, 0x74, 0x42, 0x49, 0x38, 0xe6, 0x56, 0x45, 0x65, 0xea, 0x19, 0x33, - 0x42, 0x5f, 0xa0, 0x91, 0xa7, 0xfd, 0xa5, 0xd5, 0x74, 0x0c, 0xd7, 0x1c, 0xbc, 0x93, 0x73, 0xfc, - 0x58, 0x75, 0x4e, 0xa6, 0x54, 0x5c, 0x2e, 0xfc, 0x5e, 0xc0, 0x66, 0x7d, 0xbe, 0x8c, 0x03, 0x71, - 0x49, 0xe3, 0x69, 0x21, 0x2a, 0x6a, 0xdd, 0x1b, 0x5d, 0xb2, 0x44, 0x0c, 0x4f, 0x71, 0xde, 0x6e, - 0xb0, 0x2c, 0xca, 0x5c, 0xff, 0x37, 0x99, 0xdb, 0x50, 0xe3, 0xe4, 0x7a, 0x41, 0xe2, 0x80, 0x58, - 0xa0, 0x86, 0xcd, 0x31, 0x7a, 0x09, 0x7b, 0x7c, 0x39, 0x8b, 0x68, 0x7c, 0x35, 0x16, 0x5e, 0x32, - 0x25, 0xc2, 0x3a, 0x50, 0xcb, 0xb7, 0x52, 0xf6, 0x5c, 0x91, 0xe8, 0x15, 0x98, 0x62, 0x39, 0xd7, - 0x77, 0xda, 0x3b, 0x3e, 0xda, 0x76, 0xcc, 0x45, 0x5c, 0xce, 0x09, 0x56, 0x35, 0xc8, 0x81, 0xc6, - 0x9c, 0x24, 0x33, 0xca, 0xf5, 0x5d, 0x4c, 0xc7, 0x70, 0x5b, 0xb8, 0x48, 0xa1, 0x4e, 0x41, 0xa0, - 0x98, 0x5b, 0x0d, 0xc7, 0x70, 0x2b, 0xdb, 0x1d, 0x3f, 0x72, 0xd4, 0x07, 0xf0, 0x23, 0x16, 0x5c, - 0x8d, 0x95, 0xf4, 0x2d, 0x99, 0x1f, 0xec, 0xaf, 0x57, 0x9d, 0x26, 0xf6, 0x6e, 0x07, 0x32, 0x31, - 0xa2, 0x77, 0x04, 0xd7, 0xfd, 0x2c, 0x94, 0x3d, 0x23, 0x16, 0x78, 0xd1, 0x78, 0x12, 0x79, 0x53, - 0x6e, 0xfd, 0xde, 0x55, 0x4d, 0x41, 0x71, 0x67, 0x92, 0x92, 0x9e, 0x0a, 0x49, 0x44, 0x04, 0x09, - 0xad, 0xaa, 0xf6, 0x54, 0x0a, 0x91, 0xbb, 0x75, 0x9b, 0xfc, 0xac, 0x36, 0xd8, 0x5b, 0xaf, 0x3a, - 0x80, 0xbd, 0xdb, 0xa1, 0x66, 0x73, 0xf7, 0x49, 0xb1, 0x62, 0x36, 0x2e, 0x2e, 0x57, 0x53, 0xbf, - 0x6a, 0xc5, 0xec, 0xf3, 0x96, 0x4c, 0x2d, 0xf6, 0xcb, 0x80, 0xea, 0x7b, 0xb6, 0x88, 0x05, 0x47, - 0x87, 0x50, 0x99, 0xd0, 0x88, 0x70, 0x65, 0xac, 0x0a, 0xd6, 0x40, 0xce, 0x1c, 0xd2, 0x44, 0x5d, - 0x8c, 0x12, 0xae, 0xa4, 0xad, 0xe0, 0x22, 0xa5, 0x0e, 0xa7, 0xcf, 0xc0, 0x95, 0xff, 0x2a, 0x38, - 0xc7, 0xc5, 0x7d, 0x4c, 0x95, 0xca, 0xf7, 0x39, 0x84, 0x8a, 0xbf, 0x14, 0x24, 0x33, 0xa6, 0x06, - 0x7f, 0x99, 0xa0, 0xfa, 0xcc, 0x04, 0x6d, 0xa8, 0xe9, 0x97, 0x37, 0x3c, 0x55, 0xe7, 0x6f, 0xe2, - 0x1c, 0x23, 0x1b, 0x0a, 0x2a, 0x5a, 0xe8, 0xb9, 0xae, 0xdd, 0x4f, 0x50, 0xd7, 0x5b, 0x8e, 0x88, - 0x40, 0x2e, 0x54, 0x03, 0x05, 0xd2, 0xd7, 0x08, 0xf2, 0x35, 0xea, 0x74, 0x6a, 0xca, 0x34, 0x2f, - 0xc7, 0x0f, 0x12, 0x22, 0x5f, 0x9d, 0x5a, 0xbc, 0x8c, 0x33, 0x38, 0x70, 0x1e, 0x7e, 0xda, 0xa5, - 0x87, 0xb5, 0x6d, 0x3c, 0xae, 0x6d, 0xe3, 0x69, 0x6d, 0x97, 0xbe, 0x6e, 0xec, 0xd2, 0xfd, 0xc6, - 0x36, 0x1e, 0x37, 0x76, 0xe9, 0xfb, 0xc6, 0x2e, 0xf9, 0x55, 0xe5, 0xbe, 0x93, 0x3f, 0x01, 0x00, - 0x00, 0xff, 0xff, 0x7e, 0x87, 0x05, 0xb2, 0xd0, 0x04, 0x00, 0x00, + // 743 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcf, 0x6f, 0xe3, 0x44, + 0x14, 0x8e, 0x37, 0x71, 0x9a, 0x3c, 0x27, 0x61, 0x77, 0x58, 0x55, 0x56, 0x24, 0x1c, 0x2b, 0x08, + 0xc9, 0xe2, 0x90, 0xb0, 0xdd, 0x1b, 0x48, 0x1c, 0xcc, 0xaa, 0x22, 0x12, 0x62, 0xd1, 0x64, 0xd5, + 0x13, 0x52, 0xe4, 0x1f, 0x93, 0x64, 0x54, 0xc7, 0x93, 0x7a, 0x26, 0xad, 0xdc, 0x1b, 0xff, 0x01, + 0x47, 0x8e, 0xfd, 0x73, 0x7a, 0xec, 0x11, 0x71, 0x88, 0x20, 0xe1, 0xc0, 0x9f, 0x81, 0x66, 0xc6, + 0x76, 0x5c, 0x2e, 0xec, 0x6d, 0xbe, 0xef, 0x3d, 0xfb, 0xbd, 0xf9, 0xbe, 0xcf, 0x86, 0x3e, 0x17, + 0xd9, 0x2e, 0x12, 0x7c, 0xb2, 0xcd, 0x98, 0x60, 0xe8, 0x45, 0x1c, 0x0e, 0x3f, 0xcf, 0xc8, 0x96, + 0xf1, 0xa9, 0x22, 0xc2, 0xdd, 0x72, 0xba, 0x62, 0x2b, 0xa6, 0x80, 0x3a, 0xe9, 0xc6, 0xe1, 0x79, + 0x42, 0x43, 0xdd, 0x12, 0xb1, 0x64, 0x1a, 0x92, 0xad, 0xe6, 0xc7, 0x37, 0x60, 0x5d, 0xd2, 0x84, + 0x5c, 0x91, 0x8c, 0x53, 0x96, 0xa2, 0xaf, 0xe0, 0xec, 0x56, 0x1f, 0x6d, 0xc3, 0x35, 0x3c, 0xeb, + 0xe2, 0xe5, 0xa4, 0x7c, 0x68, 0x72, 0x45, 0x22, 0xc1, 0x32, 0xbf, 0xf5, 0xb8, 0x1f, 0x35, 0x70, + 0xd9, 0x86, 0xce, 0xa1, 0x1d, 0x93, 0x5b, 0x1a, 0x11, 0xfb, 0x85, 0x6b, 0x78, 0x3d, 0x5c, 0x20, + 0x64, 0xc3, 0x19, 0x4d, 0x6f, 0x83, 0x84, 0xc6, 0x76, 0xd3, 0x35, 0xbc, 0x0e, 0x2e, 0xe1, 0xf8, + 0x12, 0xac, 0x62, 0xdc, 0x0f, 0x94, 0x0b, 0xf4, 0x06, 0x3a, 0xc5, 0xbb, 0xb8, 0x6d, 0xb8, 0x4d, + 0xcf, 0xba, 0xf8, 0x64, 0x12, 0x87, 0x93, 0xda, 0x56, 0xc5, 0xc8, 0xaa, 0xed, 0xeb, 0xd6, 0x6f, + 0x0f, 0xa3, 0xc6, 0xf8, 0x17, 0x13, 0x5e, 0xc9, 0xae, 0x59, 0xba, 0x64, 0x1f, 0xb2, 0x5d, 0x1a, + 0x05, 0x82, 0xc4, 0x08, 0x41, 0x2b, 0x0d, 0x36, 0x44, 0xad, 0xdf, 0xc5, 0xea, 0x2c, 0x39, 0x4e, + 0xef, 0x89, 0x5a, 0xa4, 0x89, 0xd5, 0x19, 0x7d, 0x06, 0xb0, 0x61, 0x31, 0x5d, 0x52, 0x12, 0x2f, + 0xb8, 0x6d, 0xaa, 0x4a, 0xb7, 0x64, 0xe6, 0xe8, 0x67, 0xb0, 0xaa, 0x72, 0x98, 0xdb, 0x3d, 0xd7, + 0xf0, 0x5a, 0xfe, 0x37, 0x72, 0x8f, 0x3f, 0xf6, 0xa3, 0xb7, 0x2b, 0x2a, 0xd6, 0xbb, 0x70, 0x12, + 0xb1, 0xcd, 0x94, 0xe7, 0x69, 0x24, 0xd6, 0x34, 0x5d, 0xd5, 0x4e, 0x75, 0xad, 0x27, 0xf3, 0x35, + 0xcb, 0xc4, 0xec, 0x1d, 0xae, 0xc6, 0xf9, 0x79, 0x5d, 0xe6, 0xee, 0xc7, 0xc9, 0x3c, 0x84, 0x0e, + 0x27, 0x37, 0x3b, 0x92, 0x46, 0xc4, 0x06, 0xb5, 0x6c, 0x85, 0xd1, 0x17, 0x30, 0xe0, 0xf9, 0x26, + 0xa1, 0xe9, 0xf5, 0x42, 0x04, 0xd9, 0x8a, 0x08, 0xfb, 0x95, 0xba, 0x7c, 0xbf, 0x60, 0x3f, 0x28, + 0x12, 0x8d, 0xc0, 0x0a, 0x13, 0x16, 0x5d, 0xf3, 0xc5, 0x3a, 0xe0, 0x6b, 0x1b, 0x29, 0xbb, 0x40, + 0x53, 0xdf, 0x07, 0x7c, 0x8d, 0xbe, 0x84, 0x96, 0xc8, 0xb7, 0xda, 0xc8, 0xc1, 0xc5, 0xf9, 0x69, + 0xa5, 0x4a, 0xe5, 0x7c, 0x4b, 0xb0, 0xea, 0x41, 0x2e, 0x58, 0x5b, 0x92, 0x6d, 0x28, 0xd7, 0xc6, + 0xb5, 0x5c, 0xc3, 0xeb, 0xe3, 0x3a, 0x25, 0xc7, 0x55, 0x0a, 0xa6, 0xdc, 0xb6, 0x5c, 0xc3, 0x33, + 0x4f, 0x22, 0xfc, 0xc8, 0xd1, 0x14, 0xf4, 0xf0, 0x85, 0xf2, 0xa6, 0x2f, 0xeb, 0xfe, 0xcb, 0xc3, + 0x7e, 0xd4, 0xc3, 0xc1, 0x9d, 0x2f, 0x0b, 0x73, 0x7a, 0x4f, 0x70, 0x37, 0x2c, 0x8f, 0x72, 0x66, + 0xc2, 0xa2, 0x20, 0x59, 0x2c, 0x93, 0x60, 0xc5, 0xed, 0x7f, 0xce, 0xd4, 0x50, 0x50, 0xdc, 0xa5, + 0xa4, 0x64, 0xe8, 0x62, 0x92, 0x10, 0x41, 0x62, 0xbb, 0xad, 0x43, 0x57, 0x40, 0xe4, 0x9d, 0xe2, + 0x28, 0x1f, 0xeb, 0xf8, 0x83, 0xc3, 0x7e, 0x04, 0x38, 0xb8, 0x9b, 0x69, 0xb6, 0x8a, 0xa7, 0x54, + 0x33, 0x65, 0x8b, 0xfa, 0xe5, 0x3a, 0xea, 0x55, 0xfd, 0x94, 0xfd, 0x74, 0x22, 0x8b, 0x0c, 0x7e, + 0x0b, 0x5d, 0xb5, 0x6a, 0x91, 0xe4, 0xb6, 0x02, 0x65, 0x8e, 0x3f, 0x3d, 0x29, 0xa8, 0x78, 0x29, + 0x61, 0xe1, 0x6b, 0xd1, 0x38, 0x7e, 0x03, 0x03, 0xbf, 0x32, 0xe0, 0x7d, 0x9a, 0xe4, 0xff, 0xeb, + 0xd2, 0xf8, 0x6f, 0x03, 0xda, 0xdf, 0xb1, 0x5d, 0x2a, 0x38, 0x7a, 0x0d, 0xe6, 0x92, 0x26, 0x84, + 0xab, 0xb0, 0x9b, 0x58, 0x03, 0x29, 0x53, 0x4c, 0x33, 0x95, 0x22, 0x4a, 0xb8, 0x72, 0xd3, 0xc4, + 0x75, 0x4a, 0x85, 0x49, 0x47, 0x83, 0xab, 0x6f, 0xc2, 0xc4, 0x15, 0xae, 0x4b, 0xd8, 0x52, 0xa5, + 0x4a, 0xc2, 0xd7, 0x60, 0x86, 0xb9, 0x20, 0xe5, 0xc7, 0xa2, 0xc1, 0xb3, 0x60, 0xb6, 0xff, 0x13, + 0xcc, 0x21, 0x74, 0xf4, 0xdf, 0x60, 0xf6, 0x4e, 0x45, 0xb2, 0x87, 0x2b, 0x8c, 0x1c, 0xa8, 0x19, + 0xa7, 0xae, 0xf9, 0xcc, 0xca, 0xf1, 0x7b, 0xe8, 0xea, 0x5b, 0xce, 0x89, 0x40, 0x1e, 0xb4, 0x23, + 0x05, 0x0a, 0x65, 0x41, 0xfe, 0x21, 0x74, 0xb9, 0x14, 0x54, 0xd7, 0xe5, 0xfa, 0x51, 0x46, 0xe4, + 0x9f, 0x40, 0x5d, 0xbc, 0x89, 0x4b, 0xe8, 0xbb, 0x8f, 0x7f, 0x39, 0x8d, 0xc7, 0x83, 0x63, 0x3c, + 0x1d, 0x1c, 0xe3, 0xcf, 0x83, 0xd3, 0xf8, 0xf5, 0xe8, 0x34, 0x1e, 0x8e, 0x8e, 0xf1, 0x74, 0x74, + 0x1a, 0xbf, 0x1f, 0x9d, 0x46, 0xd8, 0x56, 0x76, 0xbd, 0xfd, 0x37, 0x00, 0x00, 0xff, 0xff, 0xb8, + 0xd0, 0x05, 0xba, 0x64, 0x05, 0x00, 0x00, } func (m *FileVersion) Marshal() (dAtA []byte, err error) { @@ -408,6 +491,15 @@ func (m *FileInfoTruncated) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0xc0 } + if len(m.BlocksHash) > 0 { + i -= len(m.BlocksHash) + copy(dAtA[i:], m.BlocksHash) + i = encodeVarintStructs(dAtA, i, uint64(len(m.BlocksHash))) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x92 + } if len(m.SymlinkTarget) > 0 { i -= len(m.SymlinkTarget) copy(dAtA[i:], m.SymlinkTarget) @@ -507,6 +599,75 @@ func (m *FileInfoTruncated) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *BlockList) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *BlockList) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *BlockList) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Blocks) > 0 { + for iNdEx := len(m.Blocks) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Blocks[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintStructs(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *BlocksHashOnly) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *BlocksHashOnly) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *BlocksHashOnly) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.BlocksHash) > 0 { + i -= len(m.BlocksHash) + copy(dAtA[i:], m.BlocksHash) + i = encodeVarintStructs(dAtA, i, uint64(len(m.BlocksHash))) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x92 + } + return len(dAtA) - i, nil +} + func (m *Counts) Marshal() (dAtA []byte, err error) { size := m.ProtoSize() dAtA = make([]byte, size) @@ -711,12 +872,44 @@ func (m *FileInfoTruncated) ProtoSize() (n int) { if l > 0 { n += 2 + l + sovStructs(uint64(l)) } + l = len(m.BlocksHash) + if l > 0 { + n += 2 + l + sovStructs(uint64(l)) + } if m.LocalFlags != 0 { n += 2 + sovStructs(uint64(m.LocalFlags)) } return n } +func (m *BlockList) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Blocks) > 0 { + for _, e := range m.Blocks { + l = e.ProtoSize() + n += 1 + l + sovStructs(uint64(l)) + } + } + return n +} + +func (m *BlocksHashOnly) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.BlocksHash) + if l > 0 { + n += 2 + l + sovStructs(uint64(l)) + } + return n +} + func (m *Counts) ProtoSize() (n int) { if m == nil { return 0 @@ -1340,6 +1533,40 @@ func (m *FileInfoTruncated) Unmarshal(dAtA []byte) error { } m.SymlinkTarget = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 18: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field BlocksHash", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStructs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthStructs + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthStructs + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.BlocksHash = append(m.BlocksHash[:0], dAtA[iNdEx:postIndex]...) + if m.BlocksHash == nil { + m.BlocksHash = []byte{} + } + iNdEx = postIndex case 1000: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field LocalFlags", wireType) @@ -1383,6 +1610,180 @@ func (m *FileInfoTruncated) Unmarshal(dAtA []byte) error { } return nil } +func (m *BlockList) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStructs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: BlockList: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: BlockList: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Blocks", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStructs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthStructs + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthStructs + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Blocks = append(m.Blocks, protocol.BlockInfo{}) + if err := m.Blocks[len(m.Blocks)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipStructs(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthStructs + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthStructs + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *BlocksHashOnly) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStructs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: BlocksHashOnly: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: BlocksHashOnly: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 18: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field BlocksHash", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStructs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthStructs + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthStructs + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.BlocksHash = append(m.BlocksHash[:0], dAtA[iNdEx:postIndex]...) + if m.BlocksHash == nil { + m.BlocksHash = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipStructs(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthStructs + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthStructs + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *Counts) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/lib/db/structs.proto b/lib/db/structs.proto index c0f5337a7..1acbaaee3 100644 --- a/lib/db/structs.proto +++ b/lib/db/structs.proto @@ -34,6 +34,7 @@ message FileInfoTruncated { int64 sequence = 10; // repeated BlockInfo Blocks = 16 string symlink_target = 17; + bytes blocks_hash = 18; protocol.FileInfoType type = 2; uint32 permissions = 4; int32 modified_ns = 11; @@ -47,6 +48,16 @@ message FileInfoTruncated { bool no_permissions = 8; } +// BlockList is the structure used to store block lists +message BlockList { + repeated protocol.BlockInfo Blocks = 1 [(gogoproto.nullable) = false]; +} + +// BlocksHashOnly is used to only unmarshal the block list hash from a FileInfo +message BlocksHashOnly { + bytes blocks_hash = 18; +} + // For each folder and device we keep one of these to track the current // counts and sequence. We also keep one for the global state of the folder. message Counts { diff --git a/lib/db/transactions.go b/lib/db/transactions.go index c0fd528c0..9a8ddff2c 100644 --- a/lib/db/transactions.go +++ b/lib/db/transactions.go @@ -58,13 +58,50 @@ func (t readOnlyTransaction) getFileTrunc(key []byte, trunc bool) (FileIntf, boo if err != nil { return nil, false, err } - f, err := unmarshalTrunc(bs, trunc) + f, err := t.unmarshalTrunc(bs, trunc) if err != nil { return nil, false, err } return f, true, nil } +func (t readOnlyTransaction) unmarshalTrunc(bs []byte, trunc bool) (FileIntf, error) { + if trunc { + var tf FileInfoTruncated + err := tf.Unmarshal(bs) + if err != nil { + return nil, err + } + return tf, nil + } + + var tf protocol.FileInfo + if err := tf.Unmarshal(bs); err != nil { + return nil, err + } + if err := t.fillBlockList(&tf); err != nil { + return nil, err + } + return tf, nil +} + +func (t readOnlyTransaction) fillBlockList(fi *protocol.FileInfo) error { + if fi.BlocksHash == nil { + return nil + } + blocksKey := t.keyer.GenerateBlockListKey(nil, fi.BlocksHash) + bs, err := t.Get(blocksKey) + if err != nil { + return err + } + var bl BlockList + if err := bl.Unmarshal(bs); err != nil { + return err + } + fi.Blocks = bl.Blocks + return nil +} + func (t readOnlyTransaction) getGlobal(keyBuf, folder, file []byte, truncate bool) ([]byte, FileIntf, bool, error) { var err error keyBuf, err = t.keyer.GenerateGlobalVersionKey(keyBuf, folder, file) @@ -132,7 +169,7 @@ func (t *readOnlyTransaction) withHave(folder, device, prefix []byte, truncate b return nil } - f, err := unmarshalTrunc(dbi.Value(), truncate) + f, err := t.unmarshalTrunc(dbi.Value(), truncate) if err != nil { l.Debugln("unmarshal error:", err) continue @@ -413,6 +450,28 @@ func (t readWriteTransaction) close() { t.WriteTransaction.Release() } +func (t readWriteTransaction) putFile(key []byte, fi protocol.FileInfo) error { + if fi.Blocks != nil { + if fi.BlocksHash == nil { + fi.BlocksHash = protocol.BlocksHash(fi.Blocks) + } + blocksKey := t.keyer.GenerateBlockListKey(nil, fi.BlocksHash) + if _, err := t.Get(blocksKey); backend.IsNotFound(err) { + // Marshal the block list and save it + blocksBs := mustMarshal(&BlockList{Blocks: fi.Blocks}) + if err := t.Put(blocksKey, blocksBs); err != nil { + return err + } + } else if err != nil { + return err + } + } + + fi.Blocks = nil + fiBs := mustMarshal(&fi) + return t.Put(key, fiBs) +} + // updateGlobal adds this device+version to the version list for the given // file. If the device is already present in the list, the version is updated. // If the file does not have an entry in the global list, it is created. diff --git a/lib/model/model.go b/lib/model/model.go index 112242ea2..d4ebbafca 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -954,8 +954,8 @@ func (m *model) handleIndex(deviceID protocol.DeviceID, folder string, fs []prot files.Drop(deviceID) } for i := range fs { - // The local flags should never be transmitted over the wire. Make - // sure they look like they weren't. + // The local attributes should never be transmitted over the wire. + // Make sure they look like they weren't. fs[i].LocalFlags = 0 } files.Update(deviceID, fs) diff --git a/lib/protocol/bep.pb.go b/lib/protocol/bep.pb.go index f7e5f275f..03036b0fe 100644 --- a/lib/protocol/bep.pb.go +++ b/lib/protocol/bep.pb.go @@ -495,8 +495,9 @@ type FileInfo struct { ModifiedBy ShortID `protobuf:"varint,12,opt,name=modified_by,json=modifiedBy,proto3,customtype=ShortID" json:"modified_by"` Version Vector `protobuf:"bytes,9,opt,name=version,proto3" json:"version"` Sequence int64 `protobuf:"varint,10,opt,name=sequence,proto3" json:"sequence,omitempty"` - Blocks []BlockInfo `protobuf:"bytes,16,rep,name=Blocks,proto3" json:"Blocks"` + Blocks []BlockInfo `protobuf:"bytes,16,rep,name=blocks,proto3" json:"blocks"` SymlinkTarget string `protobuf:"bytes,17,opt,name=symlink_target,json=symlinkTarget,proto3" json:"symlink_target,omitempty"` + BlocksHash []byte `protobuf:"bytes,18,opt,name=blocks_hash,json=blocksHash,proto3" json:"blocks_hash,omitempty"` Type FileInfoType `protobuf:"varint,2,opt,name=type,proto3,enum=protocol.FileInfoType" json:"type,omitempty"` Permissions uint32 `protobuf:"varint,4,opt,name=permissions,proto3" json:"permissions,omitempty"` ModifiedNs int32 `protobuf:"varint,11,opt,name=modified_ns,json=modifiedNs,proto3" json:"modified_ns,omitempty"` @@ -920,120 +921,121 @@ func init() { func init() { proto.RegisterFile("bep.proto", fileDescriptor_e3f59eb60afbbc6e) } var fileDescriptor_e3f59eb60afbbc6e = []byte{ - // 1803 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcf, 0x6f, 0xdb, 0xc8, - 0x15, 0x16, 0x25, 0xea, 0xd7, 0x93, 0xec, 0xa5, 0x27, 0x89, 0xcb, 0x32, 0x59, 0x89, 0x51, 0x92, - 0x8d, 0xd6, 0xd8, 0x26, 0xe9, 0xee, 0xb6, 0x45, 0x8b, 0xb6, 0x80, 0x7e, 0xd0, 0x8e, 0x50, 0x47, + // 1816 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4f, 0x6f, 0xdb, 0xc8, + 0x15, 0x17, 0x25, 0xea, 0xdf, 0x93, 0xec, 0xa5, 0x27, 0x89, 0xcb, 0x32, 0x59, 0x89, 0x51, 0x92, + 0x8d, 0xd6, 0xd8, 0x26, 0xe9, 0xee, 0xb6, 0x45, 0x8b, 0xb6, 0x80, 0xfe, 0xd0, 0x8e, 0x50, 0x47, 0x72, 0x47, 0x72, 0xb6, 0xd9, 0x43, 0x09, 0x5a, 0x1c, 0xc9, 0x44, 0x28, 0x8e, 0x4a, 0x52, 0x76, - 0xb4, 0x7f, 0x82, 0x4e, 0x3d, 0xf6, 0x22, 0x60, 0x81, 0x9e, 0xfa, 0x9f, 0xe4, 0x98, 0xf6, 0x50, - 0x14, 0x3d, 0x18, 0x5d, 0xe7, 0xb2, 0xc7, 0xfe, 0x05, 0x45, 0x31, 0x33, 0xa4, 0x44, 0xd9, 0x9b, - 0x45, 0x0e, 0x7b, 0xd2, 0xcc, 0x7b, 0x1f, 0xdf, 0xcc, 0x7c, 0xf3, 0xbd, 0x6f, 0x04, 0xc5, 0x13, - 0x32, 0x7d, 0x34, 0xf5, 0x69, 0x48, 0x51, 0x81, 0xff, 0x0c, 0xa9, 0xab, 0xdd, 0xf3, 0xc9, 0x94, - 0x06, 0x8f, 0xf9, 0xfc, 0x64, 0x36, 0x7a, 0x3c, 0xa6, 0x63, 0xca, 0x27, 0x7c, 0x24, 0xe0, 0xb5, - 0x29, 0x64, 0x9f, 0x12, 0xd7, 0xa5, 0xa8, 0x0a, 0x25, 0x9b, 0x9c, 0x39, 0x43, 0x62, 0x7a, 0xd6, - 0x84, 0xa8, 0x92, 0x2e, 0xd5, 0x8b, 0x18, 0x44, 0xa8, 0x6b, 0x4d, 0x08, 0x03, 0x0c, 0x5d, 0x87, - 0x78, 0xa1, 0x00, 0xa4, 0x05, 0x40, 0x84, 0x38, 0xe0, 0x01, 0x6c, 0x47, 0x80, 0x33, 0xe2, 0x07, - 0x0e, 0xf5, 0xd4, 0x0c, 0xc7, 0x6c, 0x89, 0xe8, 0x73, 0x11, 0xac, 0x05, 0x90, 0x7b, 0x4a, 0x2c, - 0x9b, 0xf8, 0xe8, 0x63, 0x90, 0xc3, 0xf9, 0x54, 0xac, 0xb5, 0xfd, 0xe9, 0xad, 0x47, 0xf1, 0xce, - 0x1f, 0x3d, 0x23, 0x41, 0x60, 0x8d, 0xc9, 0x60, 0x3e, 0x25, 0x98, 0x43, 0xd0, 0x6f, 0xa1, 0x34, - 0xa4, 0x93, 0xa9, 0x4f, 0x02, 0x5e, 0x38, 0xcd, 0xbf, 0xb8, 0x73, 0xed, 0x8b, 0xd6, 0x1a, 0x83, - 0x93, 0x1f, 0xd4, 0x1a, 0xb0, 0xd5, 0x72, 0x67, 0x41, 0x48, 0xfc, 0x16, 0xf5, 0x46, 0xce, 0x18, - 0x3d, 0x81, 0xfc, 0x88, 0xba, 0x36, 0xf1, 0x03, 0x55, 0xd2, 0x33, 0xf5, 0xd2, 0xa7, 0xca, 0xba, - 0xd8, 0x3e, 0x4f, 0x34, 0xe5, 0xd7, 0x17, 0xd5, 0x14, 0x8e, 0x61, 0xb5, 0xbf, 0xa6, 0x21, 0x27, - 0x32, 0x68, 0x17, 0xd2, 0x8e, 0x2d, 0x28, 0x6a, 0xe6, 0x2e, 0x2f, 0xaa, 0xe9, 0x4e, 0x1b, 0xa7, - 0x1d, 0x1b, 0xdd, 0x84, 0xac, 0x6b, 0x9d, 0x10, 0x37, 0x22, 0x47, 0x4c, 0xd0, 0x6d, 0x28, 0xfa, - 0xc4, 0xb2, 0x4d, 0xea, 0xb9, 0x73, 0x4e, 0x49, 0x01, 0x17, 0x58, 0xa0, 0xe7, 0xb9, 0x73, 0xf4, - 0x13, 0x40, 0xce, 0xd8, 0xa3, 0x3e, 0x31, 0xa7, 0xc4, 0x9f, 0x38, 0x7c, 0xb7, 0x81, 0x2a, 0x73, - 0xd4, 0x8e, 0xc8, 0x1c, 0xad, 0x13, 0xe8, 0x1e, 0x6c, 0x45, 0x70, 0x9b, 0xb8, 0x24, 0x24, 0x6a, - 0x96, 0x23, 0xcb, 0x22, 0xd8, 0xe6, 0x31, 0xf4, 0x04, 0x6e, 0xda, 0x4e, 0x60, 0x9d, 0xb8, 0xc4, - 0x0c, 0xc9, 0x64, 0x6a, 0x3a, 0x9e, 0x4d, 0x5e, 0x91, 0x40, 0xcd, 0x71, 0x2c, 0x8a, 0x72, 0x03, - 0x32, 0x99, 0x76, 0x44, 0x06, 0xed, 0x42, 0x6e, 0x6a, 0xcd, 0x02, 0x62, 0xab, 0x79, 0x8e, 0x89, - 0x66, 0x8c, 0x25, 0xa1, 0x80, 0x40, 0x55, 0xae, 0xb2, 0xd4, 0xe6, 0x89, 0x98, 0xa5, 0x08, 0x56, - 0xfb, 0x6f, 0x1a, 0x72, 0x22, 0x83, 0x3e, 0x5a, 0xb1, 0x54, 0x6e, 0xee, 0x32, 0xd4, 0xbf, 0x2f, - 0xaa, 0x05, 0x91, 0xeb, 0xb4, 0x13, 0xac, 0x21, 0x90, 0x13, 0x8a, 0xe2, 0x63, 0x74, 0x07, 0x8a, - 0x96, 0x6d, 0xb3, 0xdb, 0x23, 0x81, 0x9a, 0xd1, 0x33, 0xf5, 0x22, 0x5e, 0x07, 0xd0, 0x2f, 0x36, - 0xd5, 0x20, 0x5f, 0xd5, 0xcf, 0xbb, 0x64, 0xc0, 0xae, 0x62, 0x48, 0xfc, 0x48, 0xc1, 0x59, 0xbe, - 0x5e, 0x81, 0x05, 0xb8, 0x7e, 0xef, 0x42, 0x79, 0x62, 0xbd, 0x32, 0x03, 0xf2, 0xa7, 0x19, 0xf1, - 0x86, 0x84, 0xd3, 0x95, 0xc1, 0xa5, 0x89, 0xf5, 0xaa, 0x1f, 0x85, 0x50, 0x05, 0xc0, 0xf1, 0x42, - 0x9f, 0xda, 0xb3, 0x21, 0xf1, 0x23, 0xae, 0x12, 0x11, 0xf4, 0x33, 0x28, 0x70, 0xb2, 0x4d, 0xc7, - 0x56, 0x0b, 0xba, 0x54, 0x97, 0x9b, 0x5a, 0x74, 0xf0, 0x3c, 0xa7, 0x9a, 0x9f, 0x3b, 0x1e, 0xe2, - 0x3c, 0xc7, 0x76, 0x6c, 0xf4, 0x6b, 0xd0, 0x82, 0x97, 0x0e, 0xbb, 0x28, 0x51, 0x29, 0x74, 0xa8, - 0x67, 0xfa, 0x64, 0x42, 0xcf, 0x2c, 0x37, 0x50, 0x8b, 0x7c, 0x19, 0x95, 0x21, 0x3a, 0x09, 0x00, - 0x8e, 0xf2, 0xb5, 0x1e, 0x64, 0x79, 0x45, 0x76, 0x8b, 0x42, 0xac, 0x51, 0xf7, 0x46, 0x33, 0xf4, - 0x08, 0xb2, 0x23, 0xc7, 0x25, 0x81, 0x9a, 0xe6, 0x77, 0x88, 0x12, 0x4a, 0x77, 0x5c, 0xd2, 0xf1, - 0x46, 0x34, 0xba, 0x45, 0x01, 0xab, 0x1d, 0x43, 0x89, 0x17, 0x3c, 0x9e, 0xda, 0x56, 0x48, 0x7e, - 0xb0, 0xb2, 0x17, 0x32, 0x14, 0xe2, 0xcc, 0xea, 0xd2, 0xa5, 0xc4, 0xa5, 0x23, 0x90, 0x03, 0xe7, - 0x2b, 0xc2, 0x7b, 0x24, 0x83, 0xf9, 0x18, 0x7d, 0x08, 0x30, 0xa1, 0xb6, 0x33, 0x72, 0x88, 0x6d, - 0x06, 0xfc, 0xca, 0x32, 0xb8, 0x18, 0x47, 0xfa, 0xe8, 0x09, 0x94, 0x56, 0xe9, 0x93, 0xb9, 0x5a, - 0xe6, 0x9c, 0x7f, 0x10, 0x73, 0xde, 0x3f, 0xa5, 0x7e, 0xd8, 0x69, 0xe3, 0x55, 0x89, 0xe6, 0x9c, - 0x49, 0x3a, 0xb6, 0x27, 0x46, 0xec, 0x86, 0xa4, 0x9f, 0x93, 0x61, 0x48, 0x57, 0x8d, 0x1f, 0xc1, - 0x90, 0x06, 0x85, 0x95, 0x26, 0x80, 0x6f, 0x60, 0x35, 0x47, 0x3f, 0x85, 0x5c, 0xd3, 0xa5, 0xc3, - 0x97, 0x71, 0x7f, 0xdc, 0x58, 0x17, 0xe3, 0xf1, 0x04, 0x0b, 0x11, 0x90, 0xd9, 0x64, 0x30, 0x9f, - 0xb8, 0x8e, 0xf7, 0xd2, 0x0c, 0x2d, 0x7f, 0x4c, 0x42, 0x75, 0x47, 0xd8, 0x64, 0x14, 0x1d, 0xf0, - 0x20, 0xda, 0x8b, 0xcc, 0x51, 0x58, 0xdd, 0xee, 0x75, 0x72, 0x13, 0xee, 0xa8, 0x43, 0xe9, 0xaa, - 0x7b, 0x6c, 0xe1, 0x64, 0x88, 0x99, 0xf7, 0x8a, 0x27, 0x2f, 0x50, 0x4b, 0xba, 0x54, 0xcf, 0xae, - 0x69, 0xe9, 0x06, 0xe8, 0x31, 0xc0, 0x09, 0xdb, 0x9f, 0xc9, 0x6f, 0x60, 0x8b, 0xe5, 0x9b, 0xca, - 0xe5, 0x45, 0xb5, 0x8c, 0xad, 0x73, 0xbe, 0xf1, 0xbe, 0xf3, 0x15, 0xc1, 0xc5, 0x93, 0x78, 0xc8, - 0xd6, 0x74, 0xe9, 0xd0, 0x72, 0xcd, 0x91, 0x6b, 0x8d, 0x03, 0xf5, 0xdb, 0x3c, 0x5f, 0x14, 0x78, - 0x6c, 0x9f, 0x85, 0x90, 0xca, 0xcc, 0x83, 0x19, 0x92, 0x1d, 0x39, 0x4f, 0x3c, 0x45, 0x75, 0xc8, - 0x3b, 0xde, 0x99, 0xe5, 0x3a, 0x91, 0xdf, 0x34, 0xb7, 0x2f, 0x2f, 0xaa, 0x80, 0xad, 0xf3, 0x8e, - 0x88, 0xe2, 0x38, 0xcd, 0xc8, 0xf2, 0xe8, 0x86, 0x35, 0x16, 0x78, 0xa9, 0x2d, 0x8f, 0x26, 0x6c, - 0xf1, 0x57, 0xf2, 0x5f, 0xbe, 0xae, 0xa6, 0x6a, 0x1e, 0x14, 0x57, 0xa4, 0x33, 0x31, 0x9d, 0x5a, - 0xc1, 0x29, 0x17, 0x53, 0x19, 0xf3, 0x31, 0x53, 0x32, 0x1d, 0x8d, 0x02, 0x12, 0x72, 0xd9, 0x65, - 0x70, 0x34, 0x5b, 0x09, 0x2f, 0xcd, 0x69, 0x11, 0xc2, 0xbb, 0x0d, 0xc5, 0x73, 0x62, 0xbd, 0x34, - 0x79, 0x11, 0xc1, 0x68, 0x81, 0x05, 0x9e, 0x5a, 0xc1, 0x69, 0xb4, 0xde, 0x6f, 0x20, 0x27, 0x14, - 0x83, 0x3e, 0x83, 0xc2, 0x90, 0xce, 0xbc, 0x70, 0xfd, 0x9c, 0xec, 0x24, 0xdd, 0x88, 0x67, 0x22, - 0x19, 0xac, 0x80, 0xb5, 0x7d, 0xc8, 0x47, 0x29, 0xf4, 0x60, 0x65, 0x95, 0x72, 0xf3, 0xd6, 0x15, - 0xf5, 0x6e, 0xbe, 0x2f, 0x67, 0x96, 0x3b, 0x13, 0x1b, 0x95, 0xb1, 0x98, 0xd4, 0xfe, 0x2e, 0x41, - 0x1e, 0x33, 0x41, 0x06, 0x61, 0xe2, 0x65, 0xca, 0x6e, 0xbc, 0x4c, 0xeb, 0x1e, 0x4e, 0x6f, 0xf4, - 0x70, 0xdc, 0x86, 0x99, 0x44, 0x1b, 0xae, 0x59, 0x92, 0xbf, 0x93, 0xa5, 0x6c, 0x82, 0xa5, 0x98, - 0xe5, 0x5c, 0x82, 0xe5, 0x07, 0xb0, 0x3d, 0xf2, 0xe9, 0x84, 0xbf, 0x3d, 0xd4, 0xb7, 0xfc, 0x79, - 0x64, 0x94, 0x5b, 0x2c, 0x3a, 0x88, 0x83, 0x9b, 0x04, 0x17, 0x36, 0x09, 0xae, 0x99, 0x50, 0xc0, - 0x24, 0x98, 0x52, 0x2f, 0x20, 0xef, 0x3c, 0x13, 0x02, 0xd9, 0xb6, 0x42, 0x8b, 0x9f, 0xa8, 0x8c, - 0xf9, 0x18, 0x3d, 0x04, 0x79, 0x48, 0x6d, 0x71, 0x9e, 0xed, 0x64, 0x37, 0x1a, 0xbe, 0x4f, 0xfd, - 0x16, 0xb5, 0x09, 0xe6, 0x80, 0xda, 0x14, 0x94, 0x36, 0x3d, 0xf7, 0x5c, 0x6a, 0xd9, 0x47, 0x3e, - 0x1d, 0xb3, 0x07, 0xe2, 0x9d, 0x46, 0xd7, 0x86, 0xfc, 0x8c, 0x5b, 0x61, 0x6c, 0x75, 0xf7, 0x37, - 0xbb, 0xf1, 0x6a, 0x21, 0xe1, 0x9b, 0xb1, 0x8d, 0x44, 0x9f, 0xd6, 0xfe, 0x29, 0x81, 0xf6, 0x6e, - 0x34, 0xea, 0x40, 0x49, 0x20, 0xcd, 0xc4, 0x7f, 0xa2, 0xfa, 0xfb, 0x2c, 0xc4, 0x8d, 0x00, 0x66, - 0xab, 0xf1, 0x77, 0x3e, 0xa8, 0x09, 0xdb, 0xcb, 0xbc, 0x9f, 0xed, 0x3d, 0x84, 0x2d, 0xe1, 0x08, - 0xf1, 0xdf, 0x07, 0x59, 0xcf, 0xd4, 0xb3, 0xcd, 0xb4, 0x92, 0xc2, 0xe5, 0x13, 0xd1, 0x66, 0x3c, - 0x5e, 0xcb, 0x81, 0x7c, 0xe4, 0x78, 0xe3, 0x5a, 0x15, 0xb2, 0x2d, 0x97, 0xf2, 0x0b, 0xcb, 0xf9, - 0xc4, 0x0a, 0xa8, 0x17, 0xf3, 0x28, 0x66, 0x7b, 0xff, 0x48, 0x43, 0x29, 0xf1, 0xd7, 0x0e, 0x3d, - 0x81, 0xed, 0xd6, 0xe1, 0x71, 0x7f, 0x60, 0x60, 0xb3, 0xd5, 0xeb, 0xee, 0x77, 0x0e, 0x94, 0x94, - 0x76, 0x67, 0xb1, 0xd4, 0xd5, 0xc9, 0x1a, 0xb4, 0xf9, 0xaf, 0xad, 0x0a, 0xd9, 0x4e, 0xb7, 0x6d, - 0xfc, 0x41, 0x91, 0xb4, 0x9b, 0x8b, 0xa5, 0xae, 0x24, 0x80, 0xe2, 0x09, 0xfc, 0x04, 0xca, 0x1c, - 0x60, 0x1e, 0x1f, 0xb5, 0x1b, 0x03, 0x43, 0x49, 0x6b, 0xda, 0x62, 0xa9, 0xef, 0x5e, 0xc5, 0x45, - 0x9c, 0xdf, 0x83, 0x3c, 0x36, 0x7e, 0x7f, 0x6c, 0xf4, 0x07, 0x4a, 0x46, 0xdb, 0x5d, 0x2c, 0x75, - 0x94, 0x00, 0xc6, 0x2d, 0xf5, 0x00, 0x0a, 0xd8, 0xe8, 0x1f, 0xf5, 0xba, 0x7d, 0x43, 0x91, 0xb5, - 0x1f, 0x2d, 0x96, 0xfa, 0x8d, 0x0d, 0x54, 0xa4, 0xd2, 0x9f, 0xc3, 0x4e, 0xbb, 0xf7, 0x45, 0xf7, - 0xb0, 0xd7, 0x68, 0x9b, 0x47, 0xb8, 0x77, 0x80, 0x8d, 0x7e, 0x5f, 0xc9, 0x6a, 0xd5, 0xc5, 0x52, - 0xbf, 0x9d, 0xc0, 0x5f, 0x13, 0xdd, 0x87, 0x20, 0x1f, 0x75, 0xba, 0x07, 0x4a, 0x4e, 0xbb, 0xb1, - 0x58, 0xea, 0x1f, 0x24, 0xa0, 0x8c, 0x54, 0x76, 0xe2, 0xd6, 0x61, 0xaf, 0x6f, 0x28, 0xf9, 0x6b, - 0x27, 0xe6, 0x64, 0xef, 0xfd, 0x11, 0xd0, 0xf5, 0x3f, 0xbf, 0xe8, 0x3e, 0xc8, 0xdd, 0x5e, 0xd7, - 0x50, 0x52, 0xe2, 0xfc, 0xd7, 0x11, 0x5d, 0xea, 0x11, 0x54, 0x83, 0xcc, 0xe1, 0x97, 0x9f, 0x2b, - 0x92, 0xf6, 0xe3, 0xc5, 0x52, 0xbf, 0x75, 0x1d, 0x74, 0xf8, 0xe5, 0xe7, 0x7b, 0x14, 0x4a, 0xc9, - 0xc2, 0x35, 0x28, 0x3c, 0x33, 0x06, 0x8d, 0x76, 0x63, 0xd0, 0x50, 0x52, 0x62, 0x4b, 0x71, 0xfa, - 0x19, 0x09, 0x2d, 0xde, 0x84, 0x77, 0x20, 0xdb, 0x35, 0x9e, 0x1b, 0x58, 0x91, 0xb4, 0x9d, 0xc5, - 0x52, 0xdf, 0x8a, 0x01, 0x5d, 0x72, 0x46, 0x7c, 0x54, 0x81, 0x5c, 0xe3, 0xf0, 0x8b, 0xc6, 0x8b, - 0xbe, 0x92, 0xd6, 0xd0, 0x62, 0xa9, 0x6f, 0xc7, 0xe9, 0x86, 0x7b, 0x6e, 0xcd, 0x83, 0xbd, 0xff, - 0x49, 0x50, 0x4e, 0xbe, 0x71, 0xa8, 0x02, 0xf2, 0x7e, 0xe7, 0xd0, 0x88, 0x97, 0x4b, 0xe6, 0xd8, - 0x18, 0xd5, 0xa1, 0xd8, 0xee, 0x60, 0xa3, 0x35, 0xe8, 0xe1, 0x17, 0xf1, 0x59, 0x92, 0xa0, 0xb6, - 0xe3, 0x73, 0x81, 0xcf, 0xd1, 0x2f, 0xa1, 0xdc, 0x7f, 0xf1, 0xec, 0xb0, 0xd3, 0xfd, 0x9d, 0xc9, - 0x2b, 0xa6, 0xb5, 0x87, 0x8b, 0xa5, 0x7e, 0x77, 0x03, 0x4c, 0xa6, 0x3e, 0x19, 0x5a, 0x21, 0xb1, - 0xfb, 0xe2, 0x39, 0x66, 0xc9, 0x82, 0x84, 0x5a, 0xb0, 0x13, 0x7f, 0xba, 0x5e, 0x2c, 0xa3, 0x7d, - 0xb2, 0x58, 0xea, 0x1f, 0x7d, 0xef, 0xf7, 0xab, 0xd5, 0x0b, 0x12, 0xba, 0x0f, 0xf9, 0xa8, 0x48, - 0xac, 0xa4, 0xe4, 0xa7, 0xd1, 0x07, 0x7b, 0x7f, 0x93, 0xa0, 0xb8, 0xb2, 0x2b, 0x46, 0x78, 0xb7, - 0x67, 0x1a, 0x18, 0xf7, 0x70, 0xcc, 0xc0, 0x2a, 0xd9, 0xa5, 0x7c, 0x88, 0xee, 0x42, 0xfe, 0xc0, - 0xe8, 0x1a, 0xb8, 0xd3, 0x8a, 0x1b, 0x63, 0x05, 0x39, 0x20, 0x1e, 0xf1, 0x9d, 0x21, 0xfa, 0x18, - 0xca, 0xdd, 0x9e, 0xd9, 0x3f, 0x6e, 0x3d, 0x8d, 0x8f, 0xce, 0xd7, 0x4f, 0x94, 0xea, 0xcf, 0x86, - 0xa7, 0x9c, 0xcf, 0x3d, 0xd6, 0x43, 0xcf, 0x1b, 0x87, 0x9d, 0xb6, 0x80, 0x66, 0x34, 0x75, 0xb1, - 0xd4, 0x6f, 0xae, 0xa0, 0xd1, 0x23, 0xcd, 0xb0, 0x7b, 0x36, 0x54, 0xbe, 0xdf, 0x98, 0x90, 0x0e, - 0xb9, 0xc6, 0xd1, 0x91, 0xd1, 0x6d, 0xc7, 0xbb, 0x5f, 0xe7, 0x1a, 0xd3, 0x29, 0xf1, 0x6c, 0x86, - 0xd8, 0xef, 0xe1, 0x03, 0x63, 0x10, 0x6f, 0x7e, 0x8d, 0xd8, 0xa7, 0xec, 0xbf, 0x50, 0xb3, 0xfe, - 0xfa, 0x9b, 0x4a, 0xea, 0xcd, 0x37, 0x95, 0xd4, 0xeb, 0xcb, 0x8a, 0xf4, 0xe6, 0xb2, 0x22, 0xfd, - 0xe7, 0xb2, 0x92, 0xfa, 0xf6, 0xb2, 0x22, 0xfd, 0xf9, 0x6d, 0x25, 0xf5, 0xf5, 0xdb, 0x8a, 0xf4, - 0xe6, 0x6d, 0x25, 0xf5, 0xaf, 0xb7, 0x95, 0xd4, 0x49, 0x8e, 0x9b, 0xda, 0x67, 0xff, 0x0f, 0x00, - 0x00, 0xff, 0xff, 0xd6, 0x0e, 0x8e, 0xa4, 0x11, 0x0f, 0x00, 0x00, + 0xb4, 0x1f, 0x41, 0xa7, 0x1e, 0x7b, 0x11, 0xb0, 0x40, 0x4f, 0xfd, 0x26, 0x39, 0xa6, 0x3d, 0x14, + 0x45, 0x0f, 0x46, 0xd7, 0xb9, 0xec, 0xb1, 0x9f, 0xa0, 0x2d, 0x66, 0x86, 0x94, 0x28, 0x7b, 0xb3, + 0xc8, 0xa1, 0x27, 0xce, 0xbc, 0xf7, 0x9b, 0x37, 0x33, 0xbf, 0xf7, 0xde, 0x6f, 0x08, 0xc5, 0x13, + 0x32, 0x7d, 0x34, 0xf5, 0x69, 0x48, 0x51, 0x81, 0x7f, 0x86, 0xd4, 0xd5, 0xee, 0xf9, 0x64, 0x4a, + 0x83, 0xc7, 0x7c, 0x7e, 0x32, 0x1b, 0x3d, 0x1e, 0xd3, 0x31, 0xe5, 0x13, 0x3e, 0x12, 0xf0, 0xda, + 0x14, 0xb2, 0x4f, 0x89, 0xeb, 0x52, 0x54, 0x85, 0x92, 0x4d, 0xce, 0x9c, 0x21, 0x31, 0x3d, 0x6b, + 0x42, 0x54, 0x49, 0x97, 0xea, 0x45, 0x0c, 0xc2, 0xd4, 0xb5, 0x26, 0x84, 0x01, 0x86, 0xae, 0x43, + 0xbc, 0x50, 0x00, 0xd2, 0x02, 0x20, 0x4c, 0x1c, 0xf0, 0x00, 0xb6, 0x23, 0xc0, 0x19, 0xf1, 0x03, + 0x87, 0x7a, 0x6a, 0x86, 0x63, 0xb6, 0x84, 0xf5, 0xb9, 0x30, 0xd6, 0x02, 0xc8, 0x3d, 0x25, 0x96, + 0x4d, 0x7c, 0xf4, 0x31, 0xc8, 0xe1, 0x7c, 0x2a, 0xf6, 0xda, 0xfe, 0xf4, 0xd6, 0xa3, 0xf8, 0xe4, + 0x8f, 0x9e, 0x91, 0x20, 0xb0, 0xc6, 0x64, 0x30, 0x9f, 0x12, 0xcc, 0x21, 0xe8, 0xd7, 0x50, 0x1a, + 0xd2, 0xc9, 0xd4, 0x27, 0x01, 0x0f, 0x9c, 0xe6, 0x2b, 0xee, 0x5c, 0x5b, 0xd1, 0x5a, 0x63, 0x70, + 0x72, 0x41, 0xad, 0x01, 0x5b, 0x2d, 0x77, 0x16, 0x84, 0xc4, 0x6f, 0x51, 0x6f, 0xe4, 0x8c, 0xd1, + 0x13, 0xc8, 0x8f, 0xa8, 0x6b, 0x13, 0x3f, 0x50, 0x25, 0x3d, 0x53, 0x2f, 0x7d, 0xaa, 0xac, 0x83, + 0xed, 0x73, 0x47, 0x53, 0x7e, 0x7d, 0x51, 0x4d, 0xe1, 0x18, 0x56, 0xfb, 0x73, 0x1a, 0x72, 0xc2, + 0x83, 0x76, 0x21, 0xed, 0xd8, 0x82, 0xa2, 0x66, 0xee, 0xf2, 0xa2, 0x9a, 0xee, 0xb4, 0x71, 0xda, + 0xb1, 0xd1, 0x4d, 0xc8, 0xba, 0xd6, 0x09, 0x71, 0x23, 0x72, 0xc4, 0x04, 0xdd, 0x86, 0xa2, 0x4f, + 0x2c, 0xdb, 0xa4, 0x9e, 0x3b, 0xe7, 0x94, 0x14, 0x70, 0x81, 0x19, 0x7a, 0x9e, 0x3b, 0x47, 0x3f, + 0x02, 0xe4, 0x8c, 0x3d, 0xea, 0x13, 0x73, 0x4a, 0xfc, 0x89, 0xc3, 0x4f, 0x1b, 0xa8, 0x32, 0x47, + 0xed, 0x08, 0xcf, 0xd1, 0xda, 0x81, 0xee, 0xc1, 0x56, 0x04, 0xb7, 0x89, 0x4b, 0x42, 0xa2, 0x66, + 0x39, 0xb2, 0x2c, 0x8c, 0x6d, 0x6e, 0x43, 0x4f, 0xe0, 0xa6, 0xed, 0x04, 0xd6, 0x89, 0x4b, 0xcc, + 0x90, 0x4c, 0xa6, 0xa6, 0xe3, 0xd9, 0xe4, 0x15, 0x09, 0xd4, 0x1c, 0xc7, 0xa2, 0xc8, 0x37, 0x20, + 0x93, 0x69, 0x47, 0x78, 0xd0, 0x2e, 0xe4, 0xa6, 0xd6, 0x2c, 0x20, 0xb6, 0x9a, 0xe7, 0x98, 0x68, + 0xc6, 0x58, 0x12, 0x15, 0x10, 0xa8, 0xca, 0x55, 0x96, 0xda, 0xdc, 0x11, 0xb3, 0x14, 0xc1, 0x6a, + 0xff, 0x4e, 0x43, 0x4e, 0x78, 0xd0, 0x47, 0x2b, 0x96, 0xca, 0xcd, 0x5d, 0x86, 0xfa, 0xe7, 0x45, + 0xb5, 0x20, 0x7c, 0x9d, 0x76, 0x82, 0x35, 0x04, 0x72, 0xa2, 0xa2, 0xf8, 0x18, 0xdd, 0x81, 0xa2, + 0x65, 0xdb, 0x2c, 0x7b, 0x24, 0x50, 0x33, 0x7a, 0xa6, 0x5e, 0xc4, 0x6b, 0x03, 0xfa, 0xd9, 0x66, + 0x35, 0xc8, 0x57, 0xeb, 0xe7, 0x5d, 0x65, 0xc0, 0x52, 0x31, 0x24, 0x7e, 0x54, 0xc1, 0x59, 0xbe, + 0x5f, 0x81, 0x19, 0x78, 0xfd, 0xde, 0x85, 0xf2, 0xc4, 0x7a, 0x65, 0x06, 0xe4, 0x0f, 0x33, 0xe2, + 0x0d, 0x09, 0xa7, 0x2b, 0x83, 0x4b, 0x13, 0xeb, 0x55, 0x3f, 0x32, 0xa1, 0x0a, 0x80, 0xe3, 0x85, + 0x3e, 0xb5, 0x67, 0x43, 0xe2, 0x47, 0x5c, 0x25, 0x2c, 0xe8, 0x27, 0x50, 0xe0, 0x64, 0x9b, 0x8e, + 0xad, 0x16, 0x74, 0xa9, 0x2e, 0x37, 0xb5, 0xe8, 0xe2, 0x79, 0x4e, 0x35, 0xbf, 0x77, 0x3c, 0xc4, + 0x79, 0x8e, 0xed, 0xd8, 0xe8, 0x97, 0xa0, 0x05, 0x2f, 0x1d, 0x96, 0x28, 0x11, 0x29, 0x74, 0xa8, + 0x67, 0xfa, 0x64, 0x42, 0xcf, 0x2c, 0x37, 0x50, 0x8b, 0x7c, 0x1b, 0x95, 0x21, 0x3a, 0x09, 0x00, + 0x8e, 0xfc, 0xb5, 0x1e, 0x64, 0x79, 0x44, 0x96, 0x45, 0x51, 0xac, 0x51, 0xf7, 0x46, 0x33, 0xf4, + 0x08, 0xb2, 0x23, 0xc7, 0x25, 0x81, 0x9a, 0xe6, 0x39, 0x44, 0x89, 0x4a, 0x77, 0x5c, 0xd2, 0xf1, + 0x46, 0x34, 0xca, 0xa2, 0x80, 0xd5, 0x8e, 0xa1, 0xc4, 0x03, 0x1e, 0x4f, 0x6d, 0x2b, 0x24, 0xff, + 0xb7, 0xb0, 0xff, 0x95, 0xa1, 0x10, 0x7b, 0x56, 0x49, 0x97, 0x12, 0x49, 0x47, 0x20, 0x07, 0xce, + 0x57, 0x84, 0xf7, 0x48, 0x06, 0xf3, 0x31, 0xfa, 0x10, 0x60, 0x42, 0x6d, 0x67, 0xe4, 0x10, 0xdb, + 0x0c, 0x78, 0xca, 0x32, 0xb8, 0x18, 0x5b, 0xfa, 0xe8, 0x09, 0x94, 0x56, 0xee, 0x93, 0xb9, 0x5a, + 0xe6, 0x9c, 0x7f, 0x10, 0x73, 0xde, 0x3f, 0xa5, 0x7e, 0xd8, 0x69, 0xe3, 0x55, 0x88, 0xe6, 0x9c, + 0x95, 0x74, 0x2c, 0x4f, 0x8c, 0xd8, 0x8d, 0x92, 0x7e, 0x4e, 0x86, 0x21, 0x5d, 0x35, 0x7e, 0x04, + 0x43, 0x1a, 0x14, 0x56, 0x35, 0x01, 0xfc, 0x00, 0xab, 0x39, 0xfa, 0x31, 0xe4, 0x4e, 0x5c, 0x3a, + 0x7c, 0x19, 0xf7, 0xc7, 0x8d, 0x75, 0xb0, 0x26, 0xb3, 0x27, 0x58, 0x88, 0x80, 0x4c, 0x26, 0x83, + 0xf9, 0xc4, 0x75, 0xbc, 0x97, 0x66, 0x68, 0xf9, 0x63, 0x12, 0xaa, 0x3b, 0x42, 0x26, 0x23, 0xeb, + 0x80, 0x1b, 0x99, 0xdc, 0x8a, 0x05, 0xe6, 0xa9, 0x15, 0x9c, 0xaa, 0x88, 0xb5, 0x11, 0x06, 0x61, + 0x7a, 0x6a, 0x05, 0xa7, 0x68, 0x2f, 0x52, 0x4f, 0xa1, 0x85, 0xbb, 0xd7, 0xd9, 0x4f, 0xc8, 0xa7, + 0x0e, 0xa5, 0xab, 0xf2, 0xb2, 0x85, 0x93, 0x26, 0xb6, 0xdd, 0x8a, 0x48, 0x2f, 0x50, 0x4b, 0xba, + 0x54, 0xcf, 0xae, 0x79, 0xeb, 0x06, 0xe8, 0x31, 0x88, 0xcd, 0x4d, 0x9e, 0xa2, 0x2d, 0xe6, 0x6f, + 0x2a, 0x97, 0x17, 0xd5, 0x32, 0xb6, 0xce, 0xf9, 0x55, 0xfb, 0xce, 0x57, 0x04, 0x17, 0x4f, 0xe2, + 0x21, 0xdb, 0xd3, 0xa5, 0x43, 0xcb, 0x35, 0x47, 0xae, 0x35, 0x0e, 0xd4, 0x6f, 0xf3, 0x7c, 0x53, + 0xe0, 0xb6, 0x7d, 0x66, 0x42, 0x2a, 0x53, 0x17, 0xa6, 0x58, 0x76, 0x24, 0x4d, 0xf1, 0x14, 0xd5, + 0x21, 0xef, 0x78, 0x67, 0x96, 0xeb, 0x44, 0x82, 0xd4, 0xdc, 0xbe, 0xbc, 0xa8, 0x02, 0xb6, 0xce, + 0x3b, 0xc2, 0x8a, 0x63, 0x37, 0x63, 0xd3, 0xa3, 0x1b, 0xda, 0x59, 0xe0, 0xa1, 0xb6, 0x3c, 0x9a, + 0xd0, 0xcd, 0x5f, 0xc8, 0x7f, 0xfa, 0xba, 0x9a, 0xaa, 0x79, 0x50, 0x5c, 0x65, 0x85, 0x55, 0x1b, + 0x67, 0x36, 0xc3, 0x99, 0xe5, 0x63, 0x56, 0xea, 0x74, 0x34, 0x0a, 0x48, 0xc8, 0xeb, 0x32, 0x83, + 0xa3, 0xd9, 0xaa, 0x32, 0xd3, 0x9c, 0x16, 0x51, 0x99, 0xb7, 0xa1, 0x78, 0x4e, 0xac, 0x97, 0x22, + 0x3d, 0x82, 0xd1, 0x02, 0x33, 0xb0, 0xe4, 0x44, 0xfb, 0xfd, 0x0a, 0x72, 0xa2, 0xa4, 0xd0, 0x67, + 0x50, 0x18, 0xd2, 0x99, 0x17, 0xae, 0xdf, 0x9b, 0x9d, 0xa4, 0x5c, 0x71, 0x4f, 0x54, 0x27, 0x2b, + 0x60, 0x6d, 0x1f, 0xf2, 0x91, 0x0b, 0x3d, 0x58, 0x69, 0xa9, 0xdc, 0xbc, 0x75, 0xa5, 0xbc, 0x37, + 0x1f, 0xa0, 0x33, 0xcb, 0x9d, 0x89, 0x83, 0xca, 0x58, 0x4c, 0x6a, 0x7f, 0x95, 0x20, 0x8f, 0x59, + 0xc5, 0x06, 0x61, 0xe2, 0xe9, 0xca, 0x6e, 0x3c, 0x5d, 0xeb, 0x26, 0x4f, 0x6f, 0x34, 0x79, 0xdc, + 0xa7, 0x99, 0x44, 0x9f, 0xae, 0x59, 0x92, 0xbf, 0x93, 0xa5, 0x6c, 0x82, 0xa5, 0x98, 0xe5, 0x5c, + 0x82, 0xe5, 0x07, 0xb0, 0x3d, 0xf2, 0xe9, 0x84, 0x3f, 0x4e, 0xd4, 0xb7, 0xfc, 0x79, 0xa4, 0xa4, + 0x5b, 0xcc, 0x3a, 0x88, 0x8d, 0x9b, 0x04, 0x17, 0x36, 0x09, 0xae, 0x99, 0x50, 0xc0, 0x24, 0x98, + 0x52, 0x2f, 0x20, 0xef, 0xbc, 0x13, 0x02, 0xd9, 0xb6, 0x42, 0x8b, 0xdf, 0xa8, 0x8c, 0xf9, 0x18, + 0x3d, 0x04, 0x79, 0x48, 0x6d, 0x71, 0x9f, 0xed, 0x64, 0xbb, 0x1a, 0xbe, 0x4f, 0xfd, 0x16, 0xb5, + 0x09, 0xe6, 0x80, 0xda, 0x14, 0x94, 0x36, 0x3d, 0xf7, 0x5c, 0x6a, 0xd9, 0x47, 0x3e, 0x1d, 0xb3, + 0x17, 0xe4, 0x9d, 0x4a, 0xd8, 0x86, 0xfc, 0x8c, 0x6b, 0x65, 0xac, 0x85, 0xf7, 0x37, 0xbb, 0xf1, + 0x6a, 0x20, 0x21, 0xac, 0xb1, 0xce, 0x44, 0x4b, 0x6b, 0x7f, 0x97, 0x40, 0x7b, 0x37, 0x1a, 0x75, + 0xa0, 0x24, 0x90, 0x66, 0xe2, 0xa7, 0xa9, 0xfe, 0x3e, 0x1b, 0x71, 0x21, 0x80, 0xd9, 0x6a, 0xfc, + 0x9d, 0x2f, 0x6e, 0x42, 0x17, 0x33, 0xef, 0xa7, 0x8b, 0x0f, 0x61, 0x4b, 0x28, 0x42, 0xfc, 0x7f, + 0x21, 0xeb, 0x99, 0x7a, 0xb6, 0x99, 0x56, 0x52, 0xb8, 0x7c, 0x22, 0xda, 0x8c, 0xdb, 0x6b, 0x39, + 0x90, 0x8f, 0x1c, 0x6f, 0x5c, 0xab, 0x42, 0xb6, 0xe5, 0x52, 0x9e, 0xb0, 0x9c, 0x4f, 0xac, 0x80, + 0x7a, 0x31, 0x8f, 0x62, 0xb6, 0xf7, 0xb7, 0x34, 0x94, 0x12, 0xff, 0x7e, 0xe8, 0x09, 0x6c, 0xb7, + 0x0e, 0x8f, 0xfb, 0x03, 0x03, 0x9b, 0xad, 0x5e, 0x77, 0xbf, 0x73, 0xa0, 0xa4, 0xb4, 0x3b, 0x8b, + 0xa5, 0xae, 0x4e, 0xd6, 0xa0, 0xcd, 0xdf, 0xba, 0x2a, 0x64, 0x3b, 0xdd, 0xb6, 0xf1, 0x3b, 0x45, + 0xd2, 0x6e, 0x2e, 0x96, 0xba, 0x92, 0x00, 0x8a, 0x37, 0xf2, 0x13, 0x28, 0x73, 0x80, 0x79, 0x7c, + 0xd4, 0x6e, 0x0c, 0x0c, 0x25, 0xad, 0x69, 0x8b, 0xa5, 0xbe, 0x7b, 0x15, 0x17, 0x71, 0x7e, 0x0f, + 0xf2, 0xd8, 0xf8, 0xed, 0xb1, 0xd1, 0x1f, 0x28, 0x19, 0x6d, 0x77, 0xb1, 0xd4, 0x51, 0x02, 0x18, + 0xb7, 0xd4, 0x03, 0x28, 0x60, 0xa3, 0x7f, 0xd4, 0xeb, 0xf6, 0x0d, 0x45, 0xd6, 0x7e, 0xb0, 0x58, + 0xea, 0x37, 0x36, 0x50, 0x51, 0x95, 0xfe, 0x14, 0x76, 0xda, 0xbd, 0x2f, 0xba, 0x87, 0xbd, 0x46, + 0xdb, 0x3c, 0xc2, 0xbd, 0x03, 0x6c, 0xf4, 0xfb, 0x4a, 0x56, 0xab, 0x2e, 0x96, 0xfa, 0xed, 0x04, + 0xfe, 0x5a, 0xd1, 0x7d, 0x08, 0xf2, 0x51, 0xa7, 0x7b, 0xa0, 0xe4, 0xb4, 0x1b, 0x8b, 0xa5, 0xfe, + 0x41, 0x02, 0xca, 0x48, 0x65, 0x37, 0x6e, 0x1d, 0xf6, 0xfa, 0x86, 0x92, 0xbf, 0x76, 0x63, 0x4e, + 0xf6, 0xde, 0xef, 0x01, 0x5d, 0xff, 0x3b, 0x46, 0xf7, 0x41, 0xee, 0xf6, 0xba, 0x86, 0x92, 0x12, + 0xf7, 0xbf, 0x8e, 0xe8, 0x52, 0x8f, 0xa0, 0x1a, 0x64, 0x0e, 0xbf, 0xfc, 0x5c, 0x91, 0xb4, 0x1f, + 0x2e, 0x96, 0xfa, 0xad, 0xeb, 0xa0, 0xc3, 0x2f, 0x3f, 0xdf, 0xa3, 0x50, 0x4a, 0x06, 0xae, 0x41, + 0xe1, 0x99, 0x31, 0x68, 0xb4, 0x1b, 0x83, 0x86, 0x92, 0x12, 0x47, 0x8a, 0xdd, 0xcf, 0x48, 0x68, + 0xf1, 0x26, 0xbc, 0x03, 0xd9, 0xae, 0xf1, 0xdc, 0xc0, 0x8a, 0xa4, 0xed, 0x2c, 0x96, 0xfa, 0x56, + 0x0c, 0xe8, 0x92, 0x33, 0xe2, 0xa3, 0x0a, 0xe4, 0x1a, 0x87, 0x5f, 0x34, 0x5e, 0xf4, 0x95, 0xb4, + 0x86, 0x16, 0x4b, 0x7d, 0x3b, 0x76, 0x37, 0xdc, 0x73, 0x6b, 0x1e, 0xec, 0xfd, 0x47, 0x82, 0x72, + 0xf2, 0x8d, 0x43, 0x15, 0x90, 0xf7, 0x3b, 0x87, 0x46, 0xbc, 0x5d, 0xd2, 0xc7, 0xc6, 0xa8, 0x0e, + 0xc5, 0x76, 0x07, 0x1b, 0xad, 0x41, 0x0f, 0xbf, 0x88, 0xef, 0x92, 0x04, 0xb5, 0x1d, 0x9f, 0x17, + 0xf8, 0x1c, 0xfd, 0x1c, 0xca, 0xfd, 0x17, 0xcf, 0x0e, 0x3b, 0xdd, 0xdf, 0x98, 0x3c, 0x62, 0x5a, + 0x7b, 0xb8, 0x58, 0xea, 0x77, 0x37, 0xc0, 0x64, 0xea, 0x93, 0xa1, 0x15, 0x12, 0xbb, 0x2f, 0xde, + 0x6b, 0xe6, 0x2c, 0x48, 0xa8, 0x05, 0x3b, 0xf1, 0xd2, 0xf5, 0x66, 0x19, 0xed, 0x93, 0xc5, 0x52, + 0xff, 0xe8, 0x7b, 0xd7, 0xaf, 0x76, 0x2f, 0x48, 0xe8, 0x3e, 0xe4, 0xa3, 0x20, 0x71, 0x25, 0x25, + 0x97, 0x46, 0x0b, 0xf6, 0xfe, 0x22, 0x41, 0x71, 0x25, 0x57, 0x8c, 0xf0, 0x6e, 0xcf, 0x34, 0x30, + 0xee, 0xe1, 0x98, 0x81, 0x95, 0xb3, 0x4b, 0xf9, 0x10, 0xdd, 0x85, 0xfc, 0x81, 0xd1, 0x35, 0x70, + 0xa7, 0x15, 0x37, 0xc6, 0x0a, 0x72, 0x40, 0x3c, 0xe2, 0x3b, 0x43, 0xf4, 0x31, 0x94, 0xbb, 0x3d, + 0xb3, 0x7f, 0xdc, 0x7a, 0x1a, 0x5f, 0x9d, 0xef, 0x9f, 0x08, 0xd5, 0x9f, 0x0d, 0x4f, 0x39, 0x9f, + 0x7b, 0xac, 0x87, 0x9e, 0x37, 0x0e, 0x3b, 0x6d, 0x01, 0xcd, 0x68, 0xea, 0x62, 0xa9, 0xdf, 0x5c, + 0x41, 0xa3, 0x47, 0x9a, 0x61, 0xf7, 0x6c, 0xa8, 0x7c, 0xbf, 0x30, 0x21, 0x1d, 0x72, 0x8d, 0xa3, + 0x23, 0xa3, 0xdb, 0x8e, 0x4f, 0xbf, 0xf6, 0x35, 0xa6, 0x53, 0xe2, 0xd9, 0x0c, 0xb1, 0xdf, 0xc3, + 0x07, 0xc6, 0x20, 0x3e, 0xfc, 0x1a, 0xb1, 0x4f, 0xd9, 0xcf, 0x52, 0xb3, 0xfe, 0xfa, 0x9b, 0x4a, + 0xea, 0xcd, 0x37, 0x95, 0xd4, 0xeb, 0xcb, 0x8a, 0xf4, 0xe6, 0xb2, 0x22, 0xfd, 0xeb, 0xb2, 0x92, + 0xfa, 0xf6, 0xb2, 0x22, 0xfd, 0xf1, 0x6d, 0x25, 0xf5, 0xf5, 0xdb, 0x8a, 0xf4, 0xe6, 0x6d, 0x25, + 0xf5, 0x8f, 0xb7, 0x95, 0xd4, 0x49, 0x8e, 0x8b, 0xda, 0x67, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, + 0x39, 0xd2, 0xc3, 0x6e, 0x32, 0x0f, 0x00, 0x00, } func (m *Hello) Marshal() (dAtA []byte, err error) { @@ -1459,6 +1461,15 @@ func (m *FileInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0xc0 } + if len(m.BlocksHash) > 0 { + i -= len(m.BlocksHash) + copy(dAtA[i:], m.BlocksHash) + i = encodeVarintBep(dAtA, i, uint64(len(m.BlocksHash))) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x92 + } if len(m.SymlinkTarget) > 0 { i -= len(m.SymlinkTarget) copy(dAtA[i:], m.SymlinkTarget) @@ -2185,6 +2196,10 @@ func (m *FileInfo) ProtoSize() (n int) { if l > 0 { n += 2 + l + sovBep(uint64(l)) } + l = len(m.BlocksHash) + if l > 0 { + n += 2 + l + sovBep(uint64(l)) + } if m.LocalFlags != 0 { n += 2 + sovBep(uint64(m.LocalFlags)) } @@ -3835,6 +3850,40 @@ func (m *FileInfo) Unmarshal(dAtA []byte) error { } m.SymlinkTarget = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 18: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field BlocksHash", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBep + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthBep + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBep + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.BlocksHash = append(m.BlocksHash[:0], dAtA[iNdEx:postIndex]...) + if m.BlocksHash == nil { + m.BlocksHash = []byte{} + } + iNdEx = postIndex case 1000: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field LocalFlags", wireType) diff --git a/lib/protocol/bep.proto b/lib/protocol/bep.proto index 2450d6589..0e848006e 100644 --- a/lib/protocol/bep.proto +++ b/lib/protocol/bep.proto @@ -106,8 +106,9 @@ message FileInfo { uint64 modified_by = 12 [(gogoproto.customtype) = "ShortID", (gogoproto.nullable) = false]; Vector version = 9 [(gogoproto.nullable) = false]; int64 sequence = 10; - repeated BlockInfo Blocks = 16 [(gogoproto.nullable) = false]; + repeated BlockInfo blocks = 16 [(gogoproto.nullable) = false]; string symlink_target = 17; + bytes blocks_hash = 18; FileInfoType type = 2; uint32 permissions = 4; int32 modified_ns = 11; diff --git a/lib/protocol/bep_extensions.go b/lib/protocol/bep_extensions.go index bb2298014..9a11a07bb 100644 --- a/lib/protocol/bep_extensions.go +++ b/lib/protocol/bep_extensions.go @@ -14,6 +14,7 @@ import ( "time" "github.com/syncthing/syncthing/lib/rand" + "github.com/syncthing/syncthing/lib/sha256" ) const ( @@ -328,3 +329,11 @@ func (f Folder) Description() string { } return fmt.Sprintf("%q (%s)", f.Label, f.ID) } + +func BlocksHash(bs []BlockInfo) []byte { + h := sha256.New() + for _, b := range bs { + _, _ = h.Write(b.Hash) + } + return h.Sum(nil) +} diff --git a/lib/scanner/blockqueue.go b/lib/scanner/blockqueue.go index e17078aba..6c22714c2 100644 --- a/lib/scanner/blockqueue.go +++ b/lib/scanner/blockqueue.go @@ -111,6 +111,7 @@ func (ph *parallelHasher) hashFiles(ctx context.Context) { } f.Blocks = blocks + f.BlocksHash = protocol.BlocksHash(blocks) // The size we saw when initially deciding to hash the file // might not have been the size it actually had when we hashed