From 8fc2dfad0c7077acad9067884be40560de065f41 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Fri, 24 Jan 2020 08:35:44 +0100 Subject: [PATCH] lib/db: Deduplicate block lists in database (fixes #5898) (#6283) * lib/db: Deduplicate block lists in database (fixes #5898) This moves the block list in the database out from being just a field on the FileInfo to being an object of its own. When putting a FileInfo we marshal the block list separately and store it keyed by the sha256 of the marshalled block list. When getting, if we are not doing a "truncated" get, we do an extra read and unmarshal for the block list. Old block lists are cleared out by a periodic GC sweep. The alternative would be to use refcounting, but: - There is a larger risk of getting that wrong and either dropping a block list in error or keeping them around forever. - It's tricky with our current database, as we don't have dirty reads. This means that if we update two FileInfos with identical block lists in the same transaction we can't just do read/modify/write for the ref counters as we wouldn't see our own first update. See above about tracking this and risks about getting it wrong. GC uses a bloom filter for keys to avoid heavy RAM usage. GC can't run concurrently with FileInfo updates so there is a new lock around those operation at the lowlevel. The end result is a much more compact database, especially for setups with many peers where files get duplicated many times. This is per-key-class stats for a large database I'm currently working with, under the current schema: ``` 0x00: 9138161 items, 870876 KB keys + 7397482 KB data, 95 B + 809 B avg, 1637651 B max 0x01: 185656 items, 10388 KB keys + 1790909 KB data, 55 B + 9646 B avg, 924525 B max 0x02: 916890 items, 84795 KB keys + 3667 KB data, 92 B + 4 B avg, 192 B max 0x03: 384 items, 27 KB keys + 5 KB data, 72 B + 15 B avg, 87 B max 0x04: 1109 items, 17 KB keys + 17 KB data, 15 B + 15 B avg, 69 B max 0x06: 383 items, 3 KB keys + 0 KB data, 9 B + 2 B avg, 18 B max 0x07: 510 items, 4 KB keys + 12 KB data, 9 B + 24 B avg, 41 B max 0x08: 1349 items, 12 KB keys + 10 KB data, 9 B + 8 B avg, 17 B max 0x09: 194 items, 0 KB keys + 123 KB data, 5 B + 634 B avg, 11484 B max 0x0a: 3 items, 0 KB keys + 0 KB data, 14 B + 7 B avg, 30 B max 0x0b: 181836 items, 2363 KB keys + 10694 KB data, 13 B + 58 B avg, 173 B max Total 10426475 items, 968490 KB keys + 9202925 KB data. ``` Note 7.4 GB of data in class 00, total size 9.2 GB. After running the migration we get this instead: ``` 0x00: 9138161 items, 870876 KB keys + 2611392 KB data, 95 B + 285 B avg, 4788 B max 0x01: 185656 items, 10388 KB keys + 1790909 KB data, 55 B + 9646 B avg, 924525 B max 0x02: 916890 items, 84795 KB keys + 3667 KB data, 92 B + 4 B avg, 192 B max 0x03: 384 items, 27 KB keys + 5 KB data, 72 B + 15 B avg, 87 B max 0x04: 1109 items, 17 KB keys + 17 KB data, 15 B + 15 B avg, 69 B max 0x06: 383 items, 3 KB keys + 0 KB data, 9 B + 2 B avg, 18 B max 0x07: 510 items, 4 KB keys + 12 KB data, 9 B + 24 B avg, 41 B max 0x09: 194 items, 0 KB keys + 123 KB data, 5 B + 634 B avg, 11484 B max 0x0a: 3 items, 0 KB keys + 0 KB data, 14 B + 17 B avg, 51 B max 0x0b: 181836 items, 2363 KB keys + 10694 KB data, 13 B + 58 B avg, 173 B max 0x0d: 44282 items, 1461 KB keys + 61081 KB data, 33 B + 1379 B avg, 1637399 B max Total 10469408 items, 969939 KB keys + 4477905 KB data. ``` Class 00 is now down to 2.6 GB, with just 61 MB added in class 0d. There will be some additional reads in some cases which theoretically hurts performance, but this will be more than compensated for by smaller writes and better compaction. On my own home setup which just has three devices and a handful of folders the difference is smaller in absolute numbers of course, but still less than half the old size: ``` 0x00: 297122 items, 20894 KB keys + 306860 KB data, 70 B + 1032 B avg, 103237 B max 0x01: 115299 items, 7738 KB keys + 17542 KB data, 67 B + 152 B avg, 419 B max 0x02: 1430537 items, 121223 KB keys + 5722 KB data, 84 B + 4 B avg, 253 B max ... Total 1947412 items, 151268 KB keys + 337485 KB data. ``` to: ``` 0x00: 297122 items, 20894 KB keys + 37038 KB data, 70 B + 124 B avg, 520 B max 0x01: 115299 items, 7738 KB keys + 17542 KB data, 67 B + 152 B avg, 419 B max 0x02: 1430537 items, 121223 KB keys + 5722 KB data, 84 B + 4 B avg, 253 B max ... 0x0d: 18041 items, 595 KB keys + 71964 KB data, 33 B + 3988 B avg, 101109 B max Total 1965447 items, 151863 KB keys + 139628 KB data. ``` * wip * wip * wip * wip --- cmd/stindex/accounting.go | 58 ++++ cmd/stindex/main.go | 11 +- go.mod | 3 + go.sum | 10 + lib/db/backend/backend.go | 1 + lib/db/backend/leveldb_backend.go | 4 + lib/db/db_test.go | 6 +- lib/db/keyer.go | 19 ++ lib/db/lowlevel.go | 147 ++++++++- lib/db/schemaupdater.go | 104 ++++--- lib/db/structs.pb.go | 493 +++++++++++++++++++++++++++--- lib/db/structs.proto | 11 + lib/db/transactions.go | 63 +++- lib/model/model.go | 4 +- lib/protocol/bep.pb.go | 277 ++++++++++------- lib/protocol/bep.proto | 3 +- lib/protocol/bep_extensions.go | 9 + lib/scanner/blockqueue.go | 1 + 18 files changed, 1002 insertions(+), 222 deletions(-) create mode 100644 cmd/stindex/accounting.go diff --git a/cmd/stindex/accounting.go b/cmd/stindex/accounting.go new file mode 100644 index 000000000..fbe73fde9 --- /dev/null +++ b/cmd/stindex/accounting.go @@ -0,0 +1,58 @@ +// Copyright (C) 2020 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package main + +import ( + "fmt" + "log" + "os" + "text/tabwriter" + + "github.com/syncthing/syncthing/lib/db/backend" +) + +// account prints key and data size statistics per class +func account(ldb backend.Backend) { + it, err := ldb.NewPrefixIterator(nil) + if err != nil { + log.Fatal(err) + } + + var ksizes [256]int + var dsizes [256]int + var counts [256]int + var max [256]int + + for it.Next() { + key := it.Key() + t := key[0] + ds := len(it.Value()) + ks := len(key) + s := ks + ds + + counts[t]++ + ksizes[t] += ks + dsizes[t] += ds + if s > max[t] { + max[t] = s + } + } + + tw := tabwriter.NewWriter(os.Stdout, 1, 1, 1, ' ', tabwriter.AlignRight) + toti, totds, totks := 0, 0, 0 + for t := range ksizes { + if ksizes[t] > 0 { + // yes metric kilobytes 🤘 + fmt.Fprintf(tw, "0x%02x:\t%d items,\t%d KB keys +\t%d KB data,\t%d B +\t%d B avg,\t%d B max\t\n", t, counts[t], ksizes[t]/1000, dsizes[t]/1000, ksizes[t]/counts[t], dsizes[t]/counts[t], max[t]) + toti += counts[t] + totds += dsizes[t] + totks += ksizes[t] + } + } + fmt.Fprintf(tw, "Total\t%d items,\t%d KB keys +\t%d KB data.\t\n", toti, totks/1000, totds/1000) + tw.Flush() +} diff --git a/cmd/stindex/main.go b/cmd/stindex/main.go index ed48c35d0..e5cd54ee2 100644 --- a/cmd/stindex/main.go +++ b/cmd/stindex/main.go @@ -35,15 +35,18 @@ func main() { log.Fatal(err) } - if mode == "dump" { + switch mode { + case "dump": dump(ldb) - } else if mode == "dumpsize" { + case "dumpsize": dumpsize(ldb) - } else if mode == "idxck" { + case "idxck": if !idxck(ldb) { os.Exit(1) } - } else { + case "account": + account(ldb) + default: fmt.Println("Unknown mode") } } diff --git a/go.mod b/go.mod index e07ae08d3..e9382ac0b 100644 --- a/go.mod +++ b/go.mod @@ -35,11 +35,14 @@ require ( github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 github.com/sasha-s/go-deadlock v0.2.0 github.com/shirou/gopsutil v0.0.0-20190714054239-47ef3260b6bf + github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/syncthing/notify v0.0.0-20190709140112-69c7a957d3e2 github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d github.com/thejerf/suture v3.0.2+incompatible github.com/urfave/cli v1.22.2 github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0 + github.com/willf/bitset v1.1.10 // indirect + github.com/willf/bloom v2.0.3+incompatible golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 golang.org/x/text v0.3.2 diff --git a/go.sum b/go.sum index 16ad30245..2d11c4eb8 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,10 @@ github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrU github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -178,7 +180,10 @@ github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= @@ -199,6 +204,10 @@ github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0 h1:okhMind4q9H1OxF44gNegWkiP4H/gsTFLalHFa4OOUI= github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0/go.mod h1:TTbGUfE+cXXceWtbTHq6lqcTvYPBKLNejBEbnUsQJtU= +github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc= +github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/willf/bloom v2.0.3+incompatible h1:QDacWdqcAUI1MPOwIQZRy9kOR7yxfyEmxX8Wdm2/JPA= +github.com/willf/bloom v2.0.3+incompatible/go.mod h1:MmAltL9pDMNTrvUkxdg0k0q5I0suxmuwp3KbyrZLOZ8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= @@ -246,6 +255,7 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= diff --git a/lib/db/backend/backend.go b/lib/db/backend/backend.go index e5f6251b7..682e64dd7 100644 --- a/lib/db/backend/backend.go +++ b/lib/db/backend/backend.go @@ -100,6 +100,7 @@ type Backend interface { NewReadTransaction() (ReadTransaction, error) NewWriteTransaction() (WriteTransaction, error) Close() error + Compact() error } type Tuning int diff --git a/lib/db/backend/leveldb_backend.go b/lib/db/backend/leveldb_backend.go index 7ac5c88a0..b714accda 100644 --- a/lib/db/backend/leveldb_backend.go +++ b/lib/db/backend/leveldb_backend.go @@ -81,6 +81,10 @@ func (b *leveldbBackend) Delete(key []byte) error { return wrapLeveldbErr(b.ldb.Delete(key, nil)) } +func (b *leveldbBackend) Compact() error { + return wrapLeveldbErr(b.ldb.CompactRange(util.Range{})) +} + // leveldbSnapshot implements backend.ReadTransaction type leveldbSnapshot struct { snap *leveldb.Snapshot diff --git a/lib/db/db_test.go b/lib/db/db_test.go index ed7b0310e..37c929883 100644 --- a/lib/db/db_test.go +++ b/lib/db/db_test.go @@ -165,7 +165,7 @@ func TestUpdate0to3(t *testing.T) { folder := []byte(update0to3Folder) - if err := updater.updateSchema0to1(); err != nil { + if err := updater.updateSchema0to1(0); err != nil { t.Fatal(err) } @@ -187,7 +187,7 @@ func TestUpdate0to3(t *testing.T) { t.Error("Invalid file wasn't added to global list") } - if err := updater.updateSchema1to2(); err != nil { + if err := updater.updateSchema1to2(1); err != nil { t.Fatal(err) } @@ -214,7 +214,7 @@ func TestUpdate0to3(t *testing.T) { t.Error("Local file wasn't added to sequence bucket", err) } - if err := updater.updateSchema2to3(); err != nil { + if err := updater.updateSchema2to3(2); err != nil { t.Fatal(err) } diff --git a/lib/db/keyer.go b/lib/db/keyer.go index 1f88d5d65..d9e7811a8 100644 --- a/lib/db/keyer.go +++ b/lib/db/keyer.go @@ -59,6 +59,9 @@ const ( // KeyTypeNeed = KeyTypeNeed = 12 + + // KeyTypeBlockList = BlockList + KeyTypeBlockList = 13 ) type keyer interface { @@ -93,6 +96,9 @@ type keyer interface { // Folder metadata GenerateFolderMetaKey(key, folder []byte) (folderMetaKey, error) + + // Block lists + GenerateBlockListKey(key []byte, hash []byte) blockListKey } // defaultKeyer implements our key scheme. It needs folder and device @@ -281,6 +287,19 @@ func (k defaultKeyer) GenerateFolderMetaKey(key, folder []byte) (folderMetaKey, return key, nil } +type blockListKey []byte + +func (k defaultKeyer) GenerateBlockListKey(key []byte, hash []byte) blockListKey { + key = resize(key, keyPrefixLen+len(hash)) + key[0] = KeyTypeBlockList + copy(key[keyPrefixLen:], hash) + return key +} + +func (k blockListKey) BlocksHash() []byte { + return k[keyPrefixLen:] +} + // resize returns a byte slice of the specified size, reusing bs if possible func resize(bs []byte, size int) []byte { if cap(bs) < size { diff --git a/lib/db/lowlevel.go b/lib/db/lowlevel.go index c08848d61..ef3290bfa 100644 --- a/lib/db/lowlevel.go +++ b/lib/db/lowlevel.go @@ -9,9 +9,23 @@ package db import ( "bytes" "encoding/binary" + "time" "github.com/syncthing/syncthing/lib/db/backend" "github.com/syncthing/syncthing/lib/protocol" + "github.com/syncthing/syncthing/lib/sync" + "github.com/willf/bloom" +) + +const ( + // We set the bloom filter capacity to handle 100k individual block lists + // with a false positive probability of 1% for the first pass. Once we know + // how many block lists we have we will use that number instead, if it's + // more than 100k. For fewer than 100k block lists we will just get better + // false positive rate instead. + blockGCBloomCapacity = 100000 + blockGCBloomFalsePositiveRate = 0.01 // 1% + blockGCInterval = 13 * time.Hour ) // Lowlevel is the lowest level database interface. It has a very simple @@ -21,9 +35,12 @@ import ( // any given backend. type Lowlevel struct { backend.Backend - folderIdx *smallIndex - deviceIdx *smallIndex - keyer keyer + folderIdx *smallIndex + deviceIdx *smallIndex + keyer keyer + gcMut sync.RWMutex + gcKeyCount int + gcStop chan struct{} } func NewLowlevel(backend backend.Backend) *Lowlevel { @@ -31,11 +48,19 @@ func NewLowlevel(backend backend.Backend) *Lowlevel { Backend: backend, folderIdx: newSmallIndex(backend, []byte{KeyTypeFolderIdx}), deviceIdx: newSmallIndex(backend, []byte{KeyTypeDeviceIdx}), + gcMut: sync.NewRWMutex(), + gcStop: make(chan struct{}), } db.keyer = newDefaultKeyer(db.folderIdx, db.deviceIdx) + go db.gcRunner() return db } +func (db *Lowlevel) Close() error { + close(db.gcStop) + return db.Backend.Close() +} + // ListFolders returns the list of folders currently in the database func (db *Lowlevel) ListFolders() []string { return db.folderIdx.Values() @@ -44,6 +69,9 @@ func (db *Lowlevel) ListFolders() []string { // updateRemoteFiles adds a list of fileinfos to the database and updates the // global versionlist and metadata. func (db *Lowlevel) updateRemoteFiles(folder, device []byte, fs []protocol.FileInfo, meta *metadataTracker) error { + db.gcMut.RLock() + defer db.gcMut.RUnlock() + t, err := db.newReadWriteTransaction() if err != nil { return err @@ -73,7 +101,7 @@ func (db *Lowlevel) updateRemoteFiles(folder, device []byte, fs []protocol.FileI meta.addFile(devID, f) l.Debugf("insert; folder=%q device=%v %v", folder, devID, f) - if err := t.Put(dk, mustMarshal(&f)); err != nil { + if err := t.putFile(dk, f); err != nil { return err } @@ -97,6 +125,9 @@ func (db *Lowlevel) updateRemoteFiles(folder, device []byte, fs []protocol.FileI // updateLocalFiles adds fileinfos to the db, and updates the global versionlist, // metadata, sequence and blockmap buckets. func (db *Lowlevel) updateLocalFiles(folder []byte, fs []protocol.FileInfo, meta *metadataTracker) error { + db.gcMut.RLock() + defer db.gcMut.RUnlock() + t, err := db.newReadWriteTransaction() if err != nil { return err @@ -151,7 +182,7 @@ func (db *Lowlevel) updateLocalFiles(folder []byte, fs []protocol.FileInfo, meta meta.addFile(protocol.LocalDeviceID, f) l.Debugf("insert (local); folder=%q %v", folder, f) - if err := t.Put(dk, mustMarshal(&f)); err != nil { + if err := t.putFile(dk, f); err != nil { return err } @@ -195,6 +226,9 @@ func (db *Lowlevel) updateLocalFiles(folder []byte, fs []protocol.FileInfo, meta } func (db *Lowlevel) dropFolder(folder []byte) error { + db.gcMut.RLock() + defer db.gcMut.RUnlock() + t, err := db.newReadWriteTransaction() if err != nil { return err @@ -250,6 +284,9 @@ func (db *Lowlevel) dropFolder(folder []byte) error { } func (db *Lowlevel) dropDeviceFolder(device, folder []byte, meta *metadataTracker) error { + db.gcMut.RLock() + defer db.gcMut.RUnlock() + t, err := db.newReadWriteTransaction() if err != nil { return err @@ -425,16 +462,100 @@ func (db *Lowlevel) dropPrefix(prefix []byte) error { return t.commit() } -func unmarshalTrunc(bs []byte, truncate bool) (FileIntf, error) { - if truncate { - var tf FileInfoTruncated - err := tf.Unmarshal(bs) - return tf, err +func (db *Lowlevel) gcRunner() { + t := time.NewTicker(blockGCInterval) + defer t.Stop() + for { + select { + case <-db.gcStop: + return + case <-t.C: + if err := db.gcBlocks(); err != nil { + l.Warnln("Database block GC failed:", err) + } + } + } +} + +func (db *Lowlevel) gcBlocks() error { + // The block GC uses a bloom filter to track used block lists. This means + // iterating over all items, adding their block lists to the filter, then + // iterating over the block lists and removing those that don't match the + // filter. The filter will give false positives so we will keep around one + // percent of block lists that we don't really need (at most). + // + // Block GC needs to run when there are no modifications to the FileInfos or + // block lists. + + db.gcMut.Lock() + defer db.gcMut.Unlock() + + t, err := db.newReadWriteTransaction() + if err != nil { + return err + } + defer t.Release() + + // Set up the bloom filter with the initial capacity and false positive + // rate, or higher capacity if we've done this before and seen lots of block + // lists. + + capacity := blockGCBloomCapacity + if db.gcKeyCount > capacity { + capacity = db.gcKeyCount + } + filter := bloom.NewWithEstimates(uint(capacity), blockGCBloomFalsePositiveRate) + + // Iterate the FileInfos, unmarshal the blocks hashes and add them to + // the filter. + + it, err := db.NewPrefixIterator([]byte{KeyTypeDevice}) + if err != nil { + return err + } + for it.Next() { + var bl BlocksHashOnly + if err := bl.Unmarshal(it.Value()); err != nil { + return err + } + filter.Add(bl.BlocksHash) + } + it.Release() + if err := it.Error(); err != nil { + return err } - var tf protocol.FileInfo - err := tf.Unmarshal(bs) - return tf, err + // Iterate over block lists, removing keys with hashes that don't match + // the filter. + + it, err = db.NewPrefixIterator([]byte{KeyTypeBlockList}) + if err != nil { + return err + } + matched := 0 + for it.Next() { + key := blockListKey(it.Key()) + if filter.Test(key.BlocksHash()) { + matched++ + continue + } + if err := t.Delete(key); err != nil { + return err + } + } + it.Release() + if err := it.Error(); err != nil { + return err + } + + // Remember the number of unique keys we kept until the next pass. + db.gcKeyCount = matched + + if err := t.Commit(); err != nil { + return err + } + + return db.Compact() } func unmarshalVersionList(data []byte) (VersionList, bool) { diff --git a/lib/db/schemaupdater.go b/lib/db/schemaupdater.go index 7f6c8b1b3..803394f31 100644 --- a/lib/db/schemaupdater.go +++ b/lib/db/schemaupdater.go @@ -22,9 +22,10 @@ import ( // 5: v0.14.49 // 6: v0.14.50 // 7: v0.14.53 +// 8: v1.4.0 const ( - dbVersion = 7 - dbMinSyncthingVersion = "v0.14.53" + dbVersion = 8 + dbMinSyncthingVersion = "v1.4.0" ) type databaseDowngradeError struct { @@ -68,35 +69,26 @@ func (db *schemaUpdater) updateSchema() error { return nil } - if prevVersion < 1 { - if err := db.updateSchema0to1(); err != nil { - return err - } + type migration struct { + schemaVersion int64 + migration func(prevVersion int) error } - if prevVersion < 2 { - if err := db.updateSchema1to2(); err != nil { - return err - } + var migrations = []migration{ + {1, db.updateSchema0to1}, + {2, db.updateSchema1to2}, + {3, db.updateSchema2to3}, + {5, db.updateSchemaTo5}, + {6, db.updateSchema5to6}, + {7, db.updateSchema6to7}, + {8, db.updateSchema7to8}, } - if prevVersion < 3 { - if err := db.updateSchema2to3(); err != nil { - return err - } - } - // This update fixes problems existing in versions 3 and 4 - if prevVersion == 3 || prevVersion == 4 { - if err := db.updateSchemaTo5(); err != nil { - return err - } - } - if prevVersion < 6 { - if err := db.updateSchema5to6(); err != nil { - return err - } - } - if prevVersion < 7 { - if err := db.updateSchema6to7(); err != nil { - return err + + for _, m := range migrations { + if prevVersion < m.schemaVersion { + l.Infof("Migrating database to schema version %d...", m.schemaVersion) + if err := m.migration(int(prevVersion)); err != nil { + return err + } } } @@ -107,10 +99,11 @@ func (db *schemaUpdater) updateSchema() error { return err } - return nil + l.Infoln("Compacting database after migration...") + return db.Compact() } -func (db *schemaUpdater) updateSchema0to1() error { +func (db *schemaUpdater) updateSchema0to1(_ int) error { t, err := db.newReadWriteTransaction() if err != nil { return err @@ -216,7 +209,7 @@ func (db *schemaUpdater) updateSchema0to1() error { // updateSchema1to2 introduces a sequenceKey->deviceKey bucket for local items // to allow iteration in sequence order (simplifies sending indexes). -func (db *schemaUpdater) updateSchema1to2() error { +func (db *schemaUpdater) updateSchema1to2(_ int) error { t, err := db.newReadWriteTransaction() if err != nil { return err @@ -251,7 +244,7 @@ func (db *schemaUpdater) updateSchema1to2() error { } // updateSchema2to3 introduces a needKey->nil bucket for locally needed files. -func (db *schemaUpdater) updateSchema2to3() error { +func (db *schemaUpdater) updateSchema2to3(_ int) error { t, err := db.newReadWriteTransaction() if err != nil { return err @@ -302,7 +295,11 @@ func (db *schemaUpdater) updateSchema2to3() error { // release candidates (dbVersion 3 and 4) // https://github.com/syncthing/syncthing/issues/5007 // https://github.com/syncthing/syncthing/issues/5053 -func (db *schemaUpdater) updateSchemaTo5() error { +func (db *schemaUpdater) updateSchemaTo5(prevVersion int) error { + if prevVersion != 3 && prevVersion != 4 { + return nil + } + t, err := db.newReadWriteTransaction() if err != nil { return err @@ -321,10 +318,10 @@ func (db *schemaUpdater) updateSchemaTo5() error { return err } - return db.updateSchema2to3() + return db.updateSchema2to3(2) } -func (db *schemaUpdater) updateSchema5to6() error { +func (db *schemaUpdater) updateSchema5to6(_ int) error { // For every local file with the Invalid bit set, clear the Invalid bit and // set LocalFlags = FlagLocalIgnored. @@ -369,7 +366,7 @@ func (db *schemaUpdater) updateSchema5to6() error { // updateSchema6to7 checks whether all currently locally needed files are really // needed and removes them if not. -func (db *schemaUpdater) updateSchema6to7() error { +func (db *schemaUpdater) updateSchema6to7(_ int) error { t, err := db.newReadWriteTransaction() if err != nil { return err @@ -423,3 +420,36 @@ func (db *schemaUpdater) updateSchema6to7() error { } return t.commit() } + +func (db *schemaUpdater) updateSchema7to8(_ int) error { + // Loads and rewrites all files with blocks, to deduplicate block lists. + + t, err := db.newReadWriteTransaction() + if err != nil { + return err + } + defer t.close() + + it, err := t.NewPrefixIterator([]byte{KeyTypeDevice}) + if err != nil { + return err + } + for it.Next() { + var fi protocol.FileInfo + if err := fi.Unmarshal(it.Value()); err != nil { + return err + } + if fi.Blocks == nil { + continue + } + if err := t.putFile(it.Key(), fi); err != nil { + return err + } + } + it.Release() + if err := it.Error(); err != nil { + return err + } + + return t.commit() +} diff --git a/lib/db/structs.pb.go b/lib/db/structs.pb.go index 8151e6749..312856cff 100644 --- a/lib/db/structs.pb.go +++ b/lib/db/structs.pb.go @@ -110,6 +110,7 @@ type FileInfoTruncated struct { Sequence int64 `protobuf:"varint,10,opt,name=sequence,proto3" json:"sequence,omitempty"` // repeated BlockInfo Blocks = 16 SymlinkTarget string `protobuf:"bytes,17,opt,name=symlink_target,json=symlinkTarget,proto3" json:"symlink_target,omitempty"` + BlocksHash []byte `protobuf:"bytes,18,opt,name=blocks_hash,json=blocksHash,proto3" json:"blocks_hash,omitempty"` Type protocol.FileInfoType `protobuf:"varint,2,opt,name=type,proto3,enum=protocol.FileInfoType" json:"type,omitempty"` Permissions uint32 `protobuf:"varint,4,opt,name=permissions,proto3" json:"permissions,omitempty"` ModifiedNs int32 `protobuf:"varint,11,opt,name=modified_ns,json=modifiedNs,proto3" json:"modified_ns,omitempty"` @@ -153,6 +154,82 @@ func (m *FileInfoTruncated) XXX_DiscardUnknown() { var xxx_messageInfo_FileInfoTruncated proto.InternalMessageInfo +// BlockList is the structure used to store block lists +type BlockList struct { + Blocks []protocol.BlockInfo `protobuf:"bytes,1,rep,name=Blocks,proto3" json:"Blocks"` +} + +func (m *BlockList) Reset() { *m = BlockList{} } +func (m *BlockList) String() string { return proto.CompactTextString(m) } +func (*BlockList) ProtoMessage() {} +func (*BlockList) Descriptor() ([]byte, []int) { + return fileDescriptor_e774e8f5f348d14d, []int{3} +} +func (m *BlockList) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *BlockList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_BlockList.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *BlockList) XXX_Merge(src proto.Message) { + xxx_messageInfo_BlockList.Merge(m, src) +} +func (m *BlockList) XXX_Size() int { + return m.ProtoSize() +} +func (m *BlockList) XXX_DiscardUnknown() { + xxx_messageInfo_BlockList.DiscardUnknown(m) +} + +var xxx_messageInfo_BlockList proto.InternalMessageInfo + +// BlocksHashOnly is used to only unmarshal the block list hash from a FileInfo +type BlocksHashOnly struct { + BlocksHash []byte `protobuf:"bytes,18,opt,name=blocks_hash,json=blocksHash,proto3" json:"blocks_hash,omitempty"` +} + +func (m *BlocksHashOnly) Reset() { *m = BlocksHashOnly{} } +func (m *BlocksHashOnly) String() string { return proto.CompactTextString(m) } +func (*BlocksHashOnly) ProtoMessage() {} +func (*BlocksHashOnly) Descriptor() ([]byte, []int) { + return fileDescriptor_e774e8f5f348d14d, []int{4} +} +func (m *BlocksHashOnly) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *BlocksHashOnly) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_BlocksHashOnly.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *BlocksHashOnly) XXX_Merge(src proto.Message) { + xxx_messageInfo_BlocksHashOnly.Merge(m, src) +} +func (m *BlocksHashOnly) XXX_Size() int { + return m.ProtoSize() +} +func (m *BlocksHashOnly) XXX_DiscardUnknown() { + xxx_messageInfo_BlocksHashOnly.DiscardUnknown(m) +} + +var xxx_messageInfo_BlocksHashOnly proto.InternalMessageInfo + // For each folder and device we keep one of these to track the current // counts and sequence. We also keep one for the global state of the folder. type Counts struct { @@ -170,7 +247,7 @@ func (m *Counts) Reset() { *m = Counts{} } func (m *Counts) String() string { return proto.CompactTextString(m) } func (*Counts) ProtoMessage() {} func (*Counts) Descriptor() ([]byte, []int) { - return fileDescriptor_e774e8f5f348d14d, []int{3} + return fileDescriptor_e774e8f5f348d14d, []int{5} } func (m *Counts) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -208,7 +285,7 @@ func (m *CountsSet) Reset() { *m = CountsSet{} } func (m *CountsSet) String() string { return proto.CompactTextString(m) } func (*CountsSet) ProtoMessage() {} func (*CountsSet) Descriptor() ([]byte, []int) { - return fileDescriptor_e774e8f5f348d14d, []int{4} + return fileDescriptor_e774e8f5f348d14d, []int{6} } func (m *CountsSet) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -241,6 +318,8 @@ func init() { proto.RegisterType((*FileVersion)(nil), "db.FileVersion") proto.RegisterType((*VersionList)(nil), "db.VersionList") proto.RegisterType((*FileInfoTruncated)(nil), "db.FileInfoTruncated") + proto.RegisterType((*BlockList)(nil), "db.BlockList") + proto.RegisterType((*BlocksHashOnly)(nil), "db.BlocksHashOnly") proto.RegisterType((*Counts)(nil), "db.Counts") proto.RegisterType((*CountsSet)(nil), "db.CountsSet") } @@ -248,50 +327,54 @@ func init() { func init() { proto.RegisterFile("structs.proto", fileDescriptor_e774e8f5f348d14d) } var fileDescriptor_e774e8f5f348d14d = []byte{ - // 683 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0x4f, 0x6f, 0xda, 0x4e, - 0x10, 0xc5, 0xc1, 0x10, 0x18, 0x20, 0xbf, 0x64, 0x15, 0x45, 0x16, 0xd2, 0xcf, 0x58, 0x54, 0x95, - 0xac, 0x1e, 0xa0, 0x4d, 0x6e, 0xed, 0x8d, 0x46, 0x91, 0x90, 0xaa, 0xb6, 0x5a, 0xa2, 0x9c, 0x2a, - 0x21, 0xff, 0x59, 0xc8, 0x2a, 0xc6, 0x4b, 0xbc, 0x4b, 0x22, 0xf2, 0x29, 0x7a, 0xec, 0x31, 0x1f, - 0x27, 0xc7, 0x1c, 0xab, 0x1e, 0x50, 0x0a, 0x3d, 0xf4, 0x63, 0x54, 0xbb, 0x6b, 0x1b, 0x37, 0xa7, - 0xde, 0xe6, 0xbd, 0x19, 0x7b, 0x66, 0xde, 0xbc, 0x85, 0x16, 0x17, 0xc9, 0x22, 0x10, 0xbc, 0x37, - 0x4f, 0x98, 0x60, 0x68, 0x27, 0xf4, 0xdb, 0x2f, 0x12, 0x32, 0x67, 0xbc, 0xaf, 0x08, 0x7f, 0x31, - 0xe9, 0x4f, 0xd9, 0x94, 0x29, 0xa0, 0x22, 0x5d, 0xd8, 0x3e, 0x8a, 0xa8, 0xaf, 0x4b, 0x02, 0x16, - 0xf5, 0x7d, 0x32, 0xd7, 0x7c, 0xf7, 0x1a, 0x1a, 0x67, 0x34, 0x22, 0x17, 0x24, 0xe1, 0x94, 0xc5, - 0xe8, 0x35, 0xec, 0xde, 0xe8, 0xd0, 0x32, 0x1c, 0xc3, 0x6d, 0x1c, 0xef, 0xf7, 0xb2, 0x8f, 0x7a, - 0x17, 0x24, 0x10, 0x2c, 0x19, 0x98, 0x0f, 0xab, 0x4e, 0x09, 0x67, 0x65, 0xe8, 0x08, 0xaa, 0x21, - 0xb9, 0xa1, 0x01, 0xb1, 0x76, 0x1c, 0xc3, 0x6d, 0xe2, 0x14, 0x21, 0x0b, 0x76, 0x69, 0x7c, 0xe3, - 0x45, 0x34, 0xb4, 0xca, 0x8e, 0xe1, 0xd6, 0x70, 0x06, 0xbb, 0x67, 0xd0, 0x48, 0xdb, 0x7d, 0xa0, - 0x5c, 0xa0, 0x37, 0x50, 0x4b, 0xff, 0xc5, 0x2d, 0xc3, 0x29, 0xbb, 0x8d, 0xe3, 0xff, 0x7a, 0xa1, - 0xdf, 0x2b, 0x4c, 0x95, 0xb6, 0xcc, 0xcb, 0xde, 0x9a, 0xdf, 0xee, 0x3b, 0xa5, 0xee, 0x93, 0x09, - 0x07, 0xb2, 0x6a, 0x18, 0x4f, 0xd8, 0x79, 0xb2, 0x88, 0x03, 0x4f, 0x90, 0x10, 0x21, 0x30, 0x63, - 0x6f, 0x46, 0xd4, 0xf8, 0x75, 0xac, 0x62, 0xc9, 0x71, 0x7a, 0x47, 0xd4, 0x20, 0x65, 0xac, 0x62, - 0xf4, 0x3f, 0xc0, 0x8c, 0x85, 0x74, 0x42, 0x49, 0x38, 0xe6, 0x56, 0x45, 0x65, 0xea, 0x19, 0x33, - 0x42, 0x5f, 0xa0, 0x91, 0xa7, 0xfd, 0xa5, 0xd5, 0x74, 0x0c, 0xd7, 0x1c, 0xbc, 0x93, 0x73, 0xfc, - 0x58, 0x75, 0x4e, 0xa6, 0x54, 0x5c, 0x2e, 0xfc, 0x5e, 0xc0, 0x66, 0x7d, 0xbe, 0x8c, 0x03, 0x71, - 0x49, 0xe3, 0x69, 0x21, 0x2a, 0x6a, 0xdd, 0x1b, 0x5d, 0xb2, 0x44, 0x0c, 0x4f, 0x71, 0xde, 0x6e, - 0xb0, 0x2c, 0xca, 0x5c, 0xff, 0x37, 0x99, 0xdb, 0x50, 0xe3, 0xe4, 0x7a, 0x41, 0xe2, 0x80, 0x58, - 0xa0, 0x86, 0xcd, 0x31, 0x7a, 0x09, 0x7b, 0x7c, 0x39, 0x8b, 0x68, 0x7c, 0x35, 0x16, 0x5e, 0x32, - 0x25, 0xc2, 0x3a, 0x50, 0xcb, 0xb7, 0x52, 0xf6, 0x5c, 0x91, 0xe8, 0x15, 0x98, 0x62, 0x39, 0xd7, - 0x77, 0xda, 0x3b, 0x3e, 0xda, 0x76, 0xcc, 0x45, 0x5c, 0xce, 0x09, 0x56, 0x35, 0xc8, 0x81, 0xc6, - 0x9c, 0x24, 0x33, 0xca, 0xf5, 0x5d, 0x4c, 0xc7, 0x70, 0x5b, 0xb8, 0x48, 0xa1, 0x4e, 0x41, 0xa0, - 0x98, 0x5b, 0x0d, 0xc7, 0x70, 0x2b, 0xdb, 0x1d, 0x3f, 0x72, 0xd4, 0x07, 0xf0, 0x23, 0x16, 0x5c, - 0x8d, 0x95, 0xf4, 0x2d, 0x99, 0x1f, 0xec, 0xaf, 0x57, 0x9d, 0x26, 0xf6, 0x6e, 0x07, 0x32, 0x31, - 0xa2, 0x77, 0x04, 0xd7, 0xfd, 0x2c, 0x94, 0x3d, 0x23, 0x16, 0x78, 0xd1, 0x78, 0x12, 0x79, 0x53, - 0x6e, 0xfd, 0xde, 0x55, 0x4d, 0x41, 0x71, 0x67, 0x92, 0x92, 0x9e, 0x0a, 0x49, 0x44, 0x04, 0x09, - 0xad, 0xaa, 0xf6, 0x54, 0x0a, 0x91, 0xbb, 0x75, 0x9b, 0xfc, 0xac, 0x36, 0xd8, 0x5b, 0xaf, 0x3a, - 0x80, 0xbd, 0xdb, 0xa1, 0x66, 0x73, 0xf7, 0x49, 0xb1, 0x62, 0x36, 0x2e, 0x2e, 0x57, 0x53, 0xbf, - 0x6a, 0xc5, 0xec, 0xf3, 0x96, 0x4c, 0x2d, 0xf6, 0xcb, 0x80, 0xea, 0x7b, 0xb6, 0x88, 0x05, 0x47, - 0x87, 0x50, 0x99, 0xd0, 0x88, 0x70, 0x65, 0xac, 0x0a, 0xd6, 0x40, 0xce, 0x1c, 0xd2, 0x44, 0x5d, - 0x8c, 0x12, 0xae, 0xa4, 0xad, 0xe0, 0x22, 0xa5, 0x0e, 0xa7, 0xcf, 0xc0, 0x95, 0xff, 0x2a, 0x38, - 0xc7, 0xc5, 0x7d, 0x4c, 0x95, 0xca, 0xf7, 0x39, 0x84, 0x8a, 0xbf, 0x14, 0x24, 0x33, 0xa6, 0x06, - 0x7f, 0x99, 0xa0, 0xfa, 0xcc, 0x04, 0x6d, 0xa8, 0xe9, 0x97, 0x37, 0x3c, 0x55, 0xe7, 0x6f, 0xe2, - 0x1c, 0x23, 0x1b, 0x0a, 0x2a, 0x5a, 0xe8, 0xb9, 0xae, 0xdd, 0x4f, 0x50, 0xd7, 0x5b, 0x8e, 0x88, - 0x40, 0x2e, 0x54, 0x03, 0x05, 0xd2, 0xd7, 0x08, 0xf2, 0x35, 0xea, 0x74, 0x6a, 0xca, 0x34, 0x2f, - 0xc7, 0x0f, 0x12, 0x22, 0x5f, 0x9d, 0x5a, 0xbc, 0x8c, 0x33, 0x38, 0x70, 0x1e, 0x7e, 0xda, 0xa5, - 0x87, 0xb5, 0x6d, 0x3c, 0xae, 0x6d, 0xe3, 0x69, 0x6d, 0x97, 0xbe, 0x6e, 0xec, 0xd2, 0xfd, 0xc6, - 0x36, 0x1e, 0x37, 0x76, 0xe9, 0xfb, 0xc6, 0x2e, 0xf9, 0x55, 0xe5, 0xbe, 0x93, 0x3f, 0x01, 0x00, - 0x00, 0xff, 0xff, 0x7e, 0x87, 0x05, 0xb2, 0xd0, 0x04, 0x00, 0x00, + // 743 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcf, 0x6f, 0xe3, 0x44, + 0x14, 0x8e, 0x37, 0x71, 0x9a, 0x3c, 0x27, 0x61, 0x77, 0x58, 0x55, 0x56, 0x24, 0x1c, 0x2b, 0x08, + 0xc9, 0xe2, 0x90, 0xb0, 0xdd, 0x1b, 0x48, 0x1c, 0xcc, 0xaa, 0x22, 0x12, 0x62, 0xd1, 0x64, 0xd5, + 0x13, 0x52, 0xe4, 0x1f, 0x93, 0x64, 0x54, 0xc7, 0x93, 0x7a, 0x26, 0xad, 0xdc, 0x1b, 0xff, 0x01, + 0x47, 0x8e, 0xfd, 0x73, 0x7a, 0xec, 0x11, 0x71, 0x88, 0x20, 0xe1, 0xc0, 0x9f, 0x81, 0x66, 0xc6, + 0x76, 0x5c, 0x2e, 0xec, 0x6d, 0xbe, 0xef, 0x3d, 0xfb, 0xbd, 0xf9, 0xbe, 0xcf, 0x86, 0x3e, 0x17, + 0xd9, 0x2e, 0x12, 0x7c, 0xb2, 0xcd, 0x98, 0x60, 0xe8, 0x45, 0x1c, 0x0e, 0x3f, 0xcf, 0xc8, 0x96, + 0xf1, 0xa9, 0x22, 0xc2, 0xdd, 0x72, 0xba, 0x62, 0x2b, 0xa6, 0x80, 0x3a, 0xe9, 0xc6, 0xe1, 0x79, + 0x42, 0x43, 0xdd, 0x12, 0xb1, 0x64, 0x1a, 0x92, 0xad, 0xe6, 0xc7, 0x37, 0x60, 0x5d, 0xd2, 0x84, + 0x5c, 0x91, 0x8c, 0x53, 0x96, 0xa2, 0xaf, 0xe0, 0xec, 0x56, 0x1f, 0x6d, 0xc3, 0x35, 0x3c, 0xeb, + 0xe2, 0xe5, 0xa4, 0x7c, 0x68, 0x72, 0x45, 0x22, 0xc1, 0x32, 0xbf, 0xf5, 0xb8, 0x1f, 0x35, 0x70, + 0xd9, 0x86, 0xce, 0xa1, 0x1d, 0x93, 0x5b, 0x1a, 0x11, 0xfb, 0x85, 0x6b, 0x78, 0x3d, 0x5c, 0x20, + 0x64, 0xc3, 0x19, 0x4d, 0x6f, 0x83, 0x84, 0xc6, 0x76, 0xd3, 0x35, 0xbc, 0x0e, 0x2e, 0xe1, 0xf8, + 0x12, 0xac, 0x62, 0xdc, 0x0f, 0x94, 0x0b, 0xf4, 0x06, 0x3a, 0xc5, 0xbb, 0xb8, 0x6d, 0xb8, 0x4d, + 0xcf, 0xba, 0xf8, 0x64, 0x12, 0x87, 0x93, 0xda, 0x56, 0xc5, 0xc8, 0xaa, 0xed, 0xeb, 0xd6, 0x6f, + 0x0f, 0xa3, 0xc6, 0xf8, 0x17, 0x13, 0x5e, 0xc9, 0xae, 0x59, 0xba, 0x64, 0x1f, 0xb2, 0x5d, 0x1a, + 0x05, 0x82, 0xc4, 0x08, 0x41, 0x2b, 0x0d, 0x36, 0x44, 0xad, 0xdf, 0xc5, 0xea, 0x2c, 0x39, 0x4e, + 0xef, 0x89, 0x5a, 0xa4, 0x89, 0xd5, 0x19, 0x7d, 0x06, 0xb0, 0x61, 0x31, 0x5d, 0x52, 0x12, 0x2f, + 0xb8, 0x6d, 0xaa, 0x4a, 0xb7, 0x64, 0xe6, 0xe8, 0x67, 0xb0, 0xaa, 0x72, 0x98, 0xdb, 0x3d, 0xd7, + 0xf0, 0x5a, 0xfe, 0x37, 0x72, 0x8f, 0x3f, 0xf6, 0xa3, 0xb7, 0x2b, 0x2a, 0xd6, 0xbb, 0x70, 0x12, + 0xb1, 0xcd, 0x94, 0xe7, 0x69, 0x24, 0xd6, 0x34, 0x5d, 0xd5, 0x4e, 0x75, 0xad, 0x27, 0xf3, 0x35, + 0xcb, 0xc4, 0xec, 0x1d, 0xae, 0xc6, 0xf9, 0x79, 0x5d, 0xe6, 0xee, 0xc7, 0xc9, 0x3c, 0x84, 0x0e, + 0x27, 0x37, 0x3b, 0x92, 0x46, 0xc4, 0x06, 0xb5, 0x6c, 0x85, 0xd1, 0x17, 0x30, 0xe0, 0xf9, 0x26, + 0xa1, 0xe9, 0xf5, 0x42, 0x04, 0xd9, 0x8a, 0x08, 0xfb, 0x95, 0xba, 0x7c, 0xbf, 0x60, 0x3f, 0x28, + 0x12, 0x8d, 0xc0, 0x0a, 0x13, 0x16, 0x5d, 0xf3, 0xc5, 0x3a, 0xe0, 0x6b, 0x1b, 0x29, 0xbb, 0x40, + 0x53, 0xdf, 0x07, 0x7c, 0x8d, 0xbe, 0x84, 0x96, 0xc8, 0xb7, 0xda, 0xc8, 0xc1, 0xc5, 0xf9, 0x69, + 0xa5, 0x4a, 0xe5, 0x7c, 0x4b, 0xb0, 0xea, 0x41, 0x2e, 0x58, 0x5b, 0x92, 0x6d, 0x28, 0xd7, 0xc6, + 0xb5, 0x5c, 0xc3, 0xeb, 0xe3, 0x3a, 0x25, 0xc7, 0x55, 0x0a, 0xa6, 0xdc, 0xb6, 0x5c, 0xc3, 0x33, + 0x4f, 0x22, 0xfc, 0xc8, 0xd1, 0x14, 0xf4, 0xf0, 0x85, 0xf2, 0xa6, 0x2f, 0xeb, 0xfe, 0xcb, 0xc3, + 0x7e, 0xd4, 0xc3, 0xc1, 0x9d, 0x2f, 0x0b, 0x73, 0x7a, 0x4f, 0x70, 0x37, 0x2c, 0x8f, 0x72, 0x66, + 0xc2, 0xa2, 0x20, 0x59, 0x2c, 0x93, 0x60, 0xc5, 0xed, 0x7f, 0xce, 0xd4, 0x50, 0x50, 0xdc, 0xa5, + 0xa4, 0x64, 0xe8, 0x62, 0x92, 0x10, 0x41, 0x62, 0xbb, 0xad, 0x43, 0x57, 0x40, 0xe4, 0x9d, 0xe2, + 0x28, 0x1f, 0xeb, 0xf8, 0x83, 0xc3, 0x7e, 0x04, 0x38, 0xb8, 0x9b, 0x69, 0xb6, 0x8a, 0xa7, 0x54, + 0x33, 0x65, 0x8b, 0xfa, 0xe5, 0x3a, 0xea, 0x55, 0xfd, 0x94, 0xfd, 0x74, 0x22, 0x8b, 0x0c, 0x7e, + 0x0b, 0x5d, 0xb5, 0x6a, 0x91, 0xe4, 0xb6, 0x02, 0x65, 0x8e, 0x3f, 0x3d, 0x29, 0xa8, 0x78, 0x29, + 0x61, 0xe1, 0x6b, 0xd1, 0x38, 0x7e, 0x03, 0x03, 0xbf, 0x32, 0xe0, 0x7d, 0x9a, 0xe4, 0xff, 0xeb, + 0xd2, 0xf8, 0x6f, 0x03, 0xda, 0xdf, 0xb1, 0x5d, 0x2a, 0x38, 0x7a, 0x0d, 0xe6, 0x92, 0x26, 0x84, + 0xab, 0xb0, 0x9b, 0x58, 0x03, 0x29, 0x53, 0x4c, 0x33, 0x95, 0x22, 0x4a, 0xb8, 0x72, 0xd3, 0xc4, + 0x75, 0x4a, 0x85, 0x49, 0x47, 0x83, 0xab, 0x6f, 0xc2, 0xc4, 0x15, 0xae, 0x4b, 0xd8, 0x52, 0xa5, + 0x4a, 0xc2, 0xd7, 0x60, 0x86, 0xb9, 0x20, 0xe5, 0xc7, 0xa2, 0xc1, 0xb3, 0x60, 0xb6, 0xff, 0x13, + 0xcc, 0x21, 0x74, 0xf4, 0xdf, 0x60, 0xf6, 0x4e, 0x45, 0xb2, 0x87, 0x2b, 0x8c, 0x1c, 0xa8, 0x19, + 0xa7, 0xae, 0xf9, 0xcc, 0xca, 0xf1, 0x7b, 0xe8, 0xea, 0x5b, 0xce, 0x89, 0x40, 0x1e, 0xb4, 0x23, + 0x05, 0x0a, 0x65, 0x41, 0xfe, 0x21, 0x74, 0xb9, 0x14, 0x54, 0xd7, 0xe5, 0xfa, 0x51, 0x46, 0xe4, + 0x9f, 0x40, 0x5d, 0xbc, 0x89, 0x4b, 0xe8, 0xbb, 0x8f, 0x7f, 0x39, 0x8d, 0xc7, 0x83, 0x63, 0x3c, + 0x1d, 0x1c, 0xe3, 0xcf, 0x83, 0xd3, 0xf8, 0xf5, 0xe8, 0x34, 0x1e, 0x8e, 0x8e, 0xf1, 0x74, 0x74, + 0x1a, 0xbf, 0x1f, 0x9d, 0x46, 0xd8, 0x56, 0x76, 0xbd, 0xfd, 0x37, 0x00, 0x00, 0xff, 0xff, 0xb8, + 0xd0, 0x05, 0xba, 0x64, 0x05, 0x00, 0x00, } func (m *FileVersion) Marshal() (dAtA []byte, err error) { @@ -408,6 +491,15 @@ func (m *FileInfoTruncated) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0xc0 } + if len(m.BlocksHash) > 0 { + i -= len(m.BlocksHash) + copy(dAtA[i:], m.BlocksHash) + i = encodeVarintStructs(dAtA, i, uint64(len(m.BlocksHash))) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x92 + } if len(m.SymlinkTarget) > 0 { i -= len(m.SymlinkTarget) copy(dAtA[i:], m.SymlinkTarget) @@ -507,6 +599,75 @@ func (m *FileInfoTruncated) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *BlockList) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *BlockList) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *BlockList) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Blocks) > 0 { + for iNdEx := len(m.Blocks) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Blocks[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintStructs(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *BlocksHashOnly) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *BlocksHashOnly) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *BlocksHashOnly) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.BlocksHash) > 0 { + i -= len(m.BlocksHash) + copy(dAtA[i:], m.BlocksHash) + i = encodeVarintStructs(dAtA, i, uint64(len(m.BlocksHash))) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x92 + } + return len(dAtA) - i, nil +} + func (m *Counts) Marshal() (dAtA []byte, err error) { size := m.ProtoSize() dAtA = make([]byte, size) @@ -711,12 +872,44 @@ func (m *FileInfoTruncated) ProtoSize() (n int) { if l > 0 { n += 2 + l + sovStructs(uint64(l)) } + l = len(m.BlocksHash) + if l > 0 { + n += 2 + l + sovStructs(uint64(l)) + } if m.LocalFlags != 0 { n += 2 + sovStructs(uint64(m.LocalFlags)) } return n } +func (m *BlockList) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Blocks) > 0 { + for _, e := range m.Blocks { + l = e.ProtoSize() + n += 1 + l + sovStructs(uint64(l)) + } + } + return n +} + +func (m *BlocksHashOnly) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.BlocksHash) + if l > 0 { + n += 2 + l + sovStructs(uint64(l)) + } + return n +} + func (m *Counts) ProtoSize() (n int) { if m == nil { return 0 @@ -1340,6 +1533,40 @@ func (m *FileInfoTruncated) Unmarshal(dAtA []byte) error { } m.SymlinkTarget = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 18: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field BlocksHash", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStructs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthStructs + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthStructs + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.BlocksHash = append(m.BlocksHash[:0], dAtA[iNdEx:postIndex]...) + if m.BlocksHash == nil { + m.BlocksHash = []byte{} + } + iNdEx = postIndex case 1000: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field LocalFlags", wireType) @@ -1383,6 +1610,180 @@ func (m *FileInfoTruncated) Unmarshal(dAtA []byte) error { } return nil } +func (m *BlockList) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStructs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: BlockList: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: BlockList: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Blocks", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStructs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthStructs + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthStructs + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Blocks = append(m.Blocks, protocol.BlockInfo{}) + if err := m.Blocks[len(m.Blocks)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipStructs(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthStructs + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthStructs + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *BlocksHashOnly) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStructs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: BlocksHashOnly: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: BlocksHashOnly: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 18: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field BlocksHash", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStructs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthStructs + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthStructs + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.BlocksHash = append(m.BlocksHash[:0], dAtA[iNdEx:postIndex]...) + if m.BlocksHash == nil { + m.BlocksHash = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipStructs(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthStructs + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthStructs + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *Counts) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/lib/db/structs.proto b/lib/db/structs.proto index c0f5337a7..1acbaaee3 100644 --- a/lib/db/structs.proto +++ b/lib/db/structs.proto @@ -34,6 +34,7 @@ message FileInfoTruncated { int64 sequence = 10; // repeated BlockInfo Blocks = 16 string symlink_target = 17; + bytes blocks_hash = 18; protocol.FileInfoType type = 2; uint32 permissions = 4; int32 modified_ns = 11; @@ -47,6 +48,16 @@ message FileInfoTruncated { bool no_permissions = 8; } +// BlockList is the structure used to store block lists +message BlockList { + repeated protocol.BlockInfo Blocks = 1 [(gogoproto.nullable) = false]; +} + +// BlocksHashOnly is used to only unmarshal the block list hash from a FileInfo +message BlocksHashOnly { + bytes blocks_hash = 18; +} + // For each folder and device we keep one of these to track the current // counts and sequence. We also keep one for the global state of the folder. message Counts { diff --git a/lib/db/transactions.go b/lib/db/transactions.go index c0fd528c0..9a8ddff2c 100644 --- a/lib/db/transactions.go +++ b/lib/db/transactions.go @@ -58,13 +58,50 @@ func (t readOnlyTransaction) getFileTrunc(key []byte, trunc bool) (FileIntf, boo if err != nil { return nil, false, err } - f, err := unmarshalTrunc(bs, trunc) + f, err := t.unmarshalTrunc(bs, trunc) if err != nil { return nil, false, err } return f, true, nil } +func (t readOnlyTransaction) unmarshalTrunc(bs []byte, trunc bool) (FileIntf, error) { + if trunc { + var tf FileInfoTruncated + err := tf.Unmarshal(bs) + if err != nil { + return nil, err + } + return tf, nil + } + + var tf protocol.FileInfo + if err := tf.Unmarshal(bs); err != nil { + return nil, err + } + if err := t.fillBlockList(&tf); err != nil { + return nil, err + } + return tf, nil +} + +func (t readOnlyTransaction) fillBlockList(fi *protocol.FileInfo) error { + if fi.BlocksHash == nil { + return nil + } + blocksKey := t.keyer.GenerateBlockListKey(nil, fi.BlocksHash) + bs, err := t.Get(blocksKey) + if err != nil { + return err + } + var bl BlockList + if err := bl.Unmarshal(bs); err != nil { + return err + } + fi.Blocks = bl.Blocks + return nil +} + func (t readOnlyTransaction) getGlobal(keyBuf, folder, file []byte, truncate bool) ([]byte, FileIntf, bool, error) { var err error keyBuf, err = t.keyer.GenerateGlobalVersionKey(keyBuf, folder, file) @@ -132,7 +169,7 @@ func (t *readOnlyTransaction) withHave(folder, device, prefix []byte, truncate b return nil } - f, err := unmarshalTrunc(dbi.Value(), truncate) + f, err := t.unmarshalTrunc(dbi.Value(), truncate) if err != nil { l.Debugln("unmarshal error:", err) continue @@ -413,6 +450,28 @@ func (t readWriteTransaction) close() { t.WriteTransaction.Release() } +func (t readWriteTransaction) putFile(key []byte, fi protocol.FileInfo) error { + if fi.Blocks != nil { + if fi.BlocksHash == nil { + fi.BlocksHash = protocol.BlocksHash(fi.Blocks) + } + blocksKey := t.keyer.GenerateBlockListKey(nil, fi.BlocksHash) + if _, err := t.Get(blocksKey); backend.IsNotFound(err) { + // Marshal the block list and save it + blocksBs := mustMarshal(&BlockList{Blocks: fi.Blocks}) + if err := t.Put(blocksKey, blocksBs); err != nil { + return err + } + } else if err != nil { + return err + } + } + + fi.Blocks = nil + fiBs := mustMarshal(&fi) + return t.Put(key, fiBs) +} + // updateGlobal adds this device+version to the version list for the given // file. If the device is already present in the list, the version is updated. // If the file does not have an entry in the global list, it is created. diff --git a/lib/model/model.go b/lib/model/model.go index 112242ea2..d4ebbafca 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -954,8 +954,8 @@ func (m *model) handleIndex(deviceID protocol.DeviceID, folder string, fs []prot files.Drop(deviceID) } for i := range fs { - // The local flags should never be transmitted over the wire. Make - // sure they look like they weren't. + // The local attributes should never be transmitted over the wire. + // Make sure they look like they weren't. fs[i].LocalFlags = 0 } files.Update(deviceID, fs) diff --git a/lib/protocol/bep.pb.go b/lib/protocol/bep.pb.go index f7e5f275f..03036b0fe 100644 --- a/lib/protocol/bep.pb.go +++ b/lib/protocol/bep.pb.go @@ -495,8 +495,9 @@ type FileInfo struct { ModifiedBy ShortID `protobuf:"varint,12,opt,name=modified_by,json=modifiedBy,proto3,customtype=ShortID" json:"modified_by"` Version Vector `protobuf:"bytes,9,opt,name=version,proto3" json:"version"` Sequence int64 `protobuf:"varint,10,opt,name=sequence,proto3" json:"sequence,omitempty"` - Blocks []BlockInfo `protobuf:"bytes,16,rep,name=Blocks,proto3" json:"Blocks"` + Blocks []BlockInfo `protobuf:"bytes,16,rep,name=blocks,proto3" json:"blocks"` SymlinkTarget string `protobuf:"bytes,17,opt,name=symlink_target,json=symlinkTarget,proto3" json:"symlink_target,omitempty"` + BlocksHash []byte `protobuf:"bytes,18,opt,name=blocks_hash,json=blocksHash,proto3" json:"blocks_hash,omitempty"` Type FileInfoType `protobuf:"varint,2,opt,name=type,proto3,enum=protocol.FileInfoType" json:"type,omitempty"` Permissions uint32 `protobuf:"varint,4,opt,name=permissions,proto3" json:"permissions,omitempty"` ModifiedNs int32 `protobuf:"varint,11,opt,name=modified_ns,json=modifiedNs,proto3" json:"modified_ns,omitempty"` @@ -920,120 +921,121 @@ func init() { func init() { proto.RegisterFile("bep.proto", fileDescriptor_e3f59eb60afbbc6e) } var fileDescriptor_e3f59eb60afbbc6e = []byte{ - // 1803 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcf, 0x6f, 0xdb, 0xc8, - 0x15, 0x16, 0x25, 0xea, 0xd7, 0x93, 0xec, 0xa5, 0x27, 0x89, 0xcb, 0x32, 0x59, 0x89, 0x51, 0x92, - 0x8d, 0xd6, 0xd8, 0x26, 0xe9, 0xee, 0xb6, 0x45, 0x8b, 0xb6, 0x80, 0x7e, 0xd0, 0x8e, 0x50, 0x47, + // 1816 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4f, 0x6f, 0xdb, 0xc8, + 0x15, 0x17, 0x25, 0xea, 0xdf, 0x93, 0xec, 0xa5, 0x27, 0x89, 0xcb, 0x32, 0x59, 0x89, 0x51, 0x92, + 0x8d, 0xd6, 0xd8, 0x26, 0xe9, 0xee, 0xb6, 0x45, 0x8b, 0xb6, 0x80, 0xfe, 0xd0, 0x8e, 0x50, 0x47, 0x72, 0x47, 0x72, 0xb6, 0xd9, 0x43, 0x09, 0x5a, 0x1c, 0xc9, 0x44, 0x28, 0x8e, 0x4a, 0x52, 0x76, - 0xb4, 0x7f, 0x82, 0x4e, 0x3d, 0xf6, 0x22, 0x60, 0x81, 0x9e, 0xfa, 0x9f, 0xe4, 0x98, 0xf6, 0x50, - 0x14, 0x3d, 0x18, 0x5d, 0xe7, 0xb2, 0xc7, 0xfe, 0x05, 0x45, 0x31, 0x33, 0xa4, 0x44, 0xd9, 0x9b, - 0x45, 0x0e, 0x7b, 0xd2, 0xcc, 0x7b, 0x1f, 0xdf, 0xcc, 0x7c, 0xf3, 0xbd, 0x6f, 0x04, 0xc5, 0x13, - 0x32, 0x7d, 0x34, 0xf5, 0x69, 0x48, 0x51, 0x81, 0xff, 0x0c, 0xa9, 0xab, 0xdd, 0xf3, 0xc9, 0x94, - 0x06, 0x8f, 0xf9, 0xfc, 0x64, 0x36, 0x7a, 0x3c, 0xa6, 0x63, 0xca, 0x27, 0x7c, 0x24, 0xe0, 0xb5, - 0x29, 0x64, 0x9f, 0x12, 0xd7, 0xa5, 0xa8, 0x0a, 0x25, 0x9b, 0x9c, 0x39, 0x43, 0x62, 0x7a, 0xd6, - 0x84, 0xa8, 0x92, 0x2e, 0xd5, 0x8b, 0x18, 0x44, 0xa8, 0x6b, 0x4d, 0x08, 0x03, 0x0c, 0x5d, 0x87, - 0x78, 0xa1, 0x00, 0xa4, 0x05, 0x40, 0x84, 0x38, 0xe0, 0x01, 0x6c, 0x47, 0x80, 0x33, 0xe2, 0x07, - 0x0e, 0xf5, 0xd4, 0x0c, 0xc7, 0x6c, 0x89, 0xe8, 0x73, 0x11, 0xac, 0x05, 0x90, 0x7b, 0x4a, 0x2c, - 0x9b, 0xf8, 0xe8, 0x63, 0x90, 0xc3, 0xf9, 0x54, 0xac, 0xb5, 0xfd, 0xe9, 0xad, 0x47, 0xf1, 0xce, - 0x1f, 0x3d, 0x23, 0x41, 0x60, 0x8d, 0xc9, 0x60, 0x3e, 0x25, 0x98, 0x43, 0xd0, 0x6f, 0xa1, 0x34, - 0xa4, 0x93, 0xa9, 0x4f, 0x02, 0x5e, 0x38, 0xcd, 0xbf, 0xb8, 0x73, 0xed, 0x8b, 0xd6, 0x1a, 0x83, - 0x93, 0x1f, 0xd4, 0x1a, 0xb0, 0xd5, 0x72, 0x67, 0x41, 0x48, 0xfc, 0x16, 0xf5, 0x46, 0xce, 0x18, - 0x3d, 0x81, 0xfc, 0x88, 0xba, 0x36, 0xf1, 0x03, 0x55, 0xd2, 0x33, 0xf5, 0xd2, 0xa7, 0xca, 0xba, - 0xd8, 0x3e, 0x4f, 0x34, 0xe5, 0xd7, 0x17, 0xd5, 0x14, 0x8e, 0x61, 0xb5, 0xbf, 0xa6, 0x21, 0x27, - 0x32, 0x68, 0x17, 0xd2, 0x8e, 0x2d, 0x28, 0x6a, 0xe6, 0x2e, 0x2f, 0xaa, 0xe9, 0x4e, 0x1b, 0xa7, - 0x1d, 0x1b, 0xdd, 0x84, 0xac, 0x6b, 0x9d, 0x10, 0x37, 0x22, 0x47, 0x4c, 0xd0, 0x6d, 0x28, 0xfa, - 0xc4, 0xb2, 0x4d, 0xea, 0xb9, 0x73, 0x4e, 0x49, 0x01, 0x17, 0x58, 0xa0, 0xe7, 0xb9, 0x73, 0xf4, - 0x13, 0x40, 0xce, 0xd8, 0xa3, 0x3e, 0x31, 0xa7, 0xc4, 0x9f, 0x38, 0x7c, 0xb7, 0x81, 0x2a, 0x73, - 0xd4, 0x8e, 0xc8, 0x1c, 0xad, 0x13, 0xe8, 0x1e, 0x6c, 0x45, 0x70, 0x9b, 0xb8, 0x24, 0x24, 0x6a, - 0x96, 0x23, 0xcb, 0x22, 0xd8, 0xe6, 0x31, 0xf4, 0x04, 0x6e, 0xda, 0x4e, 0x60, 0x9d, 0xb8, 0xc4, - 0x0c, 0xc9, 0x64, 0x6a, 0x3a, 0x9e, 0x4d, 0x5e, 0x91, 0x40, 0xcd, 0x71, 0x2c, 0x8a, 0x72, 0x03, - 0x32, 0x99, 0x76, 0x44, 0x06, 0xed, 0x42, 0x6e, 0x6a, 0xcd, 0x02, 0x62, 0xab, 0x79, 0x8e, 0x89, - 0x66, 0x8c, 0x25, 0xa1, 0x80, 0x40, 0x55, 0xae, 0xb2, 0xd4, 0xe6, 0x89, 0x98, 0xa5, 0x08, 0x56, - 0xfb, 0x6f, 0x1a, 0x72, 0x22, 0x83, 0x3e, 0x5a, 0xb1, 0x54, 0x6e, 0xee, 0x32, 0xd4, 0xbf, 0x2f, - 0xaa, 0x05, 0x91, 0xeb, 0xb4, 0x13, 0xac, 0x21, 0x90, 0x13, 0x8a, 0xe2, 0x63, 0x74, 0x07, 0x8a, - 0x96, 0x6d, 0xb3, 0xdb, 0x23, 0x81, 0x9a, 0xd1, 0x33, 0xf5, 0x22, 0x5e, 0x07, 0xd0, 0x2f, 0x36, - 0xd5, 0x20, 0x5f, 0xd5, 0xcf, 0xbb, 0x64, 0xc0, 0xae, 0x62, 0x48, 0xfc, 0x48, 0xc1, 0x59, 0xbe, - 0x5e, 0x81, 0x05, 0xb8, 0x7e, 0xef, 0x42, 0x79, 0x62, 0xbd, 0x32, 0x03, 0xf2, 0xa7, 0x19, 0xf1, - 0x86, 0x84, 0xd3, 0x95, 0xc1, 0xa5, 0x89, 0xf5, 0xaa, 0x1f, 0x85, 0x50, 0x05, 0xc0, 0xf1, 0x42, - 0x9f, 0xda, 0xb3, 0x21, 0xf1, 0x23, 0xae, 0x12, 0x11, 0xf4, 0x33, 0x28, 0x70, 0xb2, 0x4d, 0xc7, - 0x56, 0x0b, 0xba, 0x54, 0x97, 0x9b, 0x5a, 0x74, 0xf0, 0x3c, 0xa7, 0x9a, 0x9f, 0x3b, 0x1e, 0xe2, - 0x3c, 0xc7, 0x76, 0x6c, 0xf4, 0x6b, 0xd0, 0x82, 0x97, 0x0e, 0xbb, 0x28, 0x51, 0x29, 0x74, 0xa8, - 0x67, 0xfa, 0x64, 0x42, 0xcf, 0x2c, 0x37, 0x50, 0x8b, 0x7c, 0x19, 0x95, 0x21, 0x3a, 0x09, 0x00, - 0x8e, 0xf2, 0xb5, 0x1e, 0x64, 0x79, 0x45, 0x76, 0x8b, 0x42, 0xac, 0x51, 0xf7, 0x46, 0x33, 0xf4, - 0x08, 0xb2, 0x23, 0xc7, 0x25, 0x81, 0x9a, 0xe6, 0x77, 0x88, 0x12, 0x4a, 0x77, 0x5c, 0xd2, 0xf1, - 0x46, 0x34, 0xba, 0x45, 0x01, 0xab, 0x1d, 0x43, 0x89, 0x17, 0x3c, 0x9e, 0xda, 0x56, 0x48, 0x7e, - 0xb0, 0xb2, 0x17, 0x32, 0x14, 0xe2, 0xcc, 0xea, 0xd2, 0xa5, 0xc4, 0xa5, 0x23, 0x90, 0x03, 0xe7, - 0x2b, 0xc2, 0x7b, 0x24, 0x83, 0xf9, 0x18, 0x7d, 0x08, 0x30, 0xa1, 0xb6, 0x33, 0x72, 0x88, 0x6d, - 0x06, 0xfc, 0xca, 0x32, 0xb8, 0x18, 0x47, 0xfa, 0xe8, 0x09, 0x94, 0x56, 0xe9, 0x93, 0xb9, 0x5a, - 0xe6, 0x9c, 0x7f, 0x10, 0x73, 0xde, 0x3f, 0xa5, 0x7e, 0xd8, 0x69, 0xe3, 0x55, 0x89, 0xe6, 0x9c, - 0x49, 0x3a, 0xb6, 0x27, 0x46, 0xec, 0x86, 0xa4, 0x9f, 0x93, 0x61, 0x48, 0x57, 0x8d, 0x1f, 0xc1, - 0x90, 0x06, 0x85, 0x95, 0x26, 0x80, 0x6f, 0x60, 0x35, 0x47, 0x3f, 0x85, 0x5c, 0xd3, 0xa5, 0xc3, - 0x97, 0x71, 0x7f, 0xdc, 0x58, 0x17, 0xe3, 0xf1, 0x04, 0x0b, 0x11, 0x90, 0xd9, 0x64, 0x30, 0x9f, - 0xb8, 0x8e, 0xf7, 0xd2, 0x0c, 0x2d, 0x7f, 0x4c, 0x42, 0x75, 0x47, 0xd8, 0x64, 0x14, 0x1d, 0xf0, - 0x20, 0xda, 0x8b, 0xcc, 0x51, 0x58, 0xdd, 0xee, 0x75, 0x72, 0x13, 0xee, 0xa8, 0x43, 0xe9, 0xaa, - 0x7b, 0x6c, 0xe1, 0x64, 0x88, 0x99, 0xf7, 0x8a, 0x27, 0x2f, 0x50, 0x4b, 0xba, 0x54, 0xcf, 0xae, - 0x69, 0xe9, 0x06, 0xe8, 0x31, 0xc0, 0x09, 0xdb, 0x9f, 0xc9, 0x6f, 0x60, 0x8b, 0xe5, 0x9b, 0xca, - 0xe5, 0x45, 0xb5, 0x8c, 0xad, 0x73, 0xbe, 0xf1, 0xbe, 0xf3, 0x15, 0xc1, 0xc5, 0x93, 0x78, 0xc8, - 0xd6, 0x74, 0xe9, 0xd0, 0x72, 0xcd, 0x91, 0x6b, 0x8d, 0x03, 0xf5, 0xdb, 0x3c, 0x5f, 0x14, 0x78, - 0x6c, 0x9f, 0x85, 0x90, 0xca, 0xcc, 0x83, 0x19, 0x92, 0x1d, 0x39, 0x4f, 0x3c, 0x45, 0x75, 0xc8, - 0x3b, 0xde, 0x99, 0xe5, 0x3a, 0x91, 0xdf, 0x34, 0xb7, 0x2f, 0x2f, 0xaa, 0x80, 0xad, 0xf3, 0x8e, - 0x88, 0xe2, 0x38, 0xcd, 0xc8, 0xf2, 0xe8, 0x86, 0x35, 0x16, 0x78, 0xa9, 0x2d, 0x8f, 0x26, 0x6c, - 0xf1, 0x57, 0xf2, 0x5f, 0xbe, 0xae, 0xa6, 0x6a, 0x1e, 0x14, 0x57, 0xa4, 0x33, 0x31, 0x9d, 0x5a, - 0xc1, 0x29, 0x17, 0x53, 0x19, 0xf3, 0x31, 0x53, 0x32, 0x1d, 0x8d, 0x02, 0x12, 0x72, 0xd9, 0x65, - 0x70, 0x34, 0x5b, 0x09, 0x2f, 0xcd, 0x69, 0x11, 0xc2, 0xbb, 0x0d, 0xc5, 0x73, 0x62, 0xbd, 0x34, - 0x79, 0x11, 0xc1, 0x68, 0x81, 0x05, 0x9e, 0x5a, 0xc1, 0x69, 0xb4, 0xde, 0x6f, 0x20, 0x27, 0x14, - 0x83, 0x3e, 0x83, 0xc2, 0x90, 0xce, 0xbc, 0x70, 0xfd, 0x9c, 0xec, 0x24, 0xdd, 0x88, 0x67, 0x22, - 0x19, 0xac, 0x80, 0xb5, 0x7d, 0xc8, 0x47, 0x29, 0xf4, 0x60, 0x65, 0x95, 0x72, 0xf3, 0xd6, 0x15, - 0xf5, 0x6e, 0xbe, 0x2f, 0x67, 0x96, 0x3b, 0x13, 0x1b, 0x95, 0xb1, 0x98, 0xd4, 0xfe, 0x2e, 0x41, - 0x1e, 0x33, 0x41, 0x06, 0x61, 0xe2, 0x65, 0xca, 0x6e, 0xbc, 0x4c, 0xeb, 0x1e, 0x4e, 0x6f, 0xf4, - 0x70, 0xdc, 0x86, 0x99, 0x44, 0x1b, 0xae, 0x59, 0x92, 0xbf, 0x93, 0xa5, 0x6c, 0x82, 0xa5, 0x98, - 0xe5, 0x5c, 0x82, 0xe5, 0x07, 0xb0, 0x3d, 0xf2, 0xe9, 0x84, 0xbf, 0x3d, 0xd4, 0xb7, 0xfc, 0x79, - 0x64, 0x94, 0x5b, 0x2c, 0x3a, 0x88, 0x83, 0x9b, 0x04, 0x17, 0x36, 0x09, 0xae, 0x99, 0x50, 0xc0, - 0x24, 0x98, 0x52, 0x2f, 0x20, 0xef, 0x3c, 0x13, 0x02, 0xd9, 0xb6, 0x42, 0x8b, 0x9f, 0xa8, 0x8c, - 0xf9, 0x18, 0x3d, 0x04, 0x79, 0x48, 0x6d, 0x71, 0x9e, 0xed, 0x64, 0x37, 0x1a, 0xbe, 0x4f, 0xfd, - 0x16, 0xb5, 0x09, 0xe6, 0x80, 0xda, 0x14, 0x94, 0x36, 0x3d, 0xf7, 0x5c, 0x6a, 0xd9, 0x47, 0x3e, - 0x1d, 0xb3, 0x07, 0xe2, 0x9d, 0x46, 0xd7, 0x86, 0xfc, 0x8c, 0x5b, 0x61, 0x6c, 0x75, 0xf7, 0x37, - 0xbb, 0xf1, 0x6a, 0x21, 0xe1, 0x9b, 0xb1, 0x8d, 0x44, 0x9f, 0xd6, 0xfe, 0x29, 0x81, 0xf6, 0x6e, - 0x34, 0xea, 0x40, 0x49, 0x20, 0xcd, 0xc4, 0x7f, 0xa2, 0xfa, 0xfb, 0x2c, 0xc4, 0x8d, 0x00, 0x66, - 0xab, 0xf1, 0x77, 0x3e, 0xa8, 0x09, 0xdb, 0xcb, 0xbc, 0x9f, 0xed, 0x3d, 0x84, 0x2d, 0xe1, 0x08, - 0xf1, 0xdf, 0x07, 0x59, 0xcf, 0xd4, 0xb3, 0xcd, 0xb4, 0x92, 0xc2, 0xe5, 0x13, 0xd1, 0x66, 0x3c, - 0x5e, 0xcb, 0x81, 0x7c, 0xe4, 0x78, 0xe3, 0x5a, 0x15, 0xb2, 0x2d, 0x97, 0xf2, 0x0b, 0xcb, 0xf9, - 0xc4, 0x0a, 0xa8, 0x17, 0xf3, 0x28, 0x66, 0x7b, 0xff, 0x48, 0x43, 0x29, 0xf1, 0xd7, 0x0e, 0x3d, - 0x81, 0xed, 0xd6, 0xe1, 0x71, 0x7f, 0x60, 0x60, 0xb3, 0xd5, 0xeb, 0xee, 0x77, 0x0e, 0x94, 0x94, - 0x76, 0x67, 0xb1, 0xd4, 0xd5, 0xc9, 0x1a, 0xb4, 0xf9, 0xaf, 0xad, 0x0a, 0xd9, 0x4e, 0xb7, 0x6d, - 0xfc, 0x41, 0x91, 0xb4, 0x9b, 0x8b, 0xa5, 0xae, 0x24, 0x80, 0xe2, 0x09, 0xfc, 0x04, 0xca, 0x1c, - 0x60, 0x1e, 0x1f, 0xb5, 0x1b, 0x03, 0x43, 0x49, 0x6b, 0xda, 0x62, 0xa9, 0xef, 0x5e, 0xc5, 0x45, - 0x9c, 0xdf, 0x83, 0x3c, 0x36, 0x7e, 0x7f, 0x6c, 0xf4, 0x07, 0x4a, 0x46, 0xdb, 0x5d, 0x2c, 0x75, - 0x94, 0x00, 0xc6, 0x2d, 0xf5, 0x00, 0x0a, 0xd8, 0xe8, 0x1f, 0xf5, 0xba, 0x7d, 0x43, 0x91, 0xb5, - 0x1f, 0x2d, 0x96, 0xfa, 0x8d, 0x0d, 0x54, 0xa4, 0xd2, 0x9f, 0xc3, 0x4e, 0xbb, 0xf7, 0x45, 0xf7, - 0xb0, 0xd7, 0x68, 0x9b, 0x47, 0xb8, 0x77, 0x80, 0x8d, 0x7e, 0x5f, 0xc9, 0x6a, 0xd5, 0xc5, 0x52, - 0xbf, 0x9d, 0xc0, 0x5f, 0x13, 0xdd, 0x87, 0x20, 0x1f, 0x75, 0xba, 0x07, 0x4a, 0x4e, 0xbb, 0xb1, - 0x58, 0xea, 0x1f, 0x24, 0xa0, 0x8c, 0x54, 0x76, 0xe2, 0xd6, 0x61, 0xaf, 0x6f, 0x28, 0xf9, 0x6b, - 0x27, 0xe6, 0x64, 0xef, 0xfd, 0x11, 0xd0, 0xf5, 0x3f, 0xbf, 0xe8, 0x3e, 0xc8, 0xdd, 0x5e, 0xd7, - 0x50, 0x52, 0xe2, 0xfc, 0xd7, 0x11, 0x5d, 0xea, 0x11, 0x54, 0x83, 0xcc, 0xe1, 0x97, 0x9f, 0x2b, - 0x92, 0xf6, 0xe3, 0xc5, 0x52, 0xbf, 0x75, 0x1d, 0x74, 0xf8, 0xe5, 0xe7, 0x7b, 0x14, 0x4a, 0xc9, - 0xc2, 0x35, 0x28, 0x3c, 0x33, 0x06, 0x8d, 0x76, 0x63, 0xd0, 0x50, 0x52, 0x62, 0x4b, 0x71, 0xfa, - 0x19, 0x09, 0x2d, 0xde, 0x84, 0x77, 0x20, 0xdb, 0x35, 0x9e, 0x1b, 0x58, 0x91, 0xb4, 0x9d, 0xc5, - 0x52, 0xdf, 0x8a, 0x01, 0x5d, 0x72, 0x46, 0x7c, 0x54, 0x81, 0x5c, 0xe3, 0xf0, 0x8b, 0xc6, 0x8b, - 0xbe, 0x92, 0xd6, 0xd0, 0x62, 0xa9, 0x6f, 0xc7, 0xe9, 0x86, 0x7b, 0x6e, 0xcd, 0x83, 0xbd, 0xff, - 0x49, 0x50, 0x4e, 0xbe, 0x71, 0xa8, 0x02, 0xf2, 0x7e, 0xe7, 0xd0, 0x88, 0x97, 0x4b, 0xe6, 0xd8, - 0x18, 0xd5, 0xa1, 0xd8, 0xee, 0x60, 0xa3, 0x35, 0xe8, 0xe1, 0x17, 0xf1, 0x59, 0x92, 0xa0, 0xb6, - 0xe3, 0x73, 0x81, 0xcf, 0xd1, 0x2f, 0xa1, 0xdc, 0x7f, 0xf1, 0xec, 0xb0, 0xd3, 0xfd, 0x9d, 0xc9, - 0x2b, 0xa6, 0xb5, 0x87, 0x8b, 0xa5, 0x7e, 0x77, 0x03, 0x4c, 0xa6, 0x3e, 0x19, 0x5a, 0x21, 0xb1, - 0xfb, 0xe2, 0x39, 0x66, 0xc9, 0x82, 0x84, 0x5a, 0xb0, 0x13, 0x7f, 0xba, 0x5e, 0x2c, 0xa3, 0x7d, - 0xb2, 0x58, 0xea, 0x1f, 0x7d, 0xef, 0xf7, 0xab, 0xd5, 0x0b, 0x12, 0xba, 0x0f, 0xf9, 0xa8, 0x48, - 0xac, 0xa4, 0xe4, 0xa7, 0xd1, 0x07, 0x7b, 0x7f, 0x93, 0xa0, 0xb8, 0xb2, 0x2b, 0x46, 0x78, 0xb7, - 0x67, 0x1a, 0x18, 0xf7, 0x70, 0xcc, 0xc0, 0x2a, 0xd9, 0xa5, 0x7c, 0x88, 0xee, 0x42, 0xfe, 0xc0, - 0xe8, 0x1a, 0xb8, 0xd3, 0x8a, 0x1b, 0x63, 0x05, 0x39, 0x20, 0x1e, 0xf1, 0x9d, 0x21, 0xfa, 0x18, - 0xca, 0xdd, 0x9e, 0xd9, 0x3f, 0x6e, 0x3d, 0x8d, 0x8f, 0xce, 0xd7, 0x4f, 0x94, 0xea, 0xcf, 0x86, - 0xa7, 0x9c, 0xcf, 0x3d, 0xd6, 0x43, 0xcf, 0x1b, 0x87, 0x9d, 0xb6, 0x80, 0x66, 0x34, 0x75, 0xb1, - 0xd4, 0x6f, 0xae, 0xa0, 0xd1, 0x23, 0xcd, 0xb0, 0x7b, 0x36, 0x54, 0xbe, 0xdf, 0x98, 0x90, 0x0e, - 0xb9, 0xc6, 0xd1, 0x91, 0xd1, 0x6d, 0xc7, 0xbb, 0x5f, 0xe7, 0x1a, 0xd3, 0x29, 0xf1, 0x6c, 0x86, - 0xd8, 0xef, 0xe1, 0x03, 0x63, 0x10, 0x6f, 0x7e, 0x8d, 0xd8, 0xa7, 0xec, 0xbf, 0x50, 0xb3, 0xfe, - 0xfa, 0x9b, 0x4a, 0xea, 0xcd, 0x37, 0x95, 0xd4, 0xeb, 0xcb, 0x8a, 0xf4, 0xe6, 0xb2, 0x22, 0xfd, - 0xe7, 0xb2, 0x92, 0xfa, 0xf6, 0xb2, 0x22, 0xfd, 0xf9, 0x6d, 0x25, 0xf5, 0xf5, 0xdb, 0x8a, 0xf4, - 0xe6, 0x6d, 0x25, 0xf5, 0xaf, 0xb7, 0x95, 0xd4, 0x49, 0x8e, 0x9b, 0xda, 0x67, 0xff, 0x0f, 0x00, - 0x00, 0xff, 0xff, 0xd6, 0x0e, 0x8e, 0xa4, 0x11, 0x0f, 0x00, 0x00, + 0xb4, 0x1f, 0x41, 0xa7, 0x1e, 0x7b, 0x11, 0xb0, 0x40, 0x4f, 0xfd, 0x26, 0x39, 0xa6, 0x3d, 0x14, + 0x45, 0x0f, 0x46, 0xd7, 0xb9, 0xec, 0xb1, 0x9f, 0xa0, 0x2d, 0x66, 0x86, 0x94, 0x28, 0x7b, 0xb3, + 0xc8, 0xa1, 0x27, 0xce, 0xbc, 0xf7, 0x9b, 0x37, 0x33, 0xbf, 0xf7, 0xde, 0x6f, 0x08, 0xc5, 0x13, + 0x32, 0x7d, 0x34, 0xf5, 0x69, 0x48, 0x51, 0x81, 0x7f, 0x86, 0xd4, 0xd5, 0xee, 0xf9, 0x64, 0x4a, + 0x83, 0xc7, 0x7c, 0x7e, 0x32, 0x1b, 0x3d, 0x1e, 0xd3, 0x31, 0xe5, 0x13, 0x3e, 0x12, 0xf0, 0xda, + 0x14, 0xb2, 0x4f, 0x89, 0xeb, 0x52, 0x54, 0x85, 0x92, 0x4d, 0xce, 0x9c, 0x21, 0x31, 0x3d, 0x6b, + 0x42, 0x54, 0x49, 0x97, 0xea, 0x45, 0x0c, 0xc2, 0xd4, 0xb5, 0x26, 0x84, 0x01, 0x86, 0xae, 0x43, + 0xbc, 0x50, 0x00, 0xd2, 0x02, 0x20, 0x4c, 0x1c, 0xf0, 0x00, 0xb6, 0x23, 0xc0, 0x19, 0xf1, 0x03, + 0x87, 0x7a, 0x6a, 0x86, 0x63, 0xb6, 0x84, 0xf5, 0xb9, 0x30, 0xd6, 0x02, 0xc8, 0x3d, 0x25, 0x96, + 0x4d, 0x7c, 0xf4, 0x31, 0xc8, 0xe1, 0x7c, 0x2a, 0xf6, 0xda, 0xfe, 0xf4, 0xd6, 0xa3, 0xf8, 0xe4, + 0x8f, 0x9e, 0x91, 0x20, 0xb0, 0xc6, 0x64, 0x30, 0x9f, 0x12, 0xcc, 0x21, 0xe8, 0xd7, 0x50, 0x1a, + 0xd2, 0xc9, 0xd4, 0x27, 0x01, 0x0f, 0x9c, 0xe6, 0x2b, 0xee, 0x5c, 0x5b, 0xd1, 0x5a, 0x63, 0x70, + 0x72, 0x41, 0xad, 0x01, 0x5b, 0x2d, 0x77, 0x16, 0x84, 0xc4, 0x6f, 0x51, 0x6f, 0xe4, 0x8c, 0xd1, + 0x13, 0xc8, 0x8f, 0xa8, 0x6b, 0x13, 0x3f, 0x50, 0x25, 0x3d, 0x53, 0x2f, 0x7d, 0xaa, 0xac, 0x83, + 0xed, 0x73, 0x47, 0x53, 0x7e, 0x7d, 0x51, 0x4d, 0xe1, 0x18, 0x56, 0xfb, 0x73, 0x1a, 0x72, 0xc2, + 0x83, 0x76, 0x21, 0xed, 0xd8, 0x82, 0xa2, 0x66, 0xee, 0xf2, 0xa2, 0x9a, 0xee, 0xb4, 0x71, 0xda, + 0xb1, 0xd1, 0x4d, 0xc8, 0xba, 0xd6, 0x09, 0x71, 0x23, 0x72, 0xc4, 0x04, 0xdd, 0x86, 0xa2, 0x4f, + 0x2c, 0xdb, 0xa4, 0x9e, 0x3b, 0xe7, 0x94, 0x14, 0x70, 0x81, 0x19, 0x7a, 0x9e, 0x3b, 0x47, 0x3f, + 0x02, 0xe4, 0x8c, 0x3d, 0xea, 0x13, 0x73, 0x4a, 0xfc, 0x89, 0xc3, 0x4f, 0x1b, 0xa8, 0x32, 0x47, + 0xed, 0x08, 0xcf, 0xd1, 0xda, 0x81, 0xee, 0xc1, 0x56, 0x04, 0xb7, 0x89, 0x4b, 0x42, 0xa2, 0x66, + 0x39, 0xb2, 0x2c, 0x8c, 0x6d, 0x6e, 0x43, 0x4f, 0xe0, 0xa6, 0xed, 0x04, 0xd6, 0x89, 0x4b, 0xcc, + 0x90, 0x4c, 0xa6, 0xa6, 0xe3, 0xd9, 0xe4, 0x15, 0x09, 0xd4, 0x1c, 0xc7, 0xa2, 0xc8, 0x37, 0x20, + 0x93, 0x69, 0x47, 0x78, 0xd0, 0x2e, 0xe4, 0xa6, 0xd6, 0x2c, 0x20, 0xb6, 0x9a, 0xe7, 0x98, 0x68, + 0xc6, 0x58, 0x12, 0x15, 0x10, 0xa8, 0xca, 0x55, 0x96, 0xda, 0xdc, 0x11, 0xb3, 0x14, 0xc1, 0x6a, + 0xff, 0x4e, 0x43, 0x4e, 0x78, 0xd0, 0x47, 0x2b, 0x96, 0xca, 0xcd, 0x5d, 0x86, 0xfa, 0xe7, 0x45, + 0xb5, 0x20, 0x7c, 0x9d, 0x76, 0x82, 0x35, 0x04, 0x72, 0xa2, 0xa2, 0xf8, 0x18, 0xdd, 0x81, 0xa2, + 0x65, 0xdb, 0x2c, 0x7b, 0x24, 0x50, 0x33, 0x7a, 0xa6, 0x5e, 0xc4, 0x6b, 0x03, 0xfa, 0xd9, 0x66, + 0x35, 0xc8, 0x57, 0xeb, 0xe7, 0x5d, 0x65, 0xc0, 0x52, 0x31, 0x24, 0x7e, 0x54, 0xc1, 0x59, 0xbe, + 0x5f, 0x81, 0x19, 0x78, 0xfd, 0xde, 0x85, 0xf2, 0xc4, 0x7a, 0x65, 0x06, 0xe4, 0x0f, 0x33, 0xe2, + 0x0d, 0x09, 0xa7, 0x2b, 0x83, 0x4b, 0x13, 0xeb, 0x55, 0x3f, 0x32, 0xa1, 0x0a, 0x80, 0xe3, 0x85, + 0x3e, 0xb5, 0x67, 0x43, 0xe2, 0x47, 0x5c, 0x25, 0x2c, 0xe8, 0x27, 0x50, 0xe0, 0x64, 0x9b, 0x8e, + 0xad, 0x16, 0x74, 0xa9, 0x2e, 0x37, 0xb5, 0xe8, 0xe2, 0x79, 0x4e, 0x35, 0xbf, 0x77, 0x3c, 0xc4, + 0x79, 0x8e, 0xed, 0xd8, 0xe8, 0x97, 0xa0, 0x05, 0x2f, 0x1d, 0x96, 0x28, 0x11, 0x29, 0x74, 0xa8, + 0x67, 0xfa, 0x64, 0x42, 0xcf, 0x2c, 0x37, 0x50, 0x8b, 0x7c, 0x1b, 0x95, 0x21, 0x3a, 0x09, 0x00, + 0x8e, 0xfc, 0xb5, 0x1e, 0x64, 0x79, 0x44, 0x96, 0x45, 0x51, 0xac, 0x51, 0xf7, 0x46, 0x33, 0xf4, + 0x08, 0xb2, 0x23, 0xc7, 0x25, 0x81, 0x9a, 0xe6, 0x39, 0x44, 0x89, 0x4a, 0x77, 0x5c, 0xd2, 0xf1, + 0x46, 0x34, 0xca, 0xa2, 0x80, 0xd5, 0x8e, 0xa1, 0xc4, 0x03, 0x1e, 0x4f, 0x6d, 0x2b, 0x24, 0xff, + 0xb7, 0xb0, 0xff, 0x95, 0xa1, 0x10, 0x7b, 0x56, 0x49, 0x97, 0x12, 0x49, 0x47, 0x20, 0x07, 0xce, + 0x57, 0x84, 0xf7, 0x48, 0x06, 0xf3, 0x31, 0xfa, 0x10, 0x60, 0x42, 0x6d, 0x67, 0xe4, 0x10, 0xdb, + 0x0c, 0x78, 0xca, 0x32, 0xb8, 0x18, 0x5b, 0xfa, 0xe8, 0x09, 0x94, 0x56, 0xee, 0x93, 0xb9, 0x5a, + 0xe6, 0x9c, 0x7f, 0x10, 0x73, 0xde, 0x3f, 0xa5, 0x7e, 0xd8, 0x69, 0xe3, 0x55, 0x88, 0xe6, 0x9c, + 0x95, 0x74, 0x2c, 0x4f, 0x8c, 0xd8, 0x8d, 0x92, 0x7e, 0x4e, 0x86, 0x21, 0x5d, 0x35, 0x7e, 0x04, + 0x43, 0x1a, 0x14, 0x56, 0x35, 0x01, 0xfc, 0x00, 0xab, 0x39, 0xfa, 0x31, 0xe4, 0x4e, 0x5c, 0x3a, + 0x7c, 0x19, 0xf7, 0xc7, 0x8d, 0x75, 0xb0, 0x26, 0xb3, 0x27, 0x58, 0x88, 0x80, 0x4c, 0x26, 0x83, + 0xf9, 0xc4, 0x75, 0xbc, 0x97, 0x66, 0x68, 0xf9, 0x63, 0x12, 0xaa, 0x3b, 0x42, 0x26, 0x23, 0xeb, + 0x80, 0x1b, 0x99, 0xdc, 0x8a, 0x05, 0xe6, 0xa9, 0x15, 0x9c, 0xaa, 0x88, 0xb5, 0x11, 0x06, 0x61, + 0x7a, 0x6a, 0x05, 0xa7, 0x68, 0x2f, 0x52, 0x4f, 0xa1, 0x85, 0xbb, 0xd7, 0xd9, 0x4f, 0xc8, 0xa7, + 0x0e, 0xa5, 0xab, 0xf2, 0xb2, 0x85, 0x93, 0x26, 0xb6, 0xdd, 0x8a, 0x48, 0x2f, 0x50, 0x4b, 0xba, + 0x54, 0xcf, 0xae, 0x79, 0xeb, 0x06, 0xe8, 0x31, 0x88, 0xcd, 0x4d, 0x9e, 0xa2, 0x2d, 0xe6, 0x6f, + 0x2a, 0x97, 0x17, 0xd5, 0x32, 0xb6, 0xce, 0xf9, 0x55, 0xfb, 0xce, 0x57, 0x04, 0x17, 0x4f, 0xe2, + 0x21, 0xdb, 0xd3, 0xa5, 0x43, 0xcb, 0x35, 0x47, 0xae, 0x35, 0x0e, 0xd4, 0x6f, 0xf3, 0x7c, 0x53, + 0xe0, 0xb6, 0x7d, 0x66, 0x42, 0x2a, 0x53, 0x17, 0xa6, 0x58, 0x76, 0x24, 0x4d, 0xf1, 0x14, 0xd5, + 0x21, 0xef, 0x78, 0x67, 0x96, 0xeb, 0x44, 0x82, 0xd4, 0xdc, 0xbe, 0xbc, 0xa8, 0x02, 0xb6, 0xce, + 0x3b, 0xc2, 0x8a, 0x63, 0x37, 0x63, 0xd3, 0xa3, 0x1b, 0xda, 0x59, 0xe0, 0xa1, 0xb6, 0x3c, 0x9a, + 0xd0, 0xcd, 0x5f, 0xc8, 0x7f, 0xfa, 0xba, 0x9a, 0xaa, 0x79, 0x50, 0x5c, 0x65, 0x85, 0x55, 0x1b, + 0x67, 0x36, 0xc3, 0x99, 0xe5, 0x63, 0x56, 0xea, 0x74, 0x34, 0x0a, 0x48, 0xc8, 0xeb, 0x32, 0x83, + 0xa3, 0xd9, 0xaa, 0x32, 0xd3, 0x9c, 0x16, 0x51, 0x99, 0xb7, 0xa1, 0x78, 0x4e, 0xac, 0x97, 0x22, + 0x3d, 0x82, 0xd1, 0x02, 0x33, 0xb0, 0xe4, 0x44, 0xfb, 0xfd, 0x0a, 0x72, 0xa2, 0xa4, 0xd0, 0x67, + 0x50, 0x18, 0xd2, 0x99, 0x17, 0xae, 0xdf, 0x9b, 0x9d, 0xa4, 0x5c, 0x71, 0x4f, 0x54, 0x27, 0x2b, + 0x60, 0x6d, 0x1f, 0xf2, 0x91, 0x0b, 0x3d, 0x58, 0x69, 0xa9, 0xdc, 0xbc, 0x75, 0xa5, 0xbc, 0x37, + 0x1f, 0xa0, 0x33, 0xcb, 0x9d, 0x89, 0x83, 0xca, 0x58, 0x4c, 0x6a, 0x7f, 0x95, 0x20, 0x8f, 0x59, + 0xc5, 0x06, 0x61, 0xe2, 0xe9, 0xca, 0x6e, 0x3c, 0x5d, 0xeb, 0x26, 0x4f, 0x6f, 0x34, 0x79, 0xdc, + 0xa7, 0x99, 0x44, 0x9f, 0xae, 0x59, 0x92, 0xbf, 0x93, 0xa5, 0x6c, 0x82, 0xa5, 0x98, 0xe5, 0x5c, + 0x82, 0xe5, 0x07, 0xb0, 0x3d, 0xf2, 0xe9, 0x84, 0x3f, 0x4e, 0xd4, 0xb7, 0xfc, 0x79, 0xa4, 0xa4, + 0x5b, 0xcc, 0x3a, 0x88, 0x8d, 0x9b, 0x04, 0x17, 0x36, 0x09, 0xae, 0x99, 0x50, 0xc0, 0x24, 0x98, + 0x52, 0x2f, 0x20, 0xef, 0xbc, 0x13, 0x02, 0xd9, 0xb6, 0x42, 0x8b, 0xdf, 0xa8, 0x8c, 0xf9, 0x18, + 0x3d, 0x04, 0x79, 0x48, 0x6d, 0x71, 0x9f, 0xed, 0x64, 0xbb, 0x1a, 0xbe, 0x4f, 0xfd, 0x16, 0xb5, + 0x09, 0xe6, 0x80, 0xda, 0x14, 0x94, 0x36, 0x3d, 0xf7, 0x5c, 0x6a, 0xd9, 0x47, 0x3e, 0x1d, 0xb3, + 0x17, 0xe4, 0x9d, 0x4a, 0xd8, 0x86, 0xfc, 0x8c, 0x6b, 0x65, 0xac, 0x85, 0xf7, 0x37, 0xbb, 0xf1, + 0x6a, 0x20, 0x21, 0xac, 0xb1, 0xce, 0x44, 0x4b, 0x6b, 0x7f, 0x97, 0x40, 0x7b, 0x37, 0x1a, 0x75, + 0xa0, 0x24, 0x90, 0x66, 0xe2, 0xa7, 0xa9, 0xfe, 0x3e, 0x1b, 0x71, 0x21, 0x80, 0xd9, 0x6a, 0xfc, + 0x9d, 0x2f, 0x6e, 0x42, 0x17, 0x33, 0xef, 0xa7, 0x8b, 0x0f, 0x61, 0x4b, 0x28, 0x42, 0xfc, 0x7f, + 0x21, 0xeb, 0x99, 0x7a, 0xb6, 0x99, 0x56, 0x52, 0xb8, 0x7c, 0x22, 0xda, 0x8c, 0xdb, 0x6b, 0x39, + 0x90, 0x8f, 0x1c, 0x6f, 0x5c, 0xab, 0x42, 0xb6, 0xe5, 0x52, 0x9e, 0xb0, 0x9c, 0x4f, 0xac, 0x80, + 0x7a, 0x31, 0x8f, 0x62, 0xb6, 0xf7, 0xb7, 0x34, 0x94, 0x12, 0xff, 0x7e, 0xe8, 0x09, 0x6c, 0xb7, + 0x0e, 0x8f, 0xfb, 0x03, 0x03, 0x9b, 0xad, 0x5e, 0x77, 0xbf, 0x73, 0xa0, 0xa4, 0xb4, 0x3b, 0x8b, + 0xa5, 0xae, 0x4e, 0xd6, 0xa0, 0xcd, 0xdf, 0xba, 0x2a, 0x64, 0x3b, 0xdd, 0xb6, 0xf1, 0x3b, 0x45, + 0xd2, 0x6e, 0x2e, 0x96, 0xba, 0x92, 0x00, 0x8a, 0x37, 0xf2, 0x13, 0x28, 0x73, 0x80, 0x79, 0x7c, + 0xd4, 0x6e, 0x0c, 0x0c, 0x25, 0xad, 0x69, 0x8b, 0xa5, 0xbe, 0x7b, 0x15, 0x17, 0x71, 0x7e, 0x0f, + 0xf2, 0xd8, 0xf8, 0xed, 0xb1, 0xd1, 0x1f, 0x28, 0x19, 0x6d, 0x77, 0xb1, 0xd4, 0x51, 0x02, 0x18, + 0xb7, 0xd4, 0x03, 0x28, 0x60, 0xa3, 0x7f, 0xd4, 0xeb, 0xf6, 0x0d, 0x45, 0xd6, 0x7e, 0xb0, 0x58, + 0xea, 0x37, 0x36, 0x50, 0x51, 0x95, 0xfe, 0x14, 0x76, 0xda, 0xbd, 0x2f, 0xba, 0x87, 0xbd, 0x46, + 0xdb, 0x3c, 0xc2, 0xbd, 0x03, 0x6c, 0xf4, 0xfb, 0x4a, 0x56, 0xab, 0x2e, 0x96, 0xfa, 0xed, 0x04, + 0xfe, 0x5a, 0xd1, 0x7d, 0x08, 0xf2, 0x51, 0xa7, 0x7b, 0xa0, 0xe4, 0xb4, 0x1b, 0x8b, 0xa5, 0xfe, + 0x41, 0x02, 0xca, 0x48, 0x65, 0x37, 0x6e, 0x1d, 0xf6, 0xfa, 0x86, 0x92, 0xbf, 0x76, 0x63, 0x4e, + 0xf6, 0xde, 0xef, 0x01, 0x5d, 0xff, 0x3b, 0x46, 0xf7, 0x41, 0xee, 0xf6, 0xba, 0x86, 0x92, 0x12, + 0xf7, 0xbf, 0x8e, 0xe8, 0x52, 0x8f, 0xa0, 0x1a, 0x64, 0x0e, 0xbf, 0xfc, 0x5c, 0x91, 0xb4, 0x1f, + 0x2e, 0x96, 0xfa, 0xad, 0xeb, 0xa0, 0xc3, 0x2f, 0x3f, 0xdf, 0xa3, 0x50, 0x4a, 0x06, 0xae, 0x41, + 0xe1, 0x99, 0x31, 0x68, 0xb4, 0x1b, 0x83, 0x86, 0x92, 0x12, 0x47, 0x8a, 0xdd, 0xcf, 0x48, 0x68, + 0xf1, 0x26, 0xbc, 0x03, 0xd9, 0xae, 0xf1, 0xdc, 0xc0, 0x8a, 0xa4, 0xed, 0x2c, 0x96, 0xfa, 0x56, + 0x0c, 0xe8, 0x92, 0x33, 0xe2, 0xa3, 0x0a, 0xe4, 0x1a, 0x87, 0x5f, 0x34, 0x5e, 0xf4, 0x95, 0xb4, + 0x86, 0x16, 0x4b, 0x7d, 0x3b, 0x76, 0x37, 0xdc, 0x73, 0x6b, 0x1e, 0xec, 0xfd, 0x47, 0x82, 0x72, + 0xf2, 0x8d, 0x43, 0x15, 0x90, 0xf7, 0x3b, 0x87, 0x46, 0xbc, 0x5d, 0xd2, 0xc7, 0xc6, 0xa8, 0x0e, + 0xc5, 0x76, 0x07, 0x1b, 0xad, 0x41, 0x0f, 0xbf, 0x88, 0xef, 0x92, 0x04, 0xb5, 0x1d, 0x9f, 0x17, + 0xf8, 0x1c, 0xfd, 0x1c, 0xca, 0xfd, 0x17, 0xcf, 0x0e, 0x3b, 0xdd, 0xdf, 0x98, 0x3c, 0x62, 0x5a, + 0x7b, 0xb8, 0x58, 0xea, 0x77, 0x37, 0xc0, 0x64, 0xea, 0x93, 0xa1, 0x15, 0x12, 0xbb, 0x2f, 0xde, + 0x6b, 0xe6, 0x2c, 0x48, 0xa8, 0x05, 0x3b, 0xf1, 0xd2, 0xf5, 0x66, 0x19, 0xed, 0x93, 0xc5, 0x52, + 0xff, 0xe8, 0x7b, 0xd7, 0xaf, 0x76, 0x2f, 0x48, 0xe8, 0x3e, 0xe4, 0xa3, 0x20, 0x71, 0x25, 0x25, + 0x97, 0x46, 0x0b, 0xf6, 0xfe, 0x22, 0x41, 0x71, 0x25, 0x57, 0x8c, 0xf0, 0x6e, 0xcf, 0x34, 0x30, + 0xee, 0xe1, 0x98, 0x81, 0x95, 0xb3, 0x4b, 0xf9, 0x10, 0xdd, 0x85, 0xfc, 0x81, 0xd1, 0x35, 0x70, + 0xa7, 0x15, 0x37, 0xc6, 0x0a, 0x72, 0x40, 0x3c, 0xe2, 0x3b, 0x43, 0xf4, 0x31, 0x94, 0xbb, 0x3d, + 0xb3, 0x7f, 0xdc, 0x7a, 0x1a, 0x5f, 0x9d, 0xef, 0x9f, 0x08, 0xd5, 0x9f, 0x0d, 0x4f, 0x39, 0x9f, + 0x7b, 0xac, 0x87, 0x9e, 0x37, 0x0e, 0x3b, 0x6d, 0x01, 0xcd, 0x68, 0xea, 0x62, 0xa9, 0xdf, 0x5c, + 0x41, 0xa3, 0x47, 0x9a, 0x61, 0xf7, 0x6c, 0xa8, 0x7c, 0xbf, 0x30, 0x21, 0x1d, 0x72, 0x8d, 0xa3, + 0x23, 0xa3, 0xdb, 0x8e, 0x4f, 0xbf, 0xf6, 0x35, 0xa6, 0x53, 0xe2, 0xd9, 0x0c, 0xb1, 0xdf, 0xc3, + 0x07, 0xc6, 0x20, 0x3e, 0xfc, 0x1a, 0xb1, 0x4f, 0xd9, 0xcf, 0x52, 0xb3, 0xfe, 0xfa, 0x9b, 0x4a, + 0xea, 0xcd, 0x37, 0x95, 0xd4, 0xeb, 0xcb, 0x8a, 0xf4, 0xe6, 0xb2, 0x22, 0xfd, 0xeb, 0xb2, 0x92, + 0xfa, 0xf6, 0xb2, 0x22, 0xfd, 0xf1, 0x6d, 0x25, 0xf5, 0xf5, 0xdb, 0x8a, 0xf4, 0xe6, 0x6d, 0x25, + 0xf5, 0x8f, 0xb7, 0x95, 0xd4, 0x49, 0x8e, 0x8b, 0xda, 0x67, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, + 0x39, 0xd2, 0xc3, 0x6e, 0x32, 0x0f, 0x00, 0x00, } func (m *Hello) Marshal() (dAtA []byte, err error) { @@ -1459,6 +1461,15 @@ func (m *FileInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0xc0 } + if len(m.BlocksHash) > 0 { + i -= len(m.BlocksHash) + copy(dAtA[i:], m.BlocksHash) + i = encodeVarintBep(dAtA, i, uint64(len(m.BlocksHash))) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x92 + } if len(m.SymlinkTarget) > 0 { i -= len(m.SymlinkTarget) copy(dAtA[i:], m.SymlinkTarget) @@ -2185,6 +2196,10 @@ func (m *FileInfo) ProtoSize() (n int) { if l > 0 { n += 2 + l + sovBep(uint64(l)) } + l = len(m.BlocksHash) + if l > 0 { + n += 2 + l + sovBep(uint64(l)) + } if m.LocalFlags != 0 { n += 2 + sovBep(uint64(m.LocalFlags)) } @@ -3835,6 +3850,40 @@ func (m *FileInfo) Unmarshal(dAtA []byte) error { } m.SymlinkTarget = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 18: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field BlocksHash", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBep + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthBep + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBep + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.BlocksHash = append(m.BlocksHash[:0], dAtA[iNdEx:postIndex]...) + if m.BlocksHash == nil { + m.BlocksHash = []byte{} + } + iNdEx = postIndex case 1000: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field LocalFlags", wireType) diff --git a/lib/protocol/bep.proto b/lib/protocol/bep.proto index 2450d6589..0e848006e 100644 --- a/lib/protocol/bep.proto +++ b/lib/protocol/bep.proto @@ -106,8 +106,9 @@ message FileInfo { uint64 modified_by = 12 [(gogoproto.customtype) = "ShortID", (gogoproto.nullable) = false]; Vector version = 9 [(gogoproto.nullable) = false]; int64 sequence = 10; - repeated BlockInfo Blocks = 16 [(gogoproto.nullable) = false]; + repeated BlockInfo blocks = 16 [(gogoproto.nullable) = false]; string symlink_target = 17; + bytes blocks_hash = 18; FileInfoType type = 2; uint32 permissions = 4; int32 modified_ns = 11; diff --git a/lib/protocol/bep_extensions.go b/lib/protocol/bep_extensions.go index bb2298014..9a11a07bb 100644 --- a/lib/protocol/bep_extensions.go +++ b/lib/protocol/bep_extensions.go @@ -14,6 +14,7 @@ import ( "time" "github.com/syncthing/syncthing/lib/rand" + "github.com/syncthing/syncthing/lib/sha256" ) const ( @@ -328,3 +329,11 @@ func (f Folder) Description() string { } return fmt.Sprintf("%q (%s)", f.Label, f.ID) } + +func BlocksHash(bs []BlockInfo) []byte { + h := sha256.New() + for _, b := range bs { + _, _ = h.Write(b.Hash) + } + return h.Sum(nil) +} diff --git a/lib/scanner/blockqueue.go b/lib/scanner/blockqueue.go index e17078aba..6c22714c2 100644 --- a/lib/scanner/blockqueue.go +++ b/lib/scanner/blockqueue.go @@ -111,6 +111,7 @@ func (ph *parallelHasher) hashFiles(ctx context.Context) { } f.Blocks = blocks + f.BlocksHash = protocol.BlocksHash(blocks) // The size we saw when initially deciding to hash the file // might not have been the size it actually had when we hashed