lib/db: Deduplicate block lists in database (fixes #5898) (#6283)

* lib/db: Deduplicate block lists in database (fixes #5898)

This moves the block list in the database out from being just a field on
the FileInfo to being an object of its own. When putting a FileInfo we
marshal the block list separately and store it keyed by the sha256 of
the marshalled block list. When getting, if we are not doing a
"truncated" get, we do an extra read and unmarshal for the block list.

Old block lists are cleared out by a periodic GC sweep. The alternative
would be to use refcounting, but:

- There is a larger risk of getting that wrong and either dropping a
  block list in error or keeping them around forever.

- It's tricky with our current database, as we don't have dirty reads.
  This means that if we update two FileInfos with identical block lists in
  the same transaction we can't just do read/modify/write for the ref
  counters as we wouldn't see our own first update. See above about
  tracking this and risks about getting it wrong.

GC uses a bloom filter for keys to avoid heavy RAM usage. GC can't run
concurrently with FileInfo updates so there is a new lock around those
operation at the lowlevel.

The end result is a much more compact database, especially for setups
with many peers where files get duplicated many times.

This is per-key-class stats for a large database I'm currently working
with, under the current schema:

```
 0x00:  9138161 items, 870876 KB keys + 7397482 KB data, 95 B +  809 B avg, 1637651 B max
 0x01:   185656 items,  10388 KB keys + 1790909 KB data, 55 B + 9646 B avg,  924525 B max
 0x02:   916890 items,  84795 KB keys +    3667 KB data, 92 B +    4 B avg,     192 B max
 0x03:      384 items,     27 KB keys +       5 KB data, 72 B +   15 B avg,      87 B max
 0x04:     1109 items,     17 KB keys +      17 KB data, 15 B +   15 B avg,      69 B max
 0x06:      383 items,      3 KB keys +       0 KB data,  9 B +    2 B avg,      18 B max
 0x07:      510 items,      4 KB keys +      12 KB data,  9 B +   24 B avg,      41 B max
 0x08:     1349 items,     12 KB keys +      10 KB data,  9 B +    8 B avg,      17 B max
 0x09:      194 items,      0 KB keys +     123 KB data,  5 B +  634 B avg,   11484 B max
 0x0a:        3 items,      0 KB keys +       0 KB data, 14 B +    7 B avg,      30 B max
 0x0b:   181836 items,   2363 KB keys +   10694 KB data, 13 B +   58 B avg,     173 B max
 Total 10426475 items, 968490 KB keys + 9202925 KB data.
```

Note 7.4 GB of data in class 00, total size 9.2 GB. After running the
migration we get this instead:

```
 0x00:  9138161 items, 870876 KB keys + 2611392 KB data, 95 B +  285 B avg,    4788 B max
 0x01:   185656 items,  10388 KB keys + 1790909 KB data, 55 B + 9646 B avg,  924525 B max
 0x02:   916890 items,  84795 KB keys +    3667 KB data, 92 B +    4 B avg,     192 B max
 0x03:      384 items,     27 KB keys +       5 KB data, 72 B +   15 B avg,      87 B max
 0x04:     1109 items,     17 KB keys +      17 KB data, 15 B +   15 B avg,      69 B max
 0x06:      383 items,      3 KB keys +       0 KB data,  9 B +    2 B avg,      18 B max
 0x07:      510 items,      4 KB keys +      12 KB data,  9 B +   24 B avg,      41 B max
 0x09:      194 items,      0 KB keys +     123 KB data,  5 B +  634 B avg,   11484 B max
 0x0a:        3 items,      0 KB keys +       0 KB data, 14 B +   17 B avg,      51 B max
 0x0b:   181836 items,   2363 KB keys +   10694 KB data, 13 B +   58 B avg,     173 B max
 0x0d:    44282 items,   1461 KB keys +   61081 KB data, 33 B + 1379 B avg, 1637399 B max
 Total 10469408 items, 969939 KB keys + 4477905 KB data.
```

Class 00 is now down to 2.6 GB, with just 61 MB added in class 0d.

There will be some additional reads in some cases which theoretically
hurts performance, but this will be more than compensated for by smaller
writes and better compaction.

On my own home setup which just has three devices and a handful of
folders the difference is smaller in absolute numbers of course, but
still less than half the old size:

```
 0x00:  297122 items,  20894 KB keys + 306860 KB data, 70 B + 1032 B avg, 103237 B max
 0x01:  115299 items,   7738 KB keys +  17542 KB data, 67 B +  152 B avg,    419 B max
 0x02: 1430537 items, 121223 KB keys +   5722 KB data, 84 B +    4 B avg,    253 B max
 ...
 Total 1947412 items, 151268 KB keys + 337485 KB data.
```

to:

```
 0x00:  297122 items,  20894 KB keys +  37038 KB data, 70 B +  124 B avg,    520 B max
 0x01:  115299 items,   7738 KB keys +  17542 KB data, 67 B +  152 B avg,    419 B max
 0x02: 1430537 items, 121223 KB keys +   5722 KB data, 84 B +    4 B avg,    253 B max
 ...
 0x0d:   18041 items,    595 KB keys +  71964 KB data, 33 B + 3988 B avg, 101109 B max
 Total 1965447 items, 151863 KB keys + 139628 KB data.
```

* wip

* wip

* wip

* wip
This commit is contained in:
Jakob Borg 2020-01-24 08:35:44 +01:00 committed by GitHub
parent 17b441c993
commit 8fc2dfad0c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 1002 additions and 222 deletions

58
cmd/stindex/accounting.go Normal file
View File

@ -0,0 +1,58 @@
// Copyright (C) 2020 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"fmt"
"log"
"os"
"text/tabwriter"
"github.com/syncthing/syncthing/lib/db/backend"
)
// account prints key and data size statistics per class
func account(ldb backend.Backend) {
it, err := ldb.NewPrefixIterator(nil)
if err != nil {
log.Fatal(err)
}
var ksizes [256]int
var dsizes [256]int
var counts [256]int
var max [256]int
for it.Next() {
key := it.Key()
t := key[0]
ds := len(it.Value())
ks := len(key)
s := ks + ds
counts[t]++
ksizes[t] += ks
dsizes[t] += ds
if s > max[t] {
max[t] = s
}
}
tw := tabwriter.NewWriter(os.Stdout, 1, 1, 1, ' ', tabwriter.AlignRight)
toti, totds, totks := 0, 0, 0
for t := range ksizes {
if ksizes[t] > 0 {
// yes metric kilobytes 🤘
fmt.Fprintf(tw, "0x%02x:\t%d items,\t%d KB keys +\t%d KB data,\t%d B +\t%d B avg,\t%d B max\t\n", t, counts[t], ksizes[t]/1000, dsizes[t]/1000, ksizes[t]/counts[t], dsizes[t]/counts[t], max[t])
toti += counts[t]
totds += dsizes[t]
totks += ksizes[t]
}
}
fmt.Fprintf(tw, "Total\t%d items,\t%d KB keys +\t%d KB data.\t\n", toti, totks/1000, totds/1000)
tw.Flush()
}

View File

@ -35,15 +35,18 @@ func main() {
log.Fatal(err)
}
if mode == "dump" {
switch mode {
case "dump":
dump(ldb)
} else if mode == "dumpsize" {
case "dumpsize":
dumpsize(ldb)
} else if mode == "idxck" {
case "idxck":
if !idxck(ldb) {
os.Exit(1)
}
} else {
case "account":
account(ldb)
default:
fmt.Println("Unknown mode")
}
}

3
go.mod
View File

@ -35,11 +35,14 @@ require (
github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563
github.com/sasha-s/go-deadlock v0.2.0
github.com/shirou/gopsutil v0.0.0-20190714054239-47ef3260b6bf
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/syncthing/notify v0.0.0-20190709140112-69c7a957d3e2
github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d
github.com/thejerf/suture v3.0.2+incompatible
github.com/urfave/cli v1.22.2
github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0
github.com/willf/bitset v1.1.10 // indirect
github.com/willf/bloom v2.0.3+incompatible
golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297
golang.org/x/text v0.3.2

10
go.sum
View File

@ -8,8 +8,10 @@ github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrU
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@ -178,7 +180,10 @@ github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
@ -199,6 +204,10 @@ github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo=
github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0 h1:okhMind4q9H1OxF44gNegWkiP4H/gsTFLalHFa4OOUI=
github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0/go.mod h1:TTbGUfE+cXXceWtbTHq6lqcTvYPBKLNejBEbnUsQJtU=
github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc=
github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
github.com/willf/bloom v2.0.3+incompatible h1:QDacWdqcAUI1MPOwIQZRy9kOR7yxfyEmxX8Wdm2/JPA=
github.com/willf/bloom v2.0.3+incompatible/go.mod h1:MmAltL9pDMNTrvUkxdg0k0q5I0suxmuwp3KbyrZLOZ8=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
@ -246,6 +255,7 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM=
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=

View File

@ -100,6 +100,7 @@ type Backend interface {
NewReadTransaction() (ReadTransaction, error)
NewWriteTransaction() (WriteTransaction, error)
Close() error
Compact() error
}
type Tuning int

View File

@ -81,6 +81,10 @@ func (b *leveldbBackend) Delete(key []byte) error {
return wrapLeveldbErr(b.ldb.Delete(key, nil))
}
func (b *leveldbBackend) Compact() error {
return wrapLeveldbErr(b.ldb.CompactRange(util.Range{}))
}
// leveldbSnapshot implements backend.ReadTransaction
type leveldbSnapshot struct {
snap *leveldb.Snapshot

View File

@ -165,7 +165,7 @@ func TestUpdate0to3(t *testing.T) {
folder := []byte(update0to3Folder)
if err := updater.updateSchema0to1(); err != nil {
if err := updater.updateSchema0to1(0); err != nil {
t.Fatal(err)
}
@ -187,7 +187,7 @@ func TestUpdate0to3(t *testing.T) {
t.Error("Invalid file wasn't added to global list")
}
if err := updater.updateSchema1to2(); err != nil {
if err := updater.updateSchema1to2(1); err != nil {
t.Fatal(err)
}
@ -214,7 +214,7 @@ func TestUpdate0to3(t *testing.T) {
t.Error("Local file wasn't added to sequence bucket", err)
}
if err := updater.updateSchema2to3(); err != nil {
if err := updater.updateSchema2to3(2); err != nil {
t.Fatal(err)
}

View File

@ -59,6 +59,9 @@ const (
// KeyTypeNeed <int32 folder ID> <file name> = <nothing>
KeyTypeNeed = 12
// KeyTypeBlockList <block list hash> = BlockList
KeyTypeBlockList = 13
)
type keyer interface {
@ -93,6 +96,9 @@ type keyer interface {
// Folder metadata
GenerateFolderMetaKey(key, folder []byte) (folderMetaKey, error)
// Block lists
GenerateBlockListKey(key []byte, hash []byte) blockListKey
}
// defaultKeyer implements our key scheme. It needs folder and device
@ -281,6 +287,19 @@ func (k defaultKeyer) GenerateFolderMetaKey(key, folder []byte) (folderMetaKey,
return key, nil
}
type blockListKey []byte
func (k defaultKeyer) GenerateBlockListKey(key []byte, hash []byte) blockListKey {
key = resize(key, keyPrefixLen+len(hash))
key[0] = KeyTypeBlockList
copy(key[keyPrefixLen:], hash)
return key
}
func (k blockListKey) BlocksHash() []byte {
return k[keyPrefixLen:]
}
// resize returns a byte slice of the specified size, reusing bs if possible
func resize(bs []byte, size int) []byte {
if cap(bs) < size {

View File

@ -9,9 +9,23 @@ package db
import (
"bytes"
"encoding/binary"
"time"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sync"
"github.com/willf/bloom"
)
const (
// We set the bloom filter capacity to handle 100k individual block lists
// with a false positive probability of 1% for the first pass. Once we know
// how many block lists we have we will use that number instead, if it's
// more than 100k. For fewer than 100k block lists we will just get better
// false positive rate instead.
blockGCBloomCapacity = 100000
blockGCBloomFalsePositiveRate = 0.01 // 1%
blockGCInterval = 13 * time.Hour
)
// Lowlevel is the lowest level database interface. It has a very simple
@ -24,6 +38,9 @@ type Lowlevel struct {
folderIdx *smallIndex
deviceIdx *smallIndex
keyer keyer
gcMut sync.RWMutex
gcKeyCount int
gcStop chan struct{}
}
func NewLowlevel(backend backend.Backend) *Lowlevel {
@ -31,11 +48,19 @@ func NewLowlevel(backend backend.Backend) *Lowlevel {
Backend: backend,
folderIdx: newSmallIndex(backend, []byte{KeyTypeFolderIdx}),
deviceIdx: newSmallIndex(backend, []byte{KeyTypeDeviceIdx}),
gcMut: sync.NewRWMutex(),
gcStop: make(chan struct{}),
}
db.keyer = newDefaultKeyer(db.folderIdx, db.deviceIdx)
go db.gcRunner()
return db
}
func (db *Lowlevel) Close() error {
close(db.gcStop)
return db.Backend.Close()
}
// ListFolders returns the list of folders currently in the database
func (db *Lowlevel) ListFolders() []string {
return db.folderIdx.Values()
@ -44,6 +69,9 @@ func (db *Lowlevel) ListFolders() []string {
// updateRemoteFiles adds a list of fileinfos to the database and updates the
// global versionlist and metadata.
func (db *Lowlevel) updateRemoteFiles(folder, device []byte, fs []protocol.FileInfo, meta *metadataTracker) error {
db.gcMut.RLock()
defer db.gcMut.RUnlock()
t, err := db.newReadWriteTransaction()
if err != nil {
return err
@ -73,7 +101,7 @@ func (db *Lowlevel) updateRemoteFiles(folder, device []byte, fs []protocol.FileI
meta.addFile(devID, f)
l.Debugf("insert; folder=%q device=%v %v", folder, devID, f)
if err := t.Put(dk, mustMarshal(&f)); err != nil {
if err := t.putFile(dk, f); err != nil {
return err
}
@ -97,6 +125,9 @@ func (db *Lowlevel) updateRemoteFiles(folder, device []byte, fs []protocol.FileI
// updateLocalFiles adds fileinfos to the db, and updates the global versionlist,
// metadata, sequence and blockmap buckets.
func (db *Lowlevel) updateLocalFiles(folder []byte, fs []protocol.FileInfo, meta *metadataTracker) error {
db.gcMut.RLock()
defer db.gcMut.RUnlock()
t, err := db.newReadWriteTransaction()
if err != nil {
return err
@ -151,7 +182,7 @@ func (db *Lowlevel) updateLocalFiles(folder []byte, fs []protocol.FileInfo, meta
meta.addFile(protocol.LocalDeviceID, f)
l.Debugf("insert (local); folder=%q %v", folder, f)
if err := t.Put(dk, mustMarshal(&f)); err != nil {
if err := t.putFile(dk, f); err != nil {
return err
}
@ -195,6 +226,9 @@ func (db *Lowlevel) updateLocalFiles(folder []byte, fs []protocol.FileInfo, meta
}
func (db *Lowlevel) dropFolder(folder []byte) error {
db.gcMut.RLock()
defer db.gcMut.RUnlock()
t, err := db.newReadWriteTransaction()
if err != nil {
return err
@ -250,6 +284,9 @@ func (db *Lowlevel) dropFolder(folder []byte) error {
}
func (db *Lowlevel) dropDeviceFolder(device, folder []byte, meta *metadataTracker) error {
db.gcMut.RLock()
defer db.gcMut.RUnlock()
t, err := db.newReadWriteTransaction()
if err != nil {
return err
@ -425,16 +462,100 @@ func (db *Lowlevel) dropPrefix(prefix []byte) error {
return t.commit()
}
func unmarshalTrunc(bs []byte, truncate bool) (FileIntf, error) {
if truncate {
var tf FileInfoTruncated
err := tf.Unmarshal(bs)
return tf, err
func (db *Lowlevel) gcRunner() {
t := time.NewTicker(blockGCInterval)
defer t.Stop()
for {
select {
case <-db.gcStop:
return
case <-t.C:
if err := db.gcBlocks(); err != nil {
l.Warnln("Database block GC failed:", err)
}
}
}
}
func (db *Lowlevel) gcBlocks() error {
// The block GC uses a bloom filter to track used block lists. This means
// iterating over all items, adding their block lists to the filter, then
// iterating over the block lists and removing those that don't match the
// filter. The filter will give false positives so we will keep around one
// percent of block lists that we don't really need (at most).
//
// Block GC needs to run when there are no modifications to the FileInfos or
// block lists.
db.gcMut.Lock()
defer db.gcMut.Unlock()
t, err := db.newReadWriteTransaction()
if err != nil {
return err
}
defer t.Release()
// Set up the bloom filter with the initial capacity and false positive
// rate, or higher capacity if we've done this before and seen lots of block
// lists.
capacity := blockGCBloomCapacity
if db.gcKeyCount > capacity {
capacity = db.gcKeyCount
}
filter := bloom.NewWithEstimates(uint(capacity), blockGCBloomFalsePositiveRate)
// Iterate the FileInfos, unmarshal the blocks hashes and add them to
// the filter.
it, err := db.NewPrefixIterator([]byte{KeyTypeDevice})
if err != nil {
return err
}
for it.Next() {
var bl BlocksHashOnly
if err := bl.Unmarshal(it.Value()); err != nil {
return err
}
filter.Add(bl.BlocksHash)
}
it.Release()
if err := it.Error(); err != nil {
return err
}
var tf protocol.FileInfo
err := tf.Unmarshal(bs)
return tf, err
// Iterate over block lists, removing keys with hashes that don't match
// the filter.
it, err = db.NewPrefixIterator([]byte{KeyTypeBlockList})
if err != nil {
return err
}
matched := 0
for it.Next() {
key := blockListKey(it.Key())
if filter.Test(key.BlocksHash()) {
matched++
continue
}
if err := t.Delete(key); err != nil {
return err
}
}
it.Release()
if err := it.Error(); err != nil {
return err
}
// Remember the number of unique keys we kept until the next pass.
db.gcKeyCount = matched
if err := t.Commit(); err != nil {
return err
}
return db.Compact()
}
func unmarshalVersionList(data []byte) (VersionList, bool) {

View File

@ -22,9 +22,10 @@ import (
// 5: v0.14.49
// 6: v0.14.50
// 7: v0.14.53
// 8: v1.4.0
const (
dbVersion = 7
dbMinSyncthingVersion = "v0.14.53"
dbVersion = 8
dbMinSyncthingVersion = "v1.4.0"
)
type databaseDowngradeError struct {
@ -68,36 +69,27 @@ func (db *schemaUpdater) updateSchema() error {
return nil
}
if prevVersion < 1 {
if err := db.updateSchema0to1(); err != nil {
type migration struct {
schemaVersion int64
migration func(prevVersion int) error
}
var migrations = []migration{
{1, db.updateSchema0to1},
{2, db.updateSchema1to2},
{3, db.updateSchema2to3},
{5, db.updateSchemaTo5},
{6, db.updateSchema5to6},
{7, db.updateSchema6to7},
{8, db.updateSchema7to8},
}
for _, m := range migrations {
if prevVersion < m.schemaVersion {
l.Infof("Migrating database to schema version %d...", m.schemaVersion)
if err := m.migration(int(prevVersion)); err != nil {
return err
}
}
if prevVersion < 2 {
if err := db.updateSchema1to2(); err != nil {
return err
}
}
if prevVersion < 3 {
if err := db.updateSchema2to3(); err != nil {
return err
}
}
// This update fixes problems existing in versions 3 and 4
if prevVersion == 3 || prevVersion == 4 {
if err := db.updateSchemaTo5(); err != nil {
return err
}
}
if prevVersion < 6 {
if err := db.updateSchema5to6(); err != nil {
return err
}
}
if prevVersion < 7 {
if err := db.updateSchema6to7(); err != nil {
return err
}
}
if err := miscDB.PutInt64("dbVersion", dbVersion); err != nil {
@ -107,10 +99,11 @@ func (db *schemaUpdater) updateSchema() error {
return err
}
return nil
l.Infoln("Compacting database after migration...")
return db.Compact()
}
func (db *schemaUpdater) updateSchema0to1() error {
func (db *schemaUpdater) updateSchema0to1(_ int) error {
t, err := db.newReadWriteTransaction()
if err != nil {
return err
@ -216,7 +209,7 @@ func (db *schemaUpdater) updateSchema0to1() error {
// updateSchema1to2 introduces a sequenceKey->deviceKey bucket for local items
// to allow iteration in sequence order (simplifies sending indexes).
func (db *schemaUpdater) updateSchema1to2() error {
func (db *schemaUpdater) updateSchema1to2(_ int) error {
t, err := db.newReadWriteTransaction()
if err != nil {
return err
@ -251,7 +244,7 @@ func (db *schemaUpdater) updateSchema1to2() error {
}
// updateSchema2to3 introduces a needKey->nil bucket for locally needed files.
func (db *schemaUpdater) updateSchema2to3() error {
func (db *schemaUpdater) updateSchema2to3(_ int) error {
t, err := db.newReadWriteTransaction()
if err != nil {
return err
@ -302,7 +295,11 @@ func (db *schemaUpdater) updateSchema2to3() error {
// release candidates (dbVersion 3 and 4)
// https://github.com/syncthing/syncthing/issues/5007
// https://github.com/syncthing/syncthing/issues/5053
func (db *schemaUpdater) updateSchemaTo5() error {
func (db *schemaUpdater) updateSchemaTo5(prevVersion int) error {
if prevVersion != 3 && prevVersion != 4 {
return nil
}
t, err := db.newReadWriteTransaction()
if err != nil {
return err
@ -321,10 +318,10 @@ func (db *schemaUpdater) updateSchemaTo5() error {
return err
}
return db.updateSchema2to3()
return db.updateSchema2to3(2)
}
func (db *schemaUpdater) updateSchema5to6() error {
func (db *schemaUpdater) updateSchema5to6(_ int) error {
// For every local file with the Invalid bit set, clear the Invalid bit and
// set LocalFlags = FlagLocalIgnored.
@ -369,7 +366,7 @@ func (db *schemaUpdater) updateSchema5to6() error {
// updateSchema6to7 checks whether all currently locally needed files are really
// needed and removes them if not.
func (db *schemaUpdater) updateSchema6to7() error {
func (db *schemaUpdater) updateSchema6to7(_ int) error {
t, err := db.newReadWriteTransaction()
if err != nil {
return err
@ -423,3 +420,36 @@ func (db *schemaUpdater) updateSchema6to7() error {
}
return t.commit()
}
func (db *schemaUpdater) updateSchema7to8(_ int) error {
// Loads and rewrites all files with blocks, to deduplicate block lists.
t, err := db.newReadWriteTransaction()
if err != nil {
return err
}
defer t.close()
it, err := t.NewPrefixIterator([]byte{KeyTypeDevice})
if err != nil {
return err
}
for it.Next() {
var fi protocol.FileInfo
if err := fi.Unmarshal(it.Value()); err != nil {
return err
}
if fi.Blocks == nil {
continue
}
if err := t.putFile(it.Key(), fi); err != nil {
return err
}
}
it.Release()
if err := it.Error(); err != nil {
return err
}
return t.commit()
}

View File

@ -110,6 +110,7 @@ type FileInfoTruncated struct {
Sequence int64 `protobuf:"varint,10,opt,name=sequence,proto3" json:"sequence,omitempty"`
// repeated BlockInfo Blocks = 16
SymlinkTarget string `protobuf:"bytes,17,opt,name=symlink_target,json=symlinkTarget,proto3" json:"symlink_target,omitempty"`
BlocksHash []byte `protobuf:"bytes,18,opt,name=blocks_hash,json=blocksHash,proto3" json:"blocks_hash,omitempty"`
Type protocol.FileInfoType `protobuf:"varint,2,opt,name=type,proto3,enum=protocol.FileInfoType" json:"type,omitempty"`
Permissions uint32 `protobuf:"varint,4,opt,name=permissions,proto3" json:"permissions,omitempty"`
ModifiedNs int32 `protobuf:"varint,11,opt,name=modified_ns,json=modifiedNs,proto3" json:"modified_ns,omitempty"`
@ -153,6 +154,82 @@ func (m *FileInfoTruncated) XXX_DiscardUnknown() {
var xxx_messageInfo_FileInfoTruncated proto.InternalMessageInfo
// BlockList is the structure used to store block lists
type BlockList struct {
Blocks []protocol.BlockInfo `protobuf:"bytes,1,rep,name=Blocks,proto3" json:"Blocks"`
}
func (m *BlockList) Reset() { *m = BlockList{} }
func (m *BlockList) String() string { return proto.CompactTextString(m) }
func (*BlockList) ProtoMessage() {}
func (*BlockList) Descriptor() ([]byte, []int) {
return fileDescriptor_e774e8f5f348d14d, []int{3}
}
func (m *BlockList) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *BlockList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_BlockList.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalToSizedBuffer(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (m *BlockList) XXX_Merge(src proto.Message) {
xxx_messageInfo_BlockList.Merge(m, src)
}
func (m *BlockList) XXX_Size() int {
return m.ProtoSize()
}
func (m *BlockList) XXX_DiscardUnknown() {
xxx_messageInfo_BlockList.DiscardUnknown(m)
}
var xxx_messageInfo_BlockList proto.InternalMessageInfo
// BlocksHashOnly is used to only unmarshal the block list hash from a FileInfo
type BlocksHashOnly struct {
BlocksHash []byte `protobuf:"bytes,18,opt,name=blocks_hash,json=blocksHash,proto3" json:"blocks_hash,omitempty"`
}
func (m *BlocksHashOnly) Reset() { *m = BlocksHashOnly{} }
func (m *BlocksHashOnly) String() string { return proto.CompactTextString(m) }
func (*BlocksHashOnly) ProtoMessage() {}
func (*BlocksHashOnly) Descriptor() ([]byte, []int) {
return fileDescriptor_e774e8f5f348d14d, []int{4}
}
func (m *BlocksHashOnly) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *BlocksHashOnly) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_BlocksHashOnly.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalToSizedBuffer(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (m *BlocksHashOnly) XXX_Merge(src proto.Message) {
xxx_messageInfo_BlocksHashOnly.Merge(m, src)
}
func (m *BlocksHashOnly) XXX_Size() int {
return m.ProtoSize()
}
func (m *BlocksHashOnly) XXX_DiscardUnknown() {
xxx_messageInfo_BlocksHashOnly.DiscardUnknown(m)
}
var xxx_messageInfo_BlocksHashOnly proto.InternalMessageInfo
// For each folder and device we keep one of these to track the current
// counts and sequence. We also keep one for the global state of the folder.
type Counts struct {
@ -170,7 +247,7 @@ func (m *Counts) Reset() { *m = Counts{} }
func (m *Counts) String() string { return proto.CompactTextString(m) }
func (*Counts) ProtoMessage() {}
func (*Counts) Descriptor() ([]byte, []int) {
return fileDescriptor_e774e8f5f348d14d, []int{3}
return fileDescriptor_e774e8f5f348d14d, []int{5}
}
func (m *Counts) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -208,7 +285,7 @@ func (m *CountsSet) Reset() { *m = CountsSet{} }
func (m *CountsSet) String() string { return proto.CompactTextString(m) }
func (*CountsSet) ProtoMessage() {}
func (*CountsSet) Descriptor() ([]byte, []int) {
return fileDescriptor_e774e8f5f348d14d, []int{4}
return fileDescriptor_e774e8f5f348d14d, []int{6}
}
func (m *CountsSet) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -241,6 +318,8 @@ func init() {
proto.RegisterType((*FileVersion)(nil), "db.FileVersion")
proto.RegisterType((*VersionList)(nil), "db.VersionList")
proto.RegisterType((*FileInfoTruncated)(nil), "db.FileInfoTruncated")
proto.RegisterType((*BlockList)(nil), "db.BlockList")
proto.RegisterType((*BlocksHashOnly)(nil), "db.BlocksHashOnly")
proto.RegisterType((*Counts)(nil), "db.Counts")
proto.RegisterType((*CountsSet)(nil), "db.CountsSet")
}
@ -248,50 +327,54 @@ func init() {
func init() { proto.RegisterFile("structs.proto", fileDescriptor_e774e8f5f348d14d) }
var fileDescriptor_e774e8f5f348d14d = []byte{
// 683 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0x4f, 0x6f, 0xda, 0x4e,
0x10, 0xc5, 0xc1, 0x10, 0x18, 0x20, 0xbf, 0x64, 0x15, 0x45, 0x16, 0xd2, 0xcf, 0x58, 0x54, 0x95,
0xac, 0x1e, 0xa0, 0x4d, 0x6e, 0xed, 0x8d, 0x46, 0x91, 0x90, 0xaa, 0xb6, 0x5a, 0xa2, 0x9c, 0x2a,
0x21, 0xff, 0x59, 0xc8, 0x2a, 0xc6, 0x4b, 0xbc, 0x4b, 0x22, 0xf2, 0x29, 0x7a, 0xec, 0x31, 0x1f,
0x27, 0xc7, 0x1c, 0xab, 0x1e, 0x50, 0x0a, 0x3d, 0xf4, 0x63, 0x54, 0xbb, 0x6b, 0x1b, 0x37, 0xa7,
0xde, 0xe6, 0xbd, 0x19, 0x7b, 0x66, 0xde, 0xbc, 0x85, 0x16, 0x17, 0xc9, 0x22, 0x10, 0xbc, 0x37,
0x4f, 0x98, 0x60, 0x68, 0x27, 0xf4, 0xdb, 0x2f, 0x12, 0x32, 0x67, 0xbc, 0xaf, 0x08, 0x7f, 0x31,
0xe9, 0x4f, 0xd9, 0x94, 0x29, 0xa0, 0x22, 0x5d, 0xd8, 0x3e, 0x8a, 0xa8, 0xaf, 0x4b, 0x02, 0x16,
0xf5, 0x7d, 0x32, 0xd7, 0x7c, 0xf7, 0x1a, 0x1a, 0x67, 0x34, 0x22, 0x17, 0x24, 0xe1, 0x94, 0xc5,
0xe8, 0x35, 0xec, 0xde, 0xe8, 0xd0, 0x32, 0x1c, 0xc3, 0x6d, 0x1c, 0xef, 0xf7, 0xb2, 0x8f, 0x7a,
0x17, 0x24, 0x10, 0x2c, 0x19, 0x98, 0x0f, 0xab, 0x4e, 0x09, 0x67, 0x65, 0xe8, 0x08, 0xaa, 0x21,
0xb9, 0xa1, 0x01, 0xb1, 0x76, 0x1c, 0xc3, 0x6d, 0xe2, 0x14, 0x21, 0x0b, 0x76, 0x69, 0x7c, 0xe3,
0x45, 0x34, 0xb4, 0xca, 0x8e, 0xe1, 0xd6, 0x70, 0x06, 0xbb, 0x67, 0xd0, 0x48, 0xdb, 0x7d, 0xa0,
0x5c, 0xa0, 0x37, 0x50, 0x4b, 0xff, 0xc5, 0x2d, 0xc3, 0x29, 0xbb, 0x8d, 0xe3, 0xff, 0x7a, 0xa1,
0xdf, 0x2b, 0x4c, 0x95, 0xb6, 0xcc, 0xcb, 0xde, 0x9a, 0xdf, 0xee, 0x3b, 0xa5, 0xee, 0x93, 0x09,
0x07, 0xb2, 0x6a, 0x18, 0x4f, 0xd8, 0x79, 0xb2, 0x88, 0x03, 0x4f, 0x90, 0x10, 0x21, 0x30, 0x63,
0x6f, 0x46, 0xd4, 0xf8, 0x75, 0xac, 0x62, 0xc9, 0x71, 0x7a, 0x47, 0xd4, 0x20, 0x65, 0xac, 0x62,
0xf4, 0x3f, 0xc0, 0x8c, 0x85, 0x74, 0x42, 0x49, 0x38, 0xe6, 0x56, 0x45, 0x65, 0xea, 0x19, 0x33,
0x42, 0x5f, 0xa0, 0x91, 0xa7, 0xfd, 0xa5, 0xd5, 0x74, 0x0c, 0xd7, 0x1c, 0xbc, 0x93, 0x73, 0xfc,
0x58, 0x75, 0x4e, 0xa6, 0x54, 0x5c, 0x2e, 0xfc, 0x5e, 0xc0, 0x66, 0x7d, 0xbe, 0x8c, 0x03, 0x71,
0x49, 0xe3, 0x69, 0x21, 0x2a, 0x6a, 0xdd, 0x1b, 0x5d, 0xb2, 0x44, 0x0c, 0x4f, 0x71, 0xde, 0x6e,
0xb0, 0x2c, 0xca, 0x5c, 0xff, 0x37, 0x99, 0xdb, 0x50, 0xe3, 0xe4, 0x7a, 0x41, 0xe2, 0x80, 0x58,
0xa0, 0x86, 0xcd, 0x31, 0x7a, 0x09, 0x7b, 0x7c, 0x39, 0x8b, 0x68, 0x7c, 0x35, 0x16, 0x5e, 0x32,
0x25, 0xc2, 0x3a, 0x50, 0xcb, 0xb7, 0x52, 0xf6, 0x5c, 0x91, 0xe8, 0x15, 0x98, 0x62, 0x39, 0xd7,
0x77, 0xda, 0x3b, 0x3e, 0xda, 0x76, 0xcc, 0x45, 0x5c, 0xce, 0x09, 0x56, 0x35, 0xc8, 0x81, 0xc6,
0x9c, 0x24, 0x33, 0xca, 0xf5, 0x5d, 0x4c, 0xc7, 0x70, 0x5b, 0xb8, 0x48, 0xa1, 0x4e, 0x41, 0xa0,
0x98, 0x5b, 0x0d, 0xc7, 0x70, 0x2b, 0xdb, 0x1d, 0x3f, 0x72, 0xd4, 0x07, 0xf0, 0x23, 0x16, 0x5c,
0x8d, 0x95, 0xf4, 0x2d, 0x99, 0x1f, 0xec, 0xaf, 0x57, 0x9d, 0x26, 0xf6, 0x6e, 0x07, 0x32, 0x31,
0xa2, 0x77, 0x04, 0xd7, 0xfd, 0x2c, 0x94, 0x3d, 0x23, 0x16, 0x78, 0xd1, 0x78, 0x12, 0x79, 0x53,
0x6e, 0xfd, 0xde, 0x55, 0x4d, 0x41, 0x71, 0x67, 0x92, 0x92, 0x9e, 0x0a, 0x49, 0x44, 0x04, 0x09,
0xad, 0xaa, 0xf6, 0x54, 0x0a, 0x91, 0xbb, 0x75, 0x9b, 0xfc, 0xac, 0x36, 0xd8, 0x5b, 0xaf, 0x3a,
0x80, 0xbd, 0xdb, 0xa1, 0x66, 0x73, 0xf7, 0x49, 0xb1, 0x62, 0x36, 0x2e, 0x2e, 0x57, 0x53, 0xbf,
0x6a, 0xc5, 0xec, 0xf3, 0x96, 0x4c, 0x2d, 0xf6, 0xcb, 0x80, 0xea, 0x7b, 0xb6, 0x88, 0x05, 0x47,
0x87, 0x50, 0x99, 0xd0, 0x88, 0x70, 0x65, 0xac, 0x0a, 0xd6, 0x40, 0xce, 0x1c, 0xd2, 0x44, 0x5d,
0x8c, 0x12, 0xae, 0xa4, 0xad, 0xe0, 0x22, 0xa5, 0x0e, 0xa7, 0xcf, 0xc0, 0x95, 0xff, 0x2a, 0x38,
0xc7, 0xc5, 0x7d, 0x4c, 0x95, 0xca, 0xf7, 0x39, 0x84, 0x8a, 0xbf, 0x14, 0x24, 0x33, 0xa6, 0x06,
0x7f, 0x99, 0xa0, 0xfa, 0xcc, 0x04, 0x6d, 0xa8, 0xe9, 0x97, 0x37, 0x3c, 0x55, 0xe7, 0x6f, 0xe2,
0x1c, 0x23, 0x1b, 0x0a, 0x2a, 0x5a, 0xe8, 0xb9, 0xae, 0xdd, 0x4f, 0x50, 0xd7, 0x5b, 0x8e, 0x88,
0x40, 0x2e, 0x54, 0x03, 0x05, 0xd2, 0xd7, 0x08, 0xf2, 0x35, 0xea, 0x74, 0x6a, 0xca, 0x34, 0x2f,
0xc7, 0x0f, 0x12, 0x22, 0x5f, 0x9d, 0x5a, 0xbc, 0x8c, 0x33, 0x38, 0x70, 0x1e, 0x7e, 0xda, 0xa5,
0x87, 0xb5, 0x6d, 0x3c, 0xae, 0x6d, 0xe3, 0x69, 0x6d, 0x97, 0xbe, 0x6e, 0xec, 0xd2, 0xfd, 0xc6,
0x36, 0x1e, 0x37, 0x76, 0xe9, 0xfb, 0xc6, 0x2e, 0xf9, 0x55, 0xe5, 0xbe, 0x93, 0x3f, 0x01, 0x00,
0x00, 0xff, 0xff, 0x7e, 0x87, 0x05, 0xb2, 0xd0, 0x04, 0x00, 0x00,
// 743 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcf, 0x6f, 0xe3, 0x44,
0x14, 0x8e, 0x37, 0x71, 0x9a, 0x3c, 0x27, 0x61, 0x77, 0x58, 0x55, 0x56, 0x24, 0x1c, 0x2b, 0x08,
0xc9, 0xe2, 0x90, 0xb0, 0xdd, 0x1b, 0x48, 0x1c, 0xcc, 0xaa, 0x22, 0x12, 0x62, 0xd1, 0x64, 0xd5,
0x13, 0x52, 0xe4, 0x1f, 0x93, 0x64, 0x54, 0xc7, 0x93, 0x7a, 0x26, 0xad, 0xdc, 0x1b, 0xff, 0x01,
0x47, 0x8e, 0xfd, 0x73, 0x7a, 0xec, 0x11, 0x71, 0x88, 0x20, 0xe1, 0xc0, 0x9f, 0x81, 0x66, 0xc6,
0x76, 0x5c, 0x2e, 0xec, 0x6d, 0xbe, 0xef, 0x3d, 0xfb, 0xbd, 0xf9, 0xbe, 0xcf, 0x86, 0x3e, 0x17,
0xd9, 0x2e, 0x12, 0x7c, 0xb2, 0xcd, 0x98, 0x60, 0xe8, 0x45, 0x1c, 0x0e, 0x3f, 0xcf, 0xc8, 0x96,
0xf1, 0xa9, 0x22, 0xc2, 0xdd, 0x72, 0xba, 0x62, 0x2b, 0xa6, 0x80, 0x3a, 0xe9, 0xc6, 0xe1, 0x79,
0x42, 0x43, 0xdd, 0x12, 0xb1, 0x64, 0x1a, 0x92, 0xad, 0xe6, 0xc7, 0x37, 0x60, 0x5d, 0xd2, 0x84,
0x5c, 0x91, 0x8c, 0x53, 0x96, 0xa2, 0xaf, 0xe0, 0xec, 0x56, 0x1f, 0x6d, 0xc3, 0x35, 0x3c, 0xeb,
0xe2, 0xe5, 0xa4, 0x7c, 0x68, 0x72, 0x45, 0x22, 0xc1, 0x32, 0xbf, 0xf5, 0xb8, 0x1f, 0x35, 0x70,
0xd9, 0x86, 0xce, 0xa1, 0x1d, 0x93, 0x5b, 0x1a, 0x11, 0xfb, 0x85, 0x6b, 0x78, 0x3d, 0x5c, 0x20,
0x64, 0xc3, 0x19, 0x4d, 0x6f, 0x83, 0x84, 0xc6, 0x76, 0xd3, 0x35, 0xbc, 0x0e, 0x2e, 0xe1, 0xf8,
0x12, 0xac, 0x62, 0xdc, 0x0f, 0x94, 0x0b, 0xf4, 0x06, 0x3a, 0xc5, 0xbb, 0xb8, 0x6d, 0xb8, 0x4d,
0xcf, 0xba, 0xf8, 0x64, 0x12, 0x87, 0x93, 0xda, 0x56, 0xc5, 0xc8, 0xaa, 0xed, 0xeb, 0xd6, 0x6f,
0x0f, 0xa3, 0xc6, 0xf8, 0x17, 0x13, 0x5e, 0xc9, 0xae, 0x59, 0xba, 0x64, 0x1f, 0xb2, 0x5d, 0x1a,
0x05, 0x82, 0xc4, 0x08, 0x41, 0x2b, 0x0d, 0x36, 0x44, 0xad, 0xdf, 0xc5, 0xea, 0x2c, 0x39, 0x4e,
0xef, 0x89, 0x5a, 0xa4, 0x89, 0xd5, 0x19, 0x7d, 0x06, 0xb0, 0x61, 0x31, 0x5d, 0x52, 0x12, 0x2f,
0xb8, 0x6d, 0xaa, 0x4a, 0xb7, 0x64, 0xe6, 0xe8, 0x67, 0xb0, 0xaa, 0x72, 0x98, 0xdb, 0x3d, 0xd7,
0xf0, 0x5a, 0xfe, 0x37, 0x72, 0x8f, 0x3f, 0xf6, 0xa3, 0xb7, 0x2b, 0x2a, 0xd6, 0xbb, 0x70, 0x12,
0xb1, 0xcd, 0x94, 0xe7, 0x69, 0x24, 0xd6, 0x34, 0x5d, 0xd5, 0x4e, 0x75, 0xad, 0x27, 0xf3, 0x35,
0xcb, 0xc4, 0xec, 0x1d, 0xae, 0xc6, 0xf9, 0x79, 0x5d, 0xe6, 0xee, 0xc7, 0xc9, 0x3c, 0x84, 0x0e,
0x27, 0x37, 0x3b, 0x92, 0x46, 0xc4, 0x06, 0xb5, 0x6c, 0x85, 0xd1, 0x17, 0x30, 0xe0, 0xf9, 0x26,
0xa1, 0xe9, 0xf5, 0x42, 0x04, 0xd9, 0x8a, 0x08, 0xfb, 0x95, 0xba, 0x7c, 0xbf, 0x60, 0x3f, 0x28,
0x12, 0x8d, 0xc0, 0x0a, 0x13, 0x16, 0x5d, 0xf3, 0xc5, 0x3a, 0xe0, 0x6b, 0x1b, 0x29, 0xbb, 0x40,
0x53, 0xdf, 0x07, 0x7c, 0x8d, 0xbe, 0x84, 0x96, 0xc8, 0xb7, 0xda, 0xc8, 0xc1, 0xc5, 0xf9, 0x69,
0xa5, 0x4a, 0xe5, 0x7c, 0x4b, 0xb0, 0xea, 0x41, 0x2e, 0x58, 0x5b, 0x92, 0x6d, 0x28, 0xd7, 0xc6,
0xb5, 0x5c, 0xc3, 0xeb, 0xe3, 0x3a, 0x25, 0xc7, 0x55, 0x0a, 0xa6, 0xdc, 0xb6, 0x5c, 0xc3, 0x33,
0x4f, 0x22, 0xfc, 0xc8, 0xd1, 0x14, 0xf4, 0xf0, 0x85, 0xf2, 0xa6, 0x2f, 0xeb, 0xfe, 0xcb, 0xc3,
0x7e, 0xd4, 0xc3, 0xc1, 0x9d, 0x2f, 0x0b, 0x73, 0x7a, 0x4f, 0x70, 0x37, 0x2c, 0x8f, 0x72, 0x66,
0xc2, 0xa2, 0x20, 0x59, 0x2c, 0x93, 0x60, 0xc5, 0xed, 0x7f, 0xce, 0xd4, 0x50, 0x50, 0xdc, 0xa5,
0xa4, 0x64, 0xe8, 0x62, 0x92, 0x10, 0x41, 0x62, 0xbb, 0xad, 0x43, 0x57, 0x40, 0xe4, 0x9d, 0xe2,
0x28, 0x1f, 0xeb, 0xf8, 0x83, 0xc3, 0x7e, 0x04, 0x38, 0xb8, 0x9b, 0x69, 0xb6, 0x8a, 0xa7, 0x54,
0x33, 0x65, 0x8b, 0xfa, 0xe5, 0x3a, 0xea, 0x55, 0xfd, 0x94, 0xfd, 0x74, 0x22, 0x8b, 0x0c, 0x7e,
0x0b, 0x5d, 0xb5, 0x6a, 0x91, 0xe4, 0xb6, 0x02, 0x65, 0x8e, 0x3f, 0x3d, 0x29, 0xa8, 0x78, 0x29,
0x61, 0xe1, 0x6b, 0xd1, 0x38, 0x7e, 0x03, 0x03, 0xbf, 0x32, 0xe0, 0x7d, 0x9a, 0xe4, 0xff, 0xeb,
0xd2, 0xf8, 0x6f, 0x03, 0xda, 0xdf, 0xb1, 0x5d, 0x2a, 0x38, 0x7a, 0x0d, 0xe6, 0x92, 0x26, 0x84,
0xab, 0xb0, 0x9b, 0x58, 0x03, 0x29, 0x53, 0x4c, 0x33, 0x95, 0x22, 0x4a, 0xb8, 0x72, 0xd3, 0xc4,
0x75, 0x4a, 0x85, 0x49, 0x47, 0x83, 0xab, 0x6f, 0xc2, 0xc4, 0x15, 0xae, 0x4b, 0xd8, 0x52, 0xa5,
0x4a, 0xc2, 0xd7, 0x60, 0x86, 0xb9, 0x20, 0xe5, 0xc7, 0xa2, 0xc1, 0xb3, 0x60, 0xb6, 0xff, 0x13,
0xcc, 0x21, 0x74, 0xf4, 0xdf, 0x60, 0xf6, 0x4e, 0x45, 0xb2, 0x87, 0x2b, 0x8c, 0x1c, 0xa8, 0x19,
0xa7, 0xae, 0xf9, 0xcc, 0xca, 0xf1, 0x7b, 0xe8, 0xea, 0x5b, 0xce, 0x89, 0x40, 0x1e, 0xb4, 0x23,
0x05, 0x0a, 0x65, 0x41, 0xfe, 0x21, 0x74, 0xb9, 0x14, 0x54, 0xd7, 0xe5, 0xfa, 0x51, 0x46, 0xe4,
0x9f, 0x40, 0x5d, 0xbc, 0x89, 0x4b, 0xe8, 0xbb, 0x8f, 0x7f, 0x39, 0x8d, 0xc7, 0x83, 0x63, 0x3c,
0x1d, 0x1c, 0xe3, 0xcf, 0x83, 0xd3, 0xf8, 0xf5, 0xe8, 0x34, 0x1e, 0x8e, 0x8e, 0xf1, 0x74, 0x74,
0x1a, 0xbf, 0x1f, 0x9d, 0x46, 0xd8, 0x56, 0x76, 0xbd, 0xfd, 0x37, 0x00, 0x00, 0xff, 0xff, 0xb8,
0xd0, 0x05, 0xba, 0x64, 0x05, 0x00, 0x00,
}
func (m *FileVersion) Marshal() (dAtA []byte, err error) {
@ -408,6 +491,15 @@ func (m *FileInfoTruncated) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i--
dAtA[i] = 0xc0
}
if len(m.BlocksHash) > 0 {
i -= len(m.BlocksHash)
copy(dAtA[i:], m.BlocksHash)
i = encodeVarintStructs(dAtA, i, uint64(len(m.BlocksHash)))
i--
dAtA[i] = 0x1
i--
dAtA[i] = 0x92
}
if len(m.SymlinkTarget) > 0 {
i -= len(m.SymlinkTarget)
copy(dAtA[i:], m.SymlinkTarget)
@ -507,6 +599,75 @@ func (m *FileInfoTruncated) MarshalToSizedBuffer(dAtA []byte) (int, error) {
return len(dAtA) - i, nil
}
func (m *BlockList) Marshal() (dAtA []byte, err error) {
size := m.ProtoSize()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *BlockList) MarshalTo(dAtA []byte) (int, error) {
size := m.ProtoSize()
return m.MarshalToSizedBuffer(dAtA[:size])
}
func (m *BlockList) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
if len(m.Blocks) > 0 {
for iNdEx := len(m.Blocks) - 1; iNdEx >= 0; iNdEx-- {
{
size, err := m.Blocks[iNdEx].MarshalToSizedBuffer(dAtA[:i])
if err != nil {
return 0, err
}
i -= size
i = encodeVarintStructs(dAtA, i, uint64(size))
}
i--
dAtA[i] = 0xa
}
}
return len(dAtA) - i, nil
}
func (m *BlocksHashOnly) Marshal() (dAtA []byte, err error) {
size := m.ProtoSize()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *BlocksHashOnly) MarshalTo(dAtA []byte) (int, error) {
size := m.ProtoSize()
return m.MarshalToSizedBuffer(dAtA[:size])
}
func (m *BlocksHashOnly) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
if len(m.BlocksHash) > 0 {
i -= len(m.BlocksHash)
copy(dAtA[i:], m.BlocksHash)
i = encodeVarintStructs(dAtA, i, uint64(len(m.BlocksHash)))
i--
dAtA[i] = 0x1
i--
dAtA[i] = 0x92
}
return len(dAtA) - i, nil
}
func (m *Counts) Marshal() (dAtA []byte, err error) {
size := m.ProtoSize()
dAtA = make([]byte, size)
@ -711,12 +872,44 @@ func (m *FileInfoTruncated) ProtoSize() (n int) {
if l > 0 {
n += 2 + l + sovStructs(uint64(l))
}
l = len(m.BlocksHash)
if l > 0 {
n += 2 + l + sovStructs(uint64(l))
}
if m.LocalFlags != 0 {
n += 2 + sovStructs(uint64(m.LocalFlags))
}
return n
}
func (m *BlockList) ProtoSize() (n int) {
if m == nil {
return 0
}
var l int
_ = l
if len(m.Blocks) > 0 {
for _, e := range m.Blocks {
l = e.ProtoSize()
n += 1 + l + sovStructs(uint64(l))
}
}
return n
}
func (m *BlocksHashOnly) ProtoSize() (n int) {
if m == nil {
return 0
}
var l int
_ = l
l = len(m.BlocksHash)
if l > 0 {
n += 2 + l + sovStructs(uint64(l))
}
return n
}
func (m *Counts) ProtoSize() (n int) {
if m == nil {
return 0
@ -1340,6 +1533,40 @@ func (m *FileInfoTruncated) Unmarshal(dAtA []byte) error {
}
m.SymlinkTarget = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 18:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field BlocksHash", wireType)
}
var byteLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
byteLen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if byteLen < 0 {
return ErrInvalidLengthStructs
}
postIndex := iNdEx + byteLen
if postIndex < 0 {
return ErrInvalidLengthStructs
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.BlocksHash = append(m.BlocksHash[:0], dAtA[iNdEx:postIndex]...)
if m.BlocksHash == nil {
m.BlocksHash = []byte{}
}
iNdEx = postIndex
case 1000:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field LocalFlags", wireType)
@ -1383,6 +1610,180 @@ func (m *FileInfoTruncated) Unmarshal(dAtA []byte) error {
}
return nil
}
func (m *BlockList) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: BlockList: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: BlockList: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Blocks", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthStructs
}
postIndex := iNdEx + msglen
if postIndex < 0 {
return ErrInvalidLengthStructs
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Blocks = append(m.Blocks, protocol.BlockInfo{})
if err := m.Blocks[len(m.Blocks)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipStructs(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthStructs
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthStructs
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *BlocksHashOnly) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: BlocksHashOnly: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: BlocksHashOnly: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 18:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field BlocksHash", wireType)
}
var byteLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
byteLen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if byteLen < 0 {
return ErrInvalidLengthStructs
}
postIndex := iNdEx + byteLen
if postIndex < 0 {
return ErrInvalidLengthStructs
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.BlocksHash = append(m.BlocksHash[:0], dAtA[iNdEx:postIndex]...)
if m.BlocksHash == nil {
m.BlocksHash = []byte{}
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipStructs(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthStructs
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthStructs
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *Counts) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0

View File

@ -34,6 +34,7 @@ message FileInfoTruncated {
int64 sequence = 10;
// repeated BlockInfo Blocks = 16
string symlink_target = 17;
bytes blocks_hash = 18;
protocol.FileInfoType type = 2;
uint32 permissions = 4;
int32 modified_ns = 11;
@ -47,6 +48,16 @@ message FileInfoTruncated {
bool no_permissions = 8;
}
// BlockList is the structure used to store block lists
message BlockList {
repeated protocol.BlockInfo Blocks = 1 [(gogoproto.nullable) = false];
}
// BlocksHashOnly is used to only unmarshal the block list hash from a FileInfo
message BlocksHashOnly {
bytes blocks_hash = 18;
}
// For each folder and device we keep one of these to track the current
// counts and sequence. We also keep one for the global state of the folder.
message Counts {

View File

@ -58,13 +58,50 @@ func (t readOnlyTransaction) getFileTrunc(key []byte, trunc bool) (FileIntf, boo
if err != nil {
return nil, false, err
}
f, err := unmarshalTrunc(bs, trunc)
f, err := t.unmarshalTrunc(bs, trunc)
if err != nil {
return nil, false, err
}
return f, true, nil
}
func (t readOnlyTransaction) unmarshalTrunc(bs []byte, trunc bool) (FileIntf, error) {
if trunc {
var tf FileInfoTruncated
err := tf.Unmarshal(bs)
if err != nil {
return nil, err
}
return tf, nil
}
var tf protocol.FileInfo
if err := tf.Unmarshal(bs); err != nil {
return nil, err
}
if err := t.fillBlockList(&tf); err != nil {
return nil, err
}
return tf, nil
}
func (t readOnlyTransaction) fillBlockList(fi *protocol.FileInfo) error {
if fi.BlocksHash == nil {
return nil
}
blocksKey := t.keyer.GenerateBlockListKey(nil, fi.BlocksHash)
bs, err := t.Get(blocksKey)
if err != nil {
return err
}
var bl BlockList
if err := bl.Unmarshal(bs); err != nil {
return err
}
fi.Blocks = bl.Blocks
return nil
}
func (t readOnlyTransaction) getGlobal(keyBuf, folder, file []byte, truncate bool) ([]byte, FileIntf, bool, error) {
var err error
keyBuf, err = t.keyer.GenerateGlobalVersionKey(keyBuf, folder, file)
@ -132,7 +169,7 @@ func (t *readOnlyTransaction) withHave(folder, device, prefix []byte, truncate b
return nil
}
f, err := unmarshalTrunc(dbi.Value(), truncate)
f, err := t.unmarshalTrunc(dbi.Value(), truncate)
if err != nil {
l.Debugln("unmarshal error:", err)
continue
@ -413,6 +450,28 @@ func (t readWriteTransaction) close() {
t.WriteTransaction.Release()
}
func (t readWriteTransaction) putFile(key []byte, fi protocol.FileInfo) error {
if fi.Blocks != nil {
if fi.BlocksHash == nil {
fi.BlocksHash = protocol.BlocksHash(fi.Blocks)
}
blocksKey := t.keyer.GenerateBlockListKey(nil, fi.BlocksHash)
if _, err := t.Get(blocksKey); backend.IsNotFound(err) {
// Marshal the block list and save it
blocksBs := mustMarshal(&BlockList{Blocks: fi.Blocks})
if err := t.Put(blocksKey, blocksBs); err != nil {
return err
}
} else if err != nil {
return err
}
}
fi.Blocks = nil
fiBs := mustMarshal(&fi)
return t.Put(key, fiBs)
}
// updateGlobal adds this device+version to the version list for the given
// file. If the device is already present in the list, the version is updated.
// If the file does not have an entry in the global list, it is created.

View File

@ -954,8 +954,8 @@ func (m *model) handleIndex(deviceID protocol.DeviceID, folder string, fs []prot
files.Drop(deviceID)
}
for i := range fs {
// The local flags should never be transmitted over the wire. Make
// sure they look like they weren't.
// The local attributes should never be transmitted over the wire.
// Make sure they look like they weren't.
fs[i].LocalFlags = 0
}
files.Update(deviceID, fs)

View File

@ -495,8 +495,9 @@ type FileInfo struct {
ModifiedBy ShortID `protobuf:"varint,12,opt,name=modified_by,json=modifiedBy,proto3,customtype=ShortID" json:"modified_by"`
Version Vector `protobuf:"bytes,9,opt,name=version,proto3" json:"version"`
Sequence int64 `protobuf:"varint,10,opt,name=sequence,proto3" json:"sequence,omitempty"`
Blocks []BlockInfo `protobuf:"bytes,16,rep,name=Blocks,proto3" json:"Blocks"`
Blocks []BlockInfo `protobuf:"bytes,16,rep,name=blocks,proto3" json:"blocks"`
SymlinkTarget string `protobuf:"bytes,17,opt,name=symlink_target,json=symlinkTarget,proto3" json:"symlink_target,omitempty"`
BlocksHash []byte `protobuf:"bytes,18,opt,name=blocks_hash,json=blocksHash,proto3" json:"blocks_hash,omitempty"`
Type FileInfoType `protobuf:"varint,2,opt,name=type,proto3,enum=protocol.FileInfoType" json:"type,omitempty"`
Permissions uint32 `protobuf:"varint,4,opt,name=permissions,proto3" json:"permissions,omitempty"`
ModifiedNs int32 `protobuf:"varint,11,opt,name=modified_ns,json=modifiedNs,proto3" json:"modified_ns,omitempty"`
@ -920,120 +921,121 @@ func init() {
func init() { proto.RegisterFile("bep.proto", fileDescriptor_e3f59eb60afbbc6e) }
var fileDescriptor_e3f59eb60afbbc6e = []byte{
// 1803 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcf, 0x6f, 0xdb, 0xc8,
0x15, 0x16, 0x25, 0xea, 0xd7, 0x93, 0xec, 0xa5, 0x27, 0x89, 0xcb, 0x32, 0x59, 0x89, 0x51, 0x92,
0x8d, 0xd6, 0xd8, 0x26, 0xe9, 0xee, 0xb6, 0x45, 0x8b, 0xb6, 0x80, 0x7e, 0xd0, 0x8e, 0x50, 0x47,
// 1816 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4f, 0x6f, 0xdb, 0xc8,
0x15, 0x17, 0x25, 0xea, 0xdf, 0x93, 0xec, 0xa5, 0x27, 0x89, 0xcb, 0x32, 0x59, 0x89, 0x51, 0x92,
0x8d, 0xd6, 0xd8, 0x26, 0xe9, 0xee, 0xb6, 0x45, 0x8b, 0xb6, 0x80, 0xfe, 0xd0, 0x8e, 0x50, 0x47,
0x72, 0x47, 0x72, 0xb6, 0xd9, 0x43, 0x09, 0x5a, 0x1c, 0xc9, 0x44, 0x28, 0x8e, 0x4a, 0x52, 0x76,
0xb4, 0x7f, 0x82, 0x4e, 0x3d, 0xf6, 0x22, 0x60, 0x81, 0x9e, 0xfa, 0x9f, 0xe4, 0x98, 0xf6, 0x50,
0x14, 0x3d, 0x18, 0x5d, 0xe7, 0xb2, 0xc7, 0xfe, 0x05, 0x45, 0x31, 0x33, 0xa4, 0x44, 0xd9, 0x9b,
0x45, 0x0e, 0x7b, 0xd2, 0xcc, 0x7b, 0x1f, 0xdf, 0xcc, 0x7c, 0xf3, 0xbd, 0x6f, 0x04, 0xc5, 0x13,
0x32, 0x7d, 0x34, 0xf5, 0x69, 0x48, 0x51, 0x81, 0xff, 0x0c, 0xa9, 0xab, 0xdd, 0xf3, 0xc9, 0x94,
0x06, 0x8f, 0xf9, 0xfc, 0x64, 0x36, 0x7a, 0x3c, 0xa6, 0x63, 0xca, 0x27, 0x7c, 0x24, 0xe0, 0xb5,
0x29, 0x64, 0x9f, 0x12, 0xd7, 0xa5, 0xa8, 0x0a, 0x25, 0x9b, 0x9c, 0x39, 0x43, 0x62, 0x7a, 0xd6,
0x84, 0xa8, 0x92, 0x2e, 0xd5, 0x8b, 0x18, 0x44, 0xa8, 0x6b, 0x4d, 0x08, 0x03, 0x0c, 0x5d, 0x87,
0x78, 0xa1, 0x00, 0xa4, 0x05, 0x40, 0x84, 0x38, 0xe0, 0x01, 0x6c, 0x47, 0x80, 0x33, 0xe2, 0x07,
0x0e, 0xf5, 0xd4, 0x0c, 0xc7, 0x6c, 0x89, 0xe8, 0x73, 0x11, 0xac, 0x05, 0x90, 0x7b, 0x4a, 0x2c,
0x9b, 0xf8, 0xe8, 0x63, 0x90, 0xc3, 0xf9, 0x54, 0xac, 0xb5, 0xfd, 0xe9, 0xad, 0x47, 0xf1, 0xce,
0x1f, 0x3d, 0x23, 0x41, 0x60, 0x8d, 0xc9, 0x60, 0x3e, 0x25, 0x98, 0x43, 0xd0, 0x6f, 0xa1, 0x34,
0xa4, 0x93, 0xa9, 0x4f, 0x02, 0x5e, 0x38, 0xcd, 0xbf, 0xb8, 0x73, 0xed, 0x8b, 0xd6, 0x1a, 0x83,
0x93, 0x1f, 0xd4, 0x1a, 0xb0, 0xd5, 0x72, 0x67, 0x41, 0x48, 0xfc, 0x16, 0xf5, 0x46, 0xce, 0x18,
0x3d, 0x81, 0xfc, 0x88, 0xba, 0x36, 0xf1, 0x03, 0x55, 0xd2, 0x33, 0xf5, 0xd2, 0xa7, 0xca, 0xba,
0xd8, 0x3e, 0x4f, 0x34, 0xe5, 0xd7, 0x17, 0xd5, 0x14, 0x8e, 0x61, 0xb5, 0xbf, 0xa6, 0x21, 0x27,
0x32, 0x68, 0x17, 0xd2, 0x8e, 0x2d, 0x28, 0x6a, 0xe6, 0x2e, 0x2f, 0xaa, 0xe9, 0x4e, 0x1b, 0xa7,
0x1d, 0x1b, 0xdd, 0x84, 0xac, 0x6b, 0x9d, 0x10, 0x37, 0x22, 0x47, 0x4c, 0xd0, 0x6d, 0x28, 0xfa,
0xc4, 0xb2, 0x4d, 0xea, 0xb9, 0x73, 0x4e, 0x49, 0x01, 0x17, 0x58, 0xa0, 0xe7, 0xb9, 0x73, 0xf4,
0x13, 0x40, 0xce, 0xd8, 0xa3, 0x3e, 0x31, 0xa7, 0xc4, 0x9f, 0x38, 0x7c, 0xb7, 0x81, 0x2a, 0x73,
0xd4, 0x8e, 0xc8, 0x1c, 0xad, 0x13, 0xe8, 0x1e, 0x6c, 0x45, 0x70, 0x9b, 0xb8, 0x24, 0x24, 0x6a,
0x96, 0x23, 0xcb, 0x22, 0xd8, 0xe6, 0x31, 0xf4, 0x04, 0x6e, 0xda, 0x4e, 0x60, 0x9d, 0xb8, 0xc4,
0x0c, 0xc9, 0x64, 0x6a, 0x3a, 0x9e, 0x4d, 0x5e, 0x91, 0x40, 0xcd, 0x71, 0x2c, 0x8a, 0x72, 0x03,
0x32, 0x99, 0x76, 0x44, 0x06, 0xed, 0x42, 0x6e, 0x6a, 0xcd, 0x02, 0x62, 0xab, 0x79, 0x8e, 0x89,
0x66, 0x8c, 0x25, 0xa1, 0x80, 0x40, 0x55, 0xae, 0xb2, 0xd4, 0xe6, 0x89, 0x98, 0xa5, 0x08, 0x56,
0xfb, 0x6f, 0x1a, 0x72, 0x22, 0x83, 0x3e, 0x5a, 0xb1, 0x54, 0x6e, 0xee, 0x32, 0xd4, 0xbf, 0x2f,
0xaa, 0x05, 0x91, 0xeb, 0xb4, 0x13, 0xac, 0x21, 0x90, 0x13, 0x8a, 0xe2, 0x63, 0x74, 0x07, 0x8a,
0x96, 0x6d, 0xb3, 0xdb, 0x23, 0x81, 0x9a, 0xd1, 0x33, 0xf5, 0x22, 0x5e, 0x07, 0xd0, 0x2f, 0x36,
0xd5, 0x20, 0x5f, 0xd5, 0xcf, 0xbb, 0x64, 0xc0, 0xae, 0x62, 0x48, 0xfc, 0x48, 0xc1, 0x59, 0xbe,
0x5e, 0x81, 0x05, 0xb8, 0x7e, 0xef, 0x42, 0x79, 0x62, 0xbd, 0x32, 0x03, 0xf2, 0xa7, 0x19, 0xf1,
0x86, 0x84, 0xd3, 0x95, 0xc1, 0xa5, 0x89, 0xf5, 0xaa, 0x1f, 0x85, 0x50, 0x05, 0xc0, 0xf1, 0x42,
0x9f, 0xda, 0xb3, 0x21, 0xf1, 0x23, 0xae, 0x12, 0x11, 0xf4, 0x33, 0x28, 0x70, 0xb2, 0x4d, 0xc7,
0x56, 0x0b, 0xba, 0x54, 0x97, 0x9b, 0x5a, 0x74, 0xf0, 0x3c, 0xa7, 0x9a, 0x9f, 0x3b, 0x1e, 0xe2,
0x3c, 0xc7, 0x76, 0x6c, 0xf4, 0x6b, 0xd0, 0x82, 0x97, 0x0e, 0xbb, 0x28, 0x51, 0x29, 0x74, 0xa8,
0x67, 0xfa, 0x64, 0x42, 0xcf, 0x2c, 0x37, 0x50, 0x8b, 0x7c, 0x19, 0x95, 0x21, 0x3a, 0x09, 0x00,
0x8e, 0xf2, 0xb5, 0x1e, 0x64, 0x79, 0x45, 0x76, 0x8b, 0x42, 0xac, 0x51, 0xf7, 0x46, 0x33, 0xf4,
0x08, 0xb2, 0x23, 0xc7, 0x25, 0x81, 0x9a, 0xe6, 0x77, 0x88, 0x12, 0x4a, 0x77, 0x5c, 0xd2, 0xf1,
0x46, 0x34, 0xba, 0x45, 0x01, 0xab, 0x1d, 0x43, 0x89, 0x17, 0x3c, 0x9e, 0xda, 0x56, 0x48, 0x7e,
0xb0, 0xb2, 0x17, 0x32, 0x14, 0xe2, 0xcc, 0xea, 0xd2, 0xa5, 0xc4, 0xa5, 0x23, 0x90, 0x03, 0xe7,
0x2b, 0xc2, 0x7b, 0x24, 0x83, 0xf9, 0x18, 0x7d, 0x08, 0x30, 0xa1, 0xb6, 0x33, 0x72, 0x88, 0x6d,
0x06, 0xfc, 0xca, 0x32, 0xb8, 0x18, 0x47, 0xfa, 0xe8, 0x09, 0x94, 0x56, 0xe9, 0x93, 0xb9, 0x5a,
0xe6, 0x9c, 0x7f, 0x10, 0x73, 0xde, 0x3f, 0xa5, 0x7e, 0xd8, 0x69, 0xe3, 0x55, 0x89, 0xe6, 0x9c,
0x49, 0x3a, 0xb6, 0x27, 0x46, 0xec, 0x86, 0xa4, 0x9f, 0x93, 0x61, 0x48, 0x57, 0x8d, 0x1f, 0xc1,
0x90, 0x06, 0x85, 0x95, 0x26, 0x80, 0x6f, 0x60, 0x35, 0x47, 0x3f, 0x85, 0x5c, 0xd3, 0xa5, 0xc3,
0x97, 0x71, 0x7f, 0xdc, 0x58, 0x17, 0xe3, 0xf1, 0x04, 0x0b, 0x11, 0x90, 0xd9, 0x64, 0x30, 0x9f,
0xb8, 0x8e, 0xf7, 0xd2, 0x0c, 0x2d, 0x7f, 0x4c, 0x42, 0x75, 0x47, 0xd8, 0x64, 0x14, 0x1d, 0xf0,
0x20, 0xda, 0x8b, 0xcc, 0x51, 0x58, 0xdd, 0xee, 0x75, 0x72, 0x13, 0xee, 0xa8, 0x43, 0xe9, 0xaa,
0x7b, 0x6c, 0xe1, 0x64, 0x88, 0x99, 0xf7, 0x8a, 0x27, 0x2f, 0x50, 0x4b, 0xba, 0x54, 0xcf, 0xae,
0x69, 0xe9, 0x06, 0xe8, 0x31, 0xc0, 0x09, 0xdb, 0x9f, 0xc9, 0x6f, 0x60, 0x8b, 0xe5, 0x9b, 0xca,
0xe5, 0x45, 0xb5, 0x8c, 0xad, 0x73, 0xbe, 0xf1, 0xbe, 0xf3, 0x15, 0xc1, 0xc5, 0x93, 0x78, 0xc8,
0xd6, 0x74, 0xe9, 0xd0, 0x72, 0xcd, 0x91, 0x6b, 0x8d, 0x03, 0xf5, 0xdb, 0x3c, 0x5f, 0x14, 0x78,
0x6c, 0x9f, 0x85, 0x90, 0xca, 0xcc, 0x83, 0x19, 0x92, 0x1d, 0x39, 0x4f, 0x3c, 0x45, 0x75, 0xc8,
0x3b, 0xde, 0x99, 0xe5, 0x3a, 0x91, 0xdf, 0x34, 0xb7, 0x2f, 0x2f, 0xaa, 0x80, 0xad, 0xf3, 0x8e,
0x88, 0xe2, 0x38, 0xcd, 0xc8, 0xf2, 0xe8, 0x86, 0x35, 0x16, 0x78, 0xa9, 0x2d, 0x8f, 0x26, 0x6c,
0xf1, 0x57, 0xf2, 0x5f, 0xbe, 0xae, 0xa6, 0x6a, 0x1e, 0x14, 0x57, 0xa4, 0x33, 0x31, 0x9d, 0x5a,
0xc1, 0x29, 0x17, 0x53, 0x19, 0xf3, 0x31, 0x53, 0x32, 0x1d, 0x8d, 0x02, 0x12, 0x72, 0xd9, 0x65,
0x70, 0x34, 0x5b, 0x09, 0x2f, 0xcd, 0x69, 0x11, 0xc2, 0xbb, 0x0d, 0xc5, 0x73, 0x62, 0xbd, 0x34,
0x79, 0x11, 0xc1, 0x68, 0x81, 0x05, 0x9e, 0x5a, 0xc1, 0x69, 0xb4, 0xde, 0x6f, 0x20, 0x27, 0x14,
0x83, 0x3e, 0x83, 0xc2, 0x90, 0xce, 0xbc, 0x70, 0xfd, 0x9c, 0xec, 0x24, 0xdd, 0x88, 0x67, 0x22,
0x19, 0xac, 0x80, 0xb5, 0x7d, 0xc8, 0x47, 0x29, 0xf4, 0x60, 0x65, 0x95, 0x72, 0xf3, 0xd6, 0x15,
0xf5, 0x6e, 0xbe, 0x2f, 0x67, 0x96, 0x3b, 0x13, 0x1b, 0x95, 0xb1, 0x98, 0xd4, 0xfe, 0x2e, 0x41,
0x1e, 0x33, 0x41, 0x06, 0x61, 0xe2, 0x65, 0xca, 0x6e, 0xbc, 0x4c, 0xeb, 0x1e, 0x4e, 0x6f, 0xf4,
0x70, 0xdc, 0x86, 0x99, 0x44, 0x1b, 0xae, 0x59, 0x92, 0xbf, 0x93, 0xa5, 0x6c, 0x82, 0xa5, 0x98,
0xe5, 0x5c, 0x82, 0xe5, 0x07, 0xb0, 0x3d, 0xf2, 0xe9, 0x84, 0xbf, 0x3d, 0xd4, 0xb7, 0xfc, 0x79,
0x64, 0x94, 0x5b, 0x2c, 0x3a, 0x88, 0x83, 0x9b, 0x04, 0x17, 0x36, 0x09, 0xae, 0x99, 0x50, 0xc0,
0x24, 0x98, 0x52, 0x2f, 0x20, 0xef, 0x3c, 0x13, 0x02, 0xd9, 0xb6, 0x42, 0x8b, 0x9f, 0xa8, 0x8c,
0xf9, 0x18, 0x3d, 0x04, 0x79, 0x48, 0x6d, 0x71, 0x9e, 0xed, 0x64, 0x37, 0x1a, 0xbe, 0x4f, 0xfd,
0x16, 0xb5, 0x09, 0xe6, 0x80, 0xda, 0x14, 0x94, 0x36, 0x3d, 0xf7, 0x5c, 0x6a, 0xd9, 0x47, 0x3e,
0x1d, 0xb3, 0x07, 0xe2, 0x9d, 0x46, 0xd7, 0x86, 0xfc, 0x8c, 0x5b, 0x61, 0x6c, 0x75, 0xf7, 0x37,
0xbb, 0xf1, 0x6a, 0x21, 0xe1, 0x9b, 0xb1, 0x8d, 0x44, 0x9f, 0xd6, 0xfe, 0x29, 0x81, 0xf6, 0x6e,
0x34, 0xea, 0x40, 0x49, 0x20, 0xcd, 0xc4, 0x7f, 0xa2, 0xfa, 0xfb, 0x2c, 0xc4, 0x8d, 0x00, 0x66,
0xab, 0xf1, 0x77, 0x3e, 0xa8, 0x09, 0xdb, 0xcb, 0xbc, 0x9f, 0xed, 0x3d, 0x84, 0x2d, 0xe1, 0x08,
0xf1, 0xdf, 0x07, 0x59, 0xcf, 0xd4, 0xb3, 0xcd, 0xb4, 0x92, 0xc2, 0xe5, 0x13, 0xd1, 0x66, 0x3c,
0x5e, 0xcb, 0x81, 0x7c, 0xe4, 0x78, 0xe3, 0x5a, 0x15, 0xb2, 0x2d, 0x97, 0xf2, 0x0b, 0xcb, 0xf9,
0xc4, 0x0a, 0xa8, 0x17, 0xf3, 0x28, 0x66, 0x7b, 0xff, 0x48, 0x43, 0x29, 0xf1, 0xd7, 0x0e, 0x3d,
0x81, 0xed, 0xd6, 0xe1, 0x71, 0x7f, 0x60, 0x60, 0xb3, 0xd5, 0xeb, 0xee, 0x77, 0x0e, 0x94, 0x94,
0x76, 0x67, 0xb1, 0xd4, 0xd5, 0xc9, 0x1a, 0xb4, 0xf9, 0xaf, 0xad, 0x0a, 0xd9, 0x4e, 0xb7, 0x6d,
0xfc, 0x41, 0x91, 0xb4, 0x9b, 0x8b, 0xa5, 0xae, 0x24, 0x80, 0xe2, 0x09, 0xfc, 0x04, 0xca, 0x1c,
0x60, 0x1e, 0x1f, 0xb5, 0x1b, 0x03, 0x43, 0x49, 0x6b, 0xda, 0x62, 0xa9, 0xef, 0x5e, 0xc5, 0x45,
0x9c, 0xdf, 0x83, 0x3c, 0x36, 0x7e, 0x7f, 0x6c, 0xf4, 0x07, 0x4a, 0x46, 0xdb, 0x5d, 0x2c, 0x75,
0x94, 0x00, 0xc6, 0x2d, 0xf5, 0x00, 0x0a, 0xd8, 0xe8, 0x1f, 0xf5, 0xba, 0x7d, 0x43, 0x91, 0xb5,
0x1f, 0x2d, 0x96, 0xfa, 0x8d, 0x0d, 0x54, 0xa4, 0xd2, 0x9f, 0xc3, 0x4e, 0xbb, 0xf7, 0x45, 0xf7,
0xb0, 0xd7, 0x68, 0x9b, 0x47, 0xb8, 0x77, 0x80, 0x8d, 0x7e, 0x5f, 0xc9, 0x6a, 0xd5, 0xc5, 0x52,
0xbf, 0x9d, 0xc0, 0x5f, 0x13, 0xdd, 0x87, 0x20, 0x1f, 0x75, 0xba, 0x07, 0x4a, 0x4e, 0xbb, 0xb1,
0x58, 0xea, 0x1f, 0x24, 0xa0, 0x8c, 0x54, 0x76, 0xe2, 0xd6, 0x61, 0xaf, 0x6f, 0x28, 0xf9, 0x6b,
0x27, 0xe6, 0x64, 0xef, 0xfd, 0x11, 0xd0, 0xf5, 0x3f, 0xbf, 0xe8, 0x3e, 0xc8, 0xdd, 0x5e, 0xd7,
0x50, 0x52, 0xe2, 0xfc, 0xd7, 0x11, 0x5d, 0xea, 0x11, 0x54, 0x83, 0xcc, 0xe1, 0x97, 0x9f, 0x2b,
0x92, 0xf6, 0xe3, 0xc5, 0x52, 0xbf, 0x75, 0x1d, 0x74, 0xf8, 0xe5, 0xe7, 0x7b, 0x14, 0x4a, 0xc9,
0xc2, 0x35, 0x28, 0x3c, 0x33, 0x06, 0x8d, 0x76, 0x63, 0xd0, 0x50, 0x52, 0x62, 0x4b, 0x71, 0xfa,
0x19, 0x09, 0x2d, 0xde, 0x84, 0x77, 0x20, 0xdb, 0x35, 0x9e, 0x1b, 0x58, 0x91, 0xb4, 0x9d, 0xc5,
0x52, 0xdf, 0x8a, 0x01, 0x5d, 0x72, 0x46, 0x7c, 0x54, 0x81, 0x5c, 0xe3, 0xf0, 0x8b, 0xc6, 0x8b,
0xbe, 0x92, 0xd6, 0xd0, 0x62, 0xa9, 0x6f, 0xc7, 0xe9, 0x86, 0x7b, 0x6e, 0xcd, 0x83, 0xbd, 0xff,
0x49, 0x50, 0x4e, 0xbe, 0x71, 0xa8, 0x02, 0xf2, 0x7e, 0xe7, 0xd0, 0x88, 0x97, 0x4b, 0xe6, 0xd8,
0x18, 0xd5, 0xa1, 0xd8, 0xee, 0x60, 0xa3, 0x35, 0xe8, 0xe1, 0x17, 0xf1, 0x59, 0x92, 0xa0, 0xb6,
0xe3, 0x73, 0x81, 0xcf, 0xd1, 0x2f, 0xa1, 0xdc, 0x7f, 0xf1, 0xec, 0xb0, 0xd3, 0xfd, 0x9d, 0xc9,
0x2b, 0xa6, 0xb5, 0x87, 0x8b, 0xa5, 0x7e, 0x77, 0x03, 0x4c, 0xa6, 0x3e, 0x19, 0x5a, 0x21, 0xb1,
0xfb, 0xe2, 0x39, 0x66, 0xc9, 0x82, 0x84, 0x5a, 0xb0, 0x13, 0x7f, 0xba, 0x5e, 0x2c, 0xa3, 0x7d,
0xb2, 0x58, 0xea, 0x1f, 0x7d, 0xef, 0xf7, 0xab, 0xd5, 0x0b, 0x12, 0xba, 0x0f, 0xf9, 0xa8, 0x48,
0xac, 0xa4, 0xe4, 0xa7, 0xd1, 0x07, 0x7b, 0x7f, 0x93, 0xa0, 0xb8, 0xb2, 0x2b, 0x46, 0x78, 0xb7,
0x67, 0x1a, 0x18, 0xf7, 0x70, 0xcc, 0xc0, 0x2a, 0xd9, 0xa5, 0x7c, 0x88, 0xee, 0x42, 0xfe, 0xc0,
0xe8, 0x1a, 0xb8, 0xd3, 0x8a, 0x1b, 0x63, 0x05, 0x39, 0x20, 0x1e, 0xf1, 0x9d, 0x21, 0xfa, 0x18,
0xca, 0xdd, 0x9e, 0xd9, 0x3f, 0x6e, 0x3d, 0x8d, 0x8f, 0xce, 0xd7, 0x4f, 0x94, 0xea, 0xcf, 0x86,
0xa7, 0x9c, 0xcf, 0x3d, 0xd6, 0x43, 0xcf, 0x1b, 0x87, 0x9d, 0xb6, 0x80, 0x66, 0x34, 0x75, 0xb1,
0xd4, 0x6f, 0xae, 0xa0, 0xd1, 0x23, 0xcd, 0xb0, 0x7b, 0x36, 0x54, 0xbe, 0xdf, 0x98, 0x90, 0x0e,
0xb9, 0xc6, 0xd1, 0x91, 0xd1, 0x6d, 0xc7, 0xbb, 0x5f, 0xe7, 0x1a, 0xd3, 0x29, 0xf1, 0x6c, 0x86,
0xd8, 0xef, 0xe1, 0x03, 0x63, 0x10, 0x6f, 0x7e, 0x8d, 0xd8, 0xa7, 0xec, 0xbf, 0x50, 0xb3, 0xfe,
0xfa, 0x9b, 0x4a, 0xea, 0xcd, 0x37, 0x95, 0xd4, 0xeb, 0xcb, 0x8a, 0xf4, 0xe6, 0xb2, 0x22, 0xfd,
0xe7, 0xb2, 0x92, 0xfa, 0xf6, 0xb2, 0x22, 0xfd, 0xf9, 0x6d, 0x25, 0xf5, 0xf5, 0xdb, 0x8a, 0xf4,
0xe6, 0x6d, 0x25, 0xf5, 0xaf, 0xb7, 0x95, 0xd4, 0x49, 0x8e, 0x9b, 0xda, 0x67, 0xff, 0x0f, 0x00,
0x00, 0xff, 0xff, 0xd6, 0x0e, 0x8e, 0xa4, 0x11, 0x0f, 0x00, 0x00,
0xb4, 0x1f, 0x41, 0xa7, 0x1e, 0x7b, 0x11, 0xb0, 0x40, 0x4f, 0xfd, 0x26, 0x39, 0xa6, 0x3d, 0x14,
0x45, 0x0f, 0x46, 0xd7, 0xb9, 0xec, 0xb1, 0x9f, 0xa0, 0x2d, 0x66, 0x86, 0x94, 0x28, 0x7b, 0xb3,
0xc8, 0xa1, 0x27, 0xce, 0xbc, 0xf7, 0x9b, 0x37, 0x33, 0xbf, 0xf7, 0xde, 0x6f, 0x08, 0xc5, 0x13,
0x32, 0x7d, 0x34, 0xf5, 0x69, 0x48, 0x51, 0x81, 0x7f, 0x86, 0xd4, 0xd5, 0xee, 0xf9, 0x64, 0x4a,
0x83, 0xc7, 0x7c, 0x7e, 0x32, 0x1b, 0x3d, 0x1e, 0xd3, 0x31, 0xe5, 0x13, 0x3e, 0x12, 0xf0, 0xda,
0x14, 0xb2, 0x4f, 0x89, 0xeb, 0x52, 0x54, 0x85, 0x92, 0x4d, 0xce, 0x9c, 0x21, 0x31, 0x3d, 0x6b,
0x42, 0x54, 0x49, 0x97, 0xea, 0x45, 0x0c, 0xc2, 0xd4, 0xb5, 0x26, 0x84, 0x01, 0x86, 0xae, 0x43,
0xbc, 0x50, 0x00, 0xd2, 0x02, 0x20, 0x4c, 0x1c, 0xf0, 0x00, 0xb6, 0x23, 0xc0, 0x19, 0xf1, 0x03,
0x87, 0x7a, 0x6a, 0x86, 0x63, 0xb6, 0x84, 0xf5, 0xb9, 0x30, 0xd6, 0x02, 0xc8, 0x3d, 0x25, 0x96,
0x4d, 0x7c, 0xf4, 0x31, 0xc8, 0xe1, 0x7c, 0x2a, 0xf6, 0xda, 0xfe, 0xf4, 0xd6, 0xa3, 0xf8, 0xe4,
0x8f, 0x9e, 0x91, 0x20, 0xb0, 0xc6, 0x64, 0x30, 0x9f, 0x12, 0xcc, 0x21, 0xe8, 0xd7, 0x50, 0x1a,
0xd2, 0xc9, 0xd4, 0x27, 0x01, 0x0f, 0x9c, 0xe6, 0x2b, 0xee, 0x5c, 0x5b, 0xd1, 0x5a, 0x63, 0x70,
0x72, 0x41, 0xad, 0x01, 0x5b, 0x2d, 0x77, 0x16, 0x84, 0xc4, 0x6f, 0x51, 0x6f, 0xe4, 0x8c, 0xd1,
0x13, 0xc8, 0x8f, 0xa8, 0x6b, 0x13, 0x3f, 0x50, 0x25, 0x3d, 0x53, 0x2f, 0x7d, 0xaa, 0xac, 0x83,
0xed, 0x73, 0x47, 0x53, 0x7e, 0x7d, 0x51, 0x4d, 0xe1, 0x18, 0x56, 0xfb, 0x73, 0x1a, 0x72, 0xc2,
0x83, 0x76, 0x21, 0xed, 0xd8, 0x82, 0xa2, 0x66, 0xee, 0xf2, 0xa2, 0x9a, 0xee, 0xb4, 0x71, 0xda,
0xb1, 0xd1, 0x4d, 0xc8, 0xba, 0xd6, 0x09, 0x71, 0x23, 0x72, 0xc4, 0x04, 0xdd, 0x86, 0xa2, 0x4f,
0x2c, 0xdb, 0xa4, 0x9e, 0x3b, 0xe7, 0x94, 0x14, 0x70, 0x81, 0x19, 0x7a, 0x9e, 0x3b, 0x47, 0x3f,
0x02, 0xe4, 0x8c, 0x3d, 0xea, 0x13, 0x73, 0x4a, 0xfc, 0x89, 0xc3, 0x4f, 0x1b, 0xa8, 0x32, 0x47,
0xed, 0x08, 0xcf, 0xd1, 0xda, 0x81, 0xee, 0xc1, 0x56, 0x04, 0xb7, 0x89, 0x4b, 0x42, 0xa2, 0x66,
0x39, 0xb2, 0x2c, 0x8c, 0x6d, 0x6e, 0x43, 0x4f, 0xe0, 0xa6, 0xed, 0x04, 0xd6, 0x89, 0x4b, 0xcc,
0x90, 0x4c, 0xa6, 0xa6, 0xe3, 0xd9, 0xe4, 0x15, 0x09, 0xd4, 0x1c, 0xc7, 0xa2, 0xc8, 0x37, 0x20,
0x93, 0x69, 0x47, 0x78, 0xd0, 0x2e, 0xe4, 0xa6, 0xd6, 0x2c, 0x20, 0xb6, 0x9a, 0xe7, 0x98, 0x68,
0xc6, 0x58, 0x12, 0x15, 0x10, 0xa8, 0xca, 0x55, 0x96, 0xda, 0xdc, 0x11, 0xb3, 0x14, 0xc1, 0x6a,
0xff, 0x4e, 0x43, 0x4e, 0x78, 0xd0, 0x47, 0x2b, 0x96, 0xca, 0xcd, 0x5d, 0x86, 0xfa, 0xe7, 0x45,
0xb5, 0x20, 0x7c, 0x9d, 0x76, 0x82, 0x35, 0x04, 0x72, 0xa2, 0xa2, 0xf8, 0x18, 0xdd, 0x81, 0xa2,
0x65, 0xdb, 0x2c, 0x7b, 0x24, 0x50, 0x33, 0x7a, 0xa6, 0x5e, 0xc4, 0x6b, 0x03, 0xfa, 0xd9, 0x66,
0x35, 0xc8, 0x57, 0xeb, 0xe7, 0x5d, 0x65, 0xc0, 0x52, 0x31, 0x24, 0x7e, 0x54, 0xc1, 0x59, 0xbe,
0x5f, 0x81, 0x19, 0x78, 0xfd, 0xde, 0x85, 0xf2, 0xc4, 0x7a, 0x65, 0x06, 0xe4, 0x0f, 0x33, 0xe2,
0x0d, 0x09, 0xa7, 0x2b, 0x83, 0x4b, 0x13, 0xeb, 0x55, 0x3f, 0x32, 0xa1, 0x0a, 0x80, 0xe3, 0x85,
0x3e, 0xb5, 0x67, 0x43, 0xe2, 0x47, 0x5c, 0x25, 0x2c, 0xe8, 0x27, 0x50, 0xe0, 0x64, 0x9b, 0x8e,
0xad, 0x16, 0x74, 0xa9, 0x2e, 0x37, 0xb5, 0xe8, 0xe2, 0x79, 0x4e, 0x35, 0xbf, 0x77, 0x3c, 0xc4,
0x79, 0x8e, 0xed, 0xd8, 0xe8, 0x97, 0xa0, 0x05, 0x2f, 0x1d, 0x96, 0x28, 0x11, 0x29, 0x74, 0xa8,
0x67, 0xfa, 0x64, 0x42, 0xcf, 0x2c, 0x37, 0x50, 0x8b, 0x7c, 0x1b, 0x95, 0x21, 0x3a, 0x09, 0x00,
0x8e, 0xfc, 0xb5, 0x1e, 0x64, 0x79, 0x44, 0x96, 0x45, 0x51, 0xac, 0x51, 0xf7, 0x46, 0x33, 0xf4,
0x08, 0xb2, 0x23, 0xc7, 0x25, 0x81, 0x9a, 0xe6, 0x39, 0x44, 0x89, 0x4a, 0x77, 0x5c, 0xd2, 0xf1,
0x46, 0x34, 0xca, 0xa2, 0x80, 0xd5, 0x8e, 0xa1, 0xc4, 0x03, 0x1e, 0x4f, 0x6d, 0x2b, 0x24, 0xff,
0xb7, 0xb0, 0xff, 0x95, 0xa1, 0x10, 0x7b, 0x56, 0x49, 0x97, 0x12, 0x49, 0x47, 0x20, 0x07, 0xce,
0x57, 0x84, 0xf7, 0x48, 0x06, 0xf3, 0x31, 0xfa, 0x10, 0x60, 0x42, 0x6d, 0x67, 0xe4, 0x10, 0xdb,
0x0c, 0x78, 0xca, 0x32, 0xb8, 0x18, 0x5b, 0xfa, 0xe8, 0x09, 0x94, 0x56, 0xee, 0x93, 0xb9, 0x5a,
0xe6, 0x9c, 0x7f, 0x10, 0x73, 0xde, 0x3f, 0xa5, 0x7e, 0xd8, 0x69, 0xe3, 0x55, 0x88, 0xe6, 0x9c,
0x95, 0x74, 0x2c, 0x4f, 0x8c, 0xd8, 0x8d, 0x92, 0x7e, 0x4e, 0x86, 0x21, 0x5d, 0x35, 0x7e, 0x04,
0x43, 0x1a, 0x14, 0x56, 0x35, 0x01, 0xfc, 0x00, 0xab, 0x39, 0xfa, 0x31, 0xe4, 0x4e, 0x5c, 0x3a,
0x7c, 0x19, 0xf7, 0xc7, 0x8d, 0x75, 0xb0, 0x26, 0xb3, 0x27, 0x58, 0x88, 0x80, 0x4c, 0x26, 0x83,
0xf9, 0xc4, 0x75, 0xbc, 0x97, 0x66, 0x68, 0xf9, 0x63, 0x12, 0xaa, 0x3b, 0x42, 0x26, 0x23, 0xeb,
0x80, 0x1b, 0x99, 0xdc, 0x8a, 0x05, 0xe6, 0xa9, 0x15, 0x9c, 0xaa, 0x88, 0xb5, 0x11, 0x06, 0x61,
0x7a, 0x6a, 0x05, 0xa7, 0x68, 0x2f, 0x52, 0x4f, 0xa1, 0x85, 0xbb, 0xd7, 0xd9, 0x4f, 0xc8, 0xa7,
0x0e, 0xa5, 0xab, 0xf2, 0xb2, 0x85, 0x93, 0x26, 0xb6, 0xdd, 0x8a, 0x48, 0x2f, 0x50, 0x4b, 0xba,
0x54, 0xcf, 0xae, 0x79, 0xeb, 0x06, 0xe8, 0x31, 0x88, 0xcd, 0x4d, 0x9e, 0xa2, 0x2d, 0xe6, 0x6f,
0x2a, 0x97, 0x17, 0xd5, 0x32, 0xb6, 0xce, 0xf9, 0x55, 0xfb, 0xce, 0x57, 0x04, 0x17, 0x4f, 0xe2,
0x21, 0xdb, 0xd3, 0xa5, 0x43, 0xcb, 0x35, 0x47, 0xae, 0x35, 0x0e, 0xd4, 0x6f, 0xf3, 0x7c, 0x53,
0xe0, 0xb6, 0x7d, 0x66, 0x42, 0x2a, 0x53, 0x17, 0xa6, 0x58, 0x76, 0x24, 0x4d, 0xf1, 0x14, 0xd5,
0x21, 0xef, 0x78, 0x67, 0x96, 0xeb, 0x44, 0x82, 0xd4, 0xdc, 0xbe, 0xbc, 0xa8, 0x02, 0xb6, 0xce,
0x3b, 0xc2, 0x8a, 0x63, 0x37, 0x63, 0xd3, 0xa3, 0x1b, 0xda, 0x59, 0xe0, 0xa1, 0xb6, 0x3c, 0x9a,
0xd0, 0xcd, 0x5f, 0xc8, 0x7f, 0xfa, 0xba, 0x9a, 0xaa, 0x79, 0x50, 0x5c, 0x65, 0x85, 0x55, 0x1b,
0x67, 0x36, 0xc3, 0x99, 0xe5, 0x63, 0x56, 0xea, 0x74, 0x34, 0x0a, 0x48, 0xc8, 0xeb, 0x32, 0x83,
0xa3, 0xd9, 0xaa, 0x32, 0xd3, 0x9c, 0x16, 0x51, 0x99, 0xb7, 0xa1, 0x78, 0x4e, 0xac, 0x97, 0x22,
0x3d, 0x82, 0xd1, 0x02, 0x33, 0xb0, 0xe4, 0x44, 0xfb, 0xfd, 0x0a, 0x72, 0xa2, 0xa4, 0xd0, 0x67,
0x50, 0x18, 0xd2, 0x99, 0x17, 0xae, 0xdf, 0x9b, 0x9d, 0xa4, 0x5c, 0x71, 0x4f, 0x54, 0x27, 0x2b,
0x60, 0x6d, 0x1f, 0xf2, 0x91, 0x0b, 0x3d, 0x58, 0x69, 0xa9, 0xdc, 0xbc, 0x75, 0xa5, 0xbc, 0x37,
0x1f, 0xa0, 0x33, 0xcb, 0x9d, 0x89, 0x83, 0xca, 0x58, 0x4c, 0x6a, 0x7f, 0x95, 0x20, 0x8f, 0x59,
0xc5, 0x06, 0x61, 0xe2, 0xe9, 0xca, 0x6e, 0x3c, 0x5d, 0xeb, 0x26, 0x4f, 0x6f, 0x34, 0x79, 0xdc,
0xa7, 0x99, 0x44, 0x9f, 0xae, 0x59, 0x92, 0xbf, 0x93, 0xa5, 0x6c, 0x82, 0xa5, 0x98, 0xe5, 0x5c,
0x82, 0xe5, 0x07, 0xb0, 0x3d, 0xf2, 0xe9, 0x84, 0x3f, 0x4e, 0xd4, 0xb7, 0xfc, 0x79, 0xa4, 0xa4,
0x5b, 0xcc, 0x3a, 0x88, 0x8d, 0x9b, 0x04, 0x17, 0x36, 0x09, 0xae, 0x99, 0x50, 0xc0, 0x24, 0x98,
0x52, 0x2f, 0x20, 0xef, 0xbc, 0x13, 0x02, 0xd9, 0xb6, 0x42, 0x8b, 0xdf, 0xa8, 0x8c, 0xf9, 0x18,
0x3d, 0x04, 0x79, 0x48, 0x6d, 0x71, 0x9f, 0xed, 0x64, 0xbb, 0x1a, 0xbe, 0x4f, 0xfd, 0x16, 0xb5,
0x09, 0xe6, 0x80, 0xda, 0x14, 0x94, 0x36, 0x3d, 0xf7, 0x5c, 0x6a, 0xd9, 0x47, 0x3e, 0x1d, 0xb3,
0x17, 0xe4, 0x9d, 0x4a, 0xd8, 0x86, 0xfc, 0x8c, 0x6b, 0x65, 0xac, 0x85, 0xf7, 0x37, 0xbb, 0xf1,
0x6a, 0x20, 0x21, 0xac, 0xb1, 0xce, 0x44, 0x4b, 0x6b, 0x7f, 0x97, 0x40, 0x7b, 0x37, 0x1a, 0x75,
0xa0, 0x24, 0x90, 0x66, 0xe2, 0xa7, 0xa9, 0xfe, 0x3e, 0x1b, 0x71, 0x21, 0x80, 0xd9, 0x6a, 0xfc,
0x9d, 0x2f, 0x6e, 0x42, 0x17, 0x33, 0xef, 0xa7, 0x8b, 0x0f, 0x61, 0x4b, 0x28, 0x42, 0xfc, 0x7f,
0x21, 0xeb, 0x99, 0x7a, 0xb6, 0x99, 0x56, 0x52, 0xb8, 0x7c, 0x22, 0xda, 0x8c, 0xdb, 0x6b, 0x39,
0x90, 0x8f, 0x1c, 0x6f, 0x5c, 0xab, 0x42, 0xb6, 0xe5, 0x52, 0x9e, 0xb0, 0x9c, 0x4f, 0xac, 0x80,
0x7a, 0x31, 0x8f, 0x62, 0xb6, 0xf7, 0xb7, 0x34, 0x94, 0x12, 0xff, 0x7e, 0xe8, 0x09, 0x6c, 0xb7,
0x0e, 0x8f, 0xfb, 0x03, 0x03, 0x9b, 0xad, 0x5e, 0x77, 0xbf, 0x73, 0xa0, 0xa4, 0xb4, 0x3b, 0x8b,
0xa5, 0xae, 0x4e, 0xd6, 0xa0, 0xcd, 0xdf, 0xba, 0x2a, 0x64, 0x3b, 0xdd, 0xb6, 0xf1, 0x3b, 0x45,
0xd2, 0x6e, 0x2e, 0x96, 0xba, 0x92, 0x00, 0x8a, 0x37, 0xf2, 0x13, 0x28, 0x73, 0x80, 0x79, 0x7c,
0xd4, 0x6e, 0x0c, 0x0c, 0x25, 0xad, 0x69, 0x8b, 0xa5, 0xbe, 0x7b, 0x15, 0x17, 0x71, 0x7e, 0x0f,
0xf2, 0xd8, 0xf8, 0xed, 0xb1, 0xd1, 0x1f, 0x28, 0x19, 0x6d, 0x77, 0xb1, 0xd4, 0x51, 0x02, 0x18,
0xb7, 0xd4, 0x03, 0x28, 0x60, 0xa3, 0x7f, 0xd4, 0xeb, 0xf6, 0x0d, 0x45, 0xd6, 0x7e, 0xb0, 0x58,
0xea, 0x37, 0x36, 0x50, 0x51, 0x95, 0xfe, 0x14, 0x76, 0xda, 0xbd, 0x2f, 0xba, 0x87, 0xbd, 0x46,
0xdb, 0x3c, 0xc2, 0xbd, 0x03, 0x6c, 0xf4, 0xfb, 0x4a, 0x56, 0xab, 0x2e, 0x96, 0xfa, 0xed, 0x04,
0xfe, 0x5a, 0xd1, 0x7d, 0x08, 0xf2, 0x51, 0xa7, 0x7b, 0xa0, 0xe4, 0xb4, 0x1b, 0x8b, 0xa5, 0xfe,
0x41, 0x02, 0xca, 0x48, 0x65, 0x37, 0x6e, 0x1d, 0xf6, 0xfa, 0x86, 0x92, 0xbf, 0x76, 0x63, 0x4e,
0xf6, 0xde, 0xef, 0x01, 0x5d, 0xff, 0x3b, 0x46, 0xf7, 0x41, 0xee, 0xf6, 0xba, 0x86, 0x92, 0x12,
0xf7, 0xbf, 0x8e, 0xe8, 0x52, 0x8f, 0xa0, 0x1a, 0x64, 0x0e, 0xbf, 0xfc, 0x5c, 0x91, 0xb4, 0x1f,
0x2e, 0x96, 0xfa, 0xad, 0xeb, 0xa0, 0xc3, 0x2f, 0x3f, 0xdf, 0xa3, 0x50, 0x4a, 0x06, 0xae, 0x41,
0xe1, 0x99, 0x31, 0x68, 0xb4, 0x1b, 0x83, 0x86, 0x92, 0x12, 0x47, 0x8a, 0xdd, 0xcf, 0x48, 0x68,
0xf1, 0x26, 0xbc, 0x03, 0xd9, 0xae, 0xf1, 0xdc, 0xc0, 0x8a, 0xa4, 0xed, 0x2c, 0x96, 0xfa, 0x56,
0x0c, 0xe8, 0x92, 0x33, 0xe2, 0xa3, 0x0a, 0xe4, 0x1a, 0x87, 0x5f, 0x34, 0x5e, 0xf4, 0x95, 0xb4,
0x86, 0x16, 0x4b, 0x7d, 0x3b, 0x76, 0x37, 0xdc, 0x73, 0x6b, 0x1e, 0xec, 0xfd, 0x47, 0x82, 0x72,
0xf2, 0x8d, 0x43, 0x15, 0x90, 0xf7, 0x3b, 0x87, 0x46, 0xbc, 0x5d, 0xd2, 0xc7, 0xc6, 0xa8, 0x0e,
0xc5, 0x76, 0x07, 0x1b, 0xad, 0x41, 0x0f, 0xbf, 0x88, 0xef, 0x92, 0x04, 0xb5, 0x1d, 0x9f, 0x17,
0xf8, 0x1c, 0xfd, 0x1c, 0xca, 0xfd, 0x17, 0xcf, 0x0e, 0x3b, 0xdd, 0xdf, 0x98, 0x3c, 0x62, 0x5a,
0x7b, 0xb8, 0x58, 0xea, 0x77, 0x37, 0xc0, 0x64, 0xea, 0x93, 0xa1, 0x15, 0x12, 0xbb, 0x2f, 0xde,
0x6b, 0xe6, 0x2c, 0x48, 0xa8, 0x05, 0x3b, 0xf1, 0xd2, 0xf5, 0x66, 0x19, 0xed, 0x93, 0xc5, 0x52,
0xff, 0xe8, 0x7b, 0xd7, 0xaf, 0x76, 0x2f, 0x48, 0xe8, 0x3e, 0xe4, 0xa3, 0x20, 0x71, 0x25, 0x25,
0x97, 0x46, 0x0b, 0xf6, 0xfe, 0x22, 0x41, 0x71, 0x25, 0x57, 0x8c, 0xf0, 0x6e, 0xcf, 0x34, 0x30,
0xee, 0xe1, 0x98, 0x81, 0x95, 0xb3, 0x4b, 0xf9, 0x10, 0xdd, 0x85, 0xfc, 0x81, 0xd1, 0x35, 0x70,
0xa7, 0x15, 0x37, 0xc6, 0x0a, 0x72, 0x40, 0x3c, 0xe2, 0x3b, 0x43, 0xf4, 0x31, 0x94, 0xbb, 0x3d,
0xb3, 0x7f, 0xdc, 0x7a, 0x1a, 0x5f, 0x9d, 0xef, 0x9f, 0x08, 0xd5, 0x9f, 0x0d, 0x4f, 0x39, 0x9f,
0x7b, 0xac, 0x87, 0x9e, 0x37, 0x0e, 0x3b, 0x6d, 0x01, 0xcd, 0x68, 0xea, 0x62, 0xa9, 0xdf, 0x5c,
0x41, 0xa3, 0x47, 0x9a, 0x61, 0xf7, 0x6c, 0xa8, 0x7c, 0xbf, 0x30, 0x21, 0x1d, 0x72, 0x8d, 0xa3,
0x23, 0xa3, 0xdb, 0x8e, 0x4f, 0xbf, 0xf6, 0x35, 0xa6, 0x53, 0xe2, 0xd9, 0x0c, 0xb1, 0xdf, 0xc3,
0x07, 0xc6, 0x20, 0x3e, 0xfc, 0x1a, 0xb1, 0x4f, 0xd9, 0xcf, 0x52, 0xb3, 0xfe, 0xfa, 0x9b, 0x4a,
0xea, 0xcd, 0x37, 0x95, 0xd4, 0xeb, 0xcb, 0x8a, 0xf4, 0xe6, 0xb2, 0x22, 0xfd, 0xeb, 0xb2, 0x92,
0xfa, 0xf6, 0xb2, 0x22, 0xfd, 0xf1, 0x6d, 0x25, 0xf5, 0xf5, 0xdb, 0x8a, 0xf4, 0xe6, 0x6d, 0x25,
0xf5, 0x8f, 0xb7, 0x95, 0xd4, 0x49, 0x8e, 0x8b, 0xda, 0x67, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff,
0x39, 0xd2, 0xc3, 0x6e, 0x32, 0x0f, 0x00, 0x00,
}
func (m *Hello) Marshal() (dAtA []byte, err error) {
@ -1459,6 +1461,15 @@ func (m *FileInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i--
dAtA[i] = 0xc0
}
if len(m.BlocksHash) > 0 {
i -= len(m.BlocksHash)
copy(dAtA[i:], m.BlocksHash)
i = encodeVarintBep(dAtA, i, uint64(len(m.BlocksHash)))
i--
dAtA[i] = 0x1
i--
dAtA[i] = 0x92
}
if len(m.SymlinkTarget) > 0 {
i -= len(m.SymlinkTarget)
copy(dAtA[i:], m.SymlinkTarget)
@ -2185,6 +2196,10 @@ func (m *FileInfo) ProtoSize() (n int) {
if l > 0 {
n += 2 + l + sovBep(uint64(l))
}
l = len(m.BlocksHash)
if l > 0 {
n += 2 + l + sovBep(uint64(l))
}
if m.LocalFlags != 0 {
n += 2 + sovBep(uint64(m.LocalFlags))
}
@ -3835,6 +3850,40 @@ func (m *FileInfo) Unmarshal(dAtA []byte) error {
}
m.SymlinkTarget = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 18:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field BlocksHash", wireType)
}
var byteLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowBep
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
byteLen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if byteLen < 0 {
return ErrInvalidLengthBep
}
postIndex := iNdEx + byteLen
if postIndex < 0 {
return ErrInvalidLengthBep
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.BlocksHash = append(m.BlocksHash[:0], dAtA[iNdEx:postIndex]...)
if m.BlocksHash == nil {
m.BlocksHash = []byte{}
}
iNdEx = postIndex
case 1000:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field LocalFlags", wireType)

View File

@ -106,8 +106,9 @@ message FileInfo {
uint64 modified_by = 12 [(gogoproto.customtype) = "ShortID", (gogoproto.nullable) = false];
Vector version = 9 [(gogoproto.nullable) = false];
int64 sequence = 10;
repeated BlockInfo Blocks = 16 [(gogoproto.nullable) = false];
repeated BlockInfo blocks = 16 [(gogoproto.nullable) = false];
string symlink_target = 17;
bytes blocks_hash = 18;
FileInfoType type = 2;
uint32 permissions = 4;
int32 modified_ns = 11;

View File

@ -14,6 +14,7 @@ import (
"time"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/sha256"
)
const (
@ -328,3 +329,11 @@ func (f Folder) Description() string {
}
return fmt.Sprintf("%q (%s)", f.Label, f.ID)
}
func BlocksHash(bs []BlockInfo) []byte {
h := sha256.New()
for _, b := range bs {
_, _ = h.Write(b.Hash)
}
return h.Sum(nil)
}

View File

@ -111,6 +111,7 @@ func (ph *parallelHasher) hashFiles(ctx context.Context) {
}
f.Blocks = blocks
f.BlocksHash = protocol.BlocksHash(blocks)
// The size we saw when initially deciding to hash the file
// might not have been the size it actually had when we hashed