From 19c7cd99f5e7fad53fb3bd9eb1d6f2342aa37da7 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Mon, 16 Apr 2018 20:08:50 +0200 Subject: [PATCH] all: Implement variable sized blocks (fixes #4807) --- cmd/stfileinfo/main.go | 4 +- cmd/syncthing/gui.go | 4 +- cmd/syncthing/mocked_model_test.go | 2 +- cmd/syncthing/usage_report.go | 4 +- lib/config/folderconfiguration.go | 1 + lib/db/set.go | 1 + lib/db/structs.go | 26 +- lib/db/structs.pb.go | 111 +++++--- lib/db/structs.proto | 3 + lib/model/model.go | 10 +- lib/model/model_test.go | 32 +-- lib/model/rwfolder.go | 12 +- lib/model/rwfolder_test.go | 10 +- lib/model/sharedpullerstate.go | 14 +- lib/protocol/bep.pb.go | 247 ++++++++++-------- lib/protocol/bep.proto | 1 + lib/protocol/bep_extensions.go | 23 +- lib/protocol/protocol.go | 47 +++- lib/protocol/protocol_test.go | 76 +++--- lib/scanner/blockqueue.go | 6 +- lib/scanner/blocks_test.go | 2 +- .../{infinitefs_test.go => virtualfs_test.go} | 42 +++ lib/scanner/walk.go | 44 +++- lib/scanner/walk_test.go | 100 +++++-- test/h1/config.xml | 2 + test/h2/config.xml | 3 + test/h3/config.xml | 2 + 27 files changed, 536 insertions(+), 293 deletions(-) rename lib/scanner/{infinitefs_test.go => virtualfs_test.go} (77%) diff --git a/cmd/stfileinfo/main.go b/cmd/stfileinfo/main.go index 4dda3ca7a..2efb99611 100644 --- a/cmd/stfileinfo/main.go +++ b/cmd/stfileinfo/main.go @@ -68,8 +68,8 @@ func main() { } blockSize := int(fi.Size()) - if *standardBlocks || blockSize < protocol.BlockSize { - blockSize = protocol.BlockSize + if *standardBlocks || blockSize < protocol.MinBlockSize { + blockSize = protocol.BlockSize(fi.Size()) } bs, err := scanner.Blocks(context.TODO(), fd, blockSize, fi.Size(), nil, true) if err != nil { diff --git a/cmd/syncthing/gui.go b/cmd/syncthing/gui.go index 0adca93c8..29fc35b9e 100644 --- a/cmd/syncthing/gui.go +++ b/cmd/syncthing/gui.go @@ -94,7 +94,7 @@ type modelIntf interface { CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool) CurrentGlobalFile(folder string, file string) (protocol.FileInfo, bool) ResetFolder(folder string) - Availability(folder, file string, version protocol.Vector, block protocol.BlockInfo) []model.Availability + Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) []model.Availability GetIgnores(folder string) ([]string, []string, error) GetFolderVersions(folder string) (map[string][]versioner.FileVersion, error) RestoreFolderVersions(folder string, versions map[string]time.Time) (map[string]string, error) @@ -828,7 +828,7 @@ func (s *apiService) getDBFile(w http.ResponseWriter, r *http.Request) { return } - av := s.model.Availability(folder, file, protocol.Vector{}, protocol.BlockInfo{}) + av := s.model.Availability(folder, gf, protocol.BlockInfo{}) sendJSON(w, map[string]interface{}{ "global": jsonFileInfo(gf), "local": jsonFileInfo(lf), diff --git a/cmd/syncthing/mocked_model_test.go b/cmd/syncthing/mocked_model_test.go index 13aed6db2..5841b99a0 100644 --- a/cmd/syncthing/mocked_model_test.go +++ b/cmd/syncthing/mocked_model_test.go @@ -64,7 +64,7 @@ func (m *mockedModel) CurrentGlobalFile(folder string, file string) (protocol.Fi func (m *mockedModel) ResetFolder(folder string) { } -func (m *mockedModel) Availability(folder, file string, version protocol.Vector, block protocol.BlockInfo) []model.Availability { +func (m *mockedModel) Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) []model.Availability { return nil } diff --git a/cmd/syncthing/usage_report.go b/cmd/syncthing/usage_report.go index 0c1da16cf..403dab39a 100644 --- a/cmd/syncthing/usage_report.go +++ b/cmd/syncthing/usage_report.go @@ -399,7 +399,7 @@ func (usageReportingService) String() string { // cpuBench returns CPU performance as a measure of single threaded SHA-256 MiB/s func cpuBench(iterations int, duration time.Duration, useWeakHash bool) float64 { - dataSize := 16 * protocol.BlockSize + dataSize := 16 * protocol.MinBlockSize bs := make([]byte, dataSize) rand.Reader.Read(bs) @@ -420,7 +420,7 @@ func cpuBenchOnce(duration time.Duration, useWeakHash bool, bs []byte) float64 { b := 0 for time.Since(t0) < duration { r := bytes.NewReader(bs) - blocksResult, _ = scanner.Blocks(context.TODO(), r, protocol.BlockSize, int64(len(bs)), nil, useWeakHash) + blocksResult, _ = scanner.Blocks(context.TODO(), r, protocol.MinBlockSize, int64(len(bs)), nil, useWeakHash) b += len(bs) } d := time.Since(t0) diff --git a/lib/config/folderconfiguration.go b/lib/config/folderconfiguration.go index 4300691a3..d40ab6513 100644 --- a/lib/config/folderconfiguration.go +++ b/lib/config/folderconfiguration.go @@ -52,6 +52,7 @@ type FolderConfiguration struct { Paused bool `xml:"paused" json:"paused"` WeakHashThresholdPct int `xml:"weakHashThresholdPct" json:"weakHashThresholdPct"` // Use weak hash if more than X percent of the file has changed. Set to -1 to always use weak hash. MarkerName string `xml:"markerName" json:"markerName"` + UseLargeBlocks bool `xml:"useLargeBlocks" json:"useLargeBlocks"` cachedFilesystem fs.Filesystem diff --git a/lib/db/set.go b/lib/db/set.go index 2d78f1a66..d6cf2630d 100644 --- a/lib/db/set.go +++ b/lib/db/set.go @@ -43,6 +43,7 @@ type FileIntf interface { IsSymlink() bool HasPermissionBits() bool SequenceNo() int64 + BlockSize() int } // The Iterator is called with either a protocol.FileInfo or a diff --git a/lib/db/structs.go b/lib/db/structs.go index c8d4e6ed5..60ff35823 100644 --- a/lib/db/structs.go +++ b/lib/db/structs.go @@ -17,8 +17,8 @@ import ( ) func (f FileInfoTruncated) String() string { - return fmt.Sprintf("File{Name:%q, Permissions:0%o, Modified:%v, Version:%v, Length:%d, Deleted:%v, Invalid:%v, NoPermissions:%v}", - f.Name, f.Permissions, f.ModTime(), f.Version, f.Size, f.Deleted, f.Invalid, f.NoPermissions) + return fmt.Sprintf("File{Name:%q, Permissions:0%o, Modified:%v, Version:%v, Length:%d, Deleted:%v, Invalid:%v, NoPermissions:%v, BlockSize:%d}", + f.Name, f.Permissions, f.ModTime(), f.Version, f.Size, f.Deleted, f.Invalid, f.NoPermissions, f.RawBlockSize) } func (f FileInfoTruncated) IsDeleted() bool { @@ -56,6 +56,13 @@ func (f FileInfoTruncated) FileSize() int64 { return f.Size } +func (f FileInfoTruncated) BlockSize() int { + if f.RawBlockSize == 0 { + return protocol.MinBlockSize + } + return int(f.RawBlockSize) +} + func (f FileInfoTruncated) FileName() string { return f.Name } @@ -70,12 +77,13 @@ func (f FileInfoTruncated) SequenceNo() int64 { func (f FileInfoTruncated) ConvertToInvalidFileInfo(invalidatedBy protocol.ShortID) protocol.FileInfo { return protocol.FileInfo{ - Name: f.Name, - Type: f.Type, - ModifiedS: f.ModifiedS, - ModifiedNs: f.ModifiedNs, - ModifiedBy: invalidatedBy, - Invalid: true, - Version: f.Version, + Name: f.Name, + Type: f.Type, + ModifiedS: f.ModifiedS, + ModifiedNs: f.ModifiedNs, + ModifiedBy: invalidatedBy, + Invalid: true, + Version: f.Version, + RawBlockSize: f.RawBlockSize, } } diff --git a/lib/db/structs.pb.go b/lib/db/structs.pb.go index fffa16e97..ae4235a3f 100644 --- a/lib/db/structs.pb.go +++ b/lib/db/structs.pb.go @@ -70,7 +70,9 @@ type FileInfoTruncated struct { NoPermissions bool `protobuf:"varint,8,opt,name=no_permissions,json=noPermissions,proto3" json:"no_permissions,omitempty"` Version protocol.Vector `protobuf:"bytes,9,opt,name=version" json:"version"` Sequence int64 `protobuf:"varint,10,opt,name=sequence,proto3" json:"sequence,omitempty"` - SymlinkTarget string `protobuf:"bytes,17,opt,name=symlink_target,json=symlinkTarget,proto3" json:"symlink_target,omitempty"` + RawBlockSize int32 `protobuf:"varint,13,opt,name=block_size,json=blockSize,proto3" json:"block_size,omitempty"` + // repeated BlockInfo Blocks = 16 + SymlinkTarget string `protobuf:"bytes,17,opt,name=symlink_target,json=symlinkTarget,proto3" json:"symlink_target,omitempty"` } func (m *FileInfoTruncated) Reset() { *m = FileInfoTruncated{} } @@ -277,6 +279,11 @@ func (m *FileInfoTruncated) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintStructs(dAtA, i, uint64(m.ModifiedBy)) } + if m.RawBlockSize != 0 { + dAtA[i] = 0x68 + i++ + i = encodeVarintStructs(dAtA, i, uint64(m.RawBlockSize)) + } if len(m.SymlinkTarget) > 0 { dAtA[i] = 0x8a i++ @@ -472,6 +479,9 @@ func (m *FileInfoTruncated) ProtoSize() (n int) { if m.ModifiedBy != 0 { n += 1 + sovStructs(uint64(m.ModifiedBy)) } + if m.RawBlockSize != 0 { + n += 1 + sovStructs(uint64(m.RawBlockSize)) + } l = len(m.SymlinkTarget) if l > 0 { n += 2 + l + sovStructs(uint64(l)) @@ -1028,6 +1038,25 @@ func (m *FileInfoTruncated) Unmarshal(dAtA []byte) error { break } } + case 13: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field RawBlockSize", wireType) + } + m.RawBlockSize = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStructs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.RawBlockSize |= (int32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } case 17: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field SymlinkTarget", wireType) @@ -1481,43 +1510,45 @@ var ( func init() { proto.RegisterFile("structs.proto", fileDescriptorStructs) } var fileDescriptorStructs = []byte{ - // 598 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0xcd, 0x6a, 0xdb, 0x4c, - 0x14, 0xb5, 0x62, 0xc9, 0xb1, 0x47, 0x71, 0xbe, 0x2f, 0x43, 0x08, 0xc2, 0x50, 0x5b, 0x18, 0x0a, - 0xa2, 0x50, 0xa5, 0x4d, 0xe8, 0xa6, 0xdd, 0xb9, 0x21, 0x10, 0x28, 0x6d, 0x99, 0x84, 0xac, 0x0a, - 0xc1, 0x92, 0xae, 0x9d, 0xa1, 0xf2, 0x8c, 0xa3, 0x19, 0x07, 0xd4, 0x27, 0xe9, 0x32, 0x0f, 0xd3, - 0x45, 0x96, 0x5d, 0x77, 0x11, 0x5a, 0xf7, 0x39, 0x0a, 0x45, 0x77, 0x24, 0x45, 0xe9, 0xaa, 0xdd, - 0xdd, 0x73, 0xff, 0xef, 0x39, 0x33, 0xa4, 0xaf, 0x74, 0xb6, 0x8a, 0xb5, 0x0a, 0x97, 0x99, 0xd4, - 0x92, 0x6e, 0x24, 0xd1, 0xe0, 0xe9, 0x9c, 0xeb, 0xcb, 0x55, 0x14, 0xc6, 0x72, 0xb1, 0x3f, 0x97, - 0x73, 0xb9, 0x8f, 0xa1, 0x68, 0x35, 0x43, 0x84, 0x00, 0x2d, 0x53, 0x32, 0x78, 0xd1, 0x48, 0x57, - 0xb9, 0x88, 0xf5, 0x25, 0x17, 0xf3, 0x86, 0x95, 0xf2, 0xc8, 0x74, 0x88, 0x65, 0xba, 0x1f, 0xc1, - 0xd2, 0x94, 0x8d, 0xaf, 0x88, 0x7b, 0xcc, 0x53, 0x38, 0x87, 0x4c, 0x71, 0x29, 0xe8, 0x33, 0xb2, - 0x79, 0x6d, 0x4c, 0xcf, 0xf2, 0xad, 0xc0, 0x3d, 0xf8, 0x3f, 0xac, 0x8a, 0xc2, 0x73, 0x88, 0xb5, - 0xcc, 0x26, 0xf6, 0xed, 0xdd, 0xa8, 0xc5, 0xaa, 0x34, 0xba, 0x47, 0x3a, 0x09, 0x5c, 0xf3, 0x18, - 0xbc, 0x0d, 0xdf, 0x0a, 0xb6, 0x58, 0x89, 0xa8, 0x47, 0x36, 0xb9, 0xb8, 0x9e, 0xa6, 0x3c, 0xf1, - 0xda, 0xbe, 0x15, 0x74, 0x59, 0x05, 0xc7, 0xc7, 0xc4, 0x2d, 0xc7, 0xbd, 0xe1, 0x4a, 0xd3, 0xe7, - 0xa4, 0x5b, 0xf6, 0x52, 0x9e, 0xe5, 0xb7, 0x03, 0xf7, 0xe0, 0xbf, 0x30, 0x89, 0xc2, 0xc6, 0x56, - 0xe5, 0xc8, 0x3a, 0xed, 0xa5, 0xfd, 0xf9, 0x66, 0xd4, 0x1a, 0xff, 0x6a, 0x93, 0x9d, 0x22, 0xeb, - 0x44, 0xcc, 0xe4, 0x59, 0xb6, 0x12, 0xf1, 0x54, 0x43, 0x42, 0x29, 0xb1, 0xc5, 0x74, 0x01, 0xb8, - 0x7e, 0x8f, 0xa1, 0x4d, 0x9f, 0x10, 0x5b, 0xe7, 0x4b, 0xb3, 0xe1, 0xf6, 0xc1, 0xde, 0xfd, 0x49, - 0x75, 0x79, 0xbe, 0x04, 0x86, 0x39, 0x45, 0xbd, 0xe2, 0x9f, 0x00, 0x97, 0x6e, 0x33, 0xb4, 0xa9, - 0x4f, 0xdc, 0x25, 0x64, 0x0b, 0xae, 0xcc, 0x96, 0xb6, 0x6f, 0x05, 0x7d, 0xd6, 0x74, 0xd1, 0x47, - 0x84, 0x2c, 0x64, 0xc2, 0x67, 0x1c, 0x92, 0x0b, 0xe5, 0x39, 0x58, 0xdb, 0xab, 0x3c, 0xa7, 0x05, - 0x19, 0x09, 0xa4, 0xa0, 0x21, 0xf1, 0x3a, 0x86, 0x8c, 0x12, 0x36, 0x69, 0xda, 0x7c, 0x40, 0x13, - 0x7d, 0x4c, 0xb6, 0x85, 0xbc, 0x68, 0xce, 0xed, 0x62, 0x42, 0x5f, 0xc8, 0xf7, 0x8d, 0xc9, 0x0d, - 0xc5, 0x7a, 0x7f, 0xa7, 0xd8, 0x80, 0x74, 0x15, 0x5c, 0xad, 0x40, 0xc4, 0xe0, 0x11, 0xdc, 0xb4, - 0xc6, 0x74, 0x44, 0xdc, 0xfa, 0x0e, 0xa1, 0x3c, 0xd7, 0xb7, 0x02, 0x87, 0xd5, 0xa7, 0xbd, 0x55, - 0xf4, 0x43, 0x23, 0x21, 0xca, 0xbd, 0x2d, 0xdf, 0x0a, 0xec, 0xc9, 0xab, 0x62, 0xc0, 0xb7, 0xbb, - 0xd1, 0xe1, 0x3f, 0xbc, 0xc1, 0xf0, 0xf4, 0x52, 0x66, 0xfa, 0xe4, 0xe8, 0xbe, 0xfb, 0x24, 0x2f, - 0x6e, 0x56, 0xf9, 0x22, 0xe5, 0xe2, 0xe3, 0x85, 0x9e, 0x66, 0x73, 0xd0, 0xde, 0x0e, 0xca, 0xd8, - 0x2f, 0xbd, 0x67, 0xe8, 0x2c, 0xf5, 0xff, 0x62, 0x91, 0xce, 0x6b, 0xb9, 0x12, 0x5a, 0xd1, 0x5d, - 0xe2, 0xcc, 0x78, 0x0a, 0x0a, 0x55, 0x77, 0x98, 0x01, 0x85, 0x6c, 0x09, 0xcf, 0x90, 0x03, 0x0e, - 0x0a, 0xd5, 0x77, 0x58, 0xd3, 0x85, 0x54, 0x98, 0xce, 0x0a, 0x05, 0x77, 0x58, 0x8d, 0x9b, 0x9a, - 0xd9, 0x18, 0xaa, 0x35, 0xdb, 0x25, 0x4e, 0x94, 0x6b, 0xa8, 0x74, 0x36, 0xe0, 0x01, 0xad, 0x9d, - 0x3f, 0x68, 0x1d, 0x90, 0xae, 0xf9, 0x16, 0x27, 0x47, 0x78, 0xd1, 0x16, 0xab, 0xf1, 0xf8, 0x1d, - 0xe9, 0x99, 0x2b, 0x4e, 0x41, 0xd3, 0x80, 0x74, 0x62, 0x04, 0xe5, 0x57, 0x20, 0xc5, 0x57, 0x30, - 0xe1, 0x52, 0xc6, 0x32, 0x5e, 0xac, 0x17, 0x67, 0x50, 0x3c, 0x79, 0x3c, 0xac, 0xcd, 0x2a, 0x38, - 0xd9, 0xbd, 0xfd, 0x31, 0x6c, 0xdd, 0xae, 0x87, 0xd6, 0xd7, 0xf5, 0xd0, 0xfa, 0xbe, 0x1e, 0xb6, - 0x6e, 0x7e, 0x0e, 0xad, 0xa8, 0x83, 0xc4, 0x1f, 0xfe, 0x0e, 0x00, 0x00, 0xff, 0xff, 0x6e, 0x86, - 0x97, 0x21, 0x6a, 0x04, 0x00, 0x00, + // 630 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0xcf, 0x6a, 0xdb, 0x4c, + 0x10, 0xb7, 0x62, 0xc9, 0xb1, 0x47, 0x76, 0xbe, 0x64, 0x09, 0x41, 0x18, 0x3e, 0x5b, 0x18, 0x0a, + 0xa2, 0x50, 0xb9, 0x4d, 0xe8, 0xa5, 0xbd, 0xb9, 0x21, 0x10, 0x28, 0x6d, 0x59, 0x87, 0x9c, 0x0a, + 0xc6, 0x92, 0xd6, 0xce, 0x12, 0x79, 0xd7, 0xd1, 0xae, 0x52, 0x94, 0x27, 0xe9, 0x31, 0x4f, 0xd0, + 0xa7, 0xe8, 0x21, 0xc7, 0x9e, 0x7b, 0x08, 0xad, 0xfb, 0x22, 0x45, 0xbb, 0x92, 0xa2, 0xf4, 0xd4, + 0xde, 0xe6, 0x37, 0xff, 0x67, 0x7e, 0x33, 0xd0, 0x13, 0x32, 0x49, 0x43, 0x29, 0xfc, 0x75, 0xc2, + 0x25, 0x47, 0x5b, 0x51, 0xd0, 0x7f, 0xb6, 0xa4, 0xf2, 0x22, 0x0d, 0xfc, 0x90, 0xaf, 0xc6, 0x4b, + 0xbe, 0xe4, 0x63, 0x65, 0x0a, 0xd2, 0x85, 0x42, 0x0a, 0x28, 0x49, 0x87, 0xf4, 0x5f, 0xd6, 0xdc, + 0x45, 0xc6, 0x42, 0x79, 0x41, 0xd9, 0xb2, 0x26, 0xc5, 0x34, 0xd0, 0x19, 0x42, 0x1e, 0x8f, 0x03, + 0xb2, 0xd6, 0x61, 0xa3, 0x2b, 0xb0, 0x4f, 0x68, 0x4c, 0xce, 0x49, 0x22, 0x28, 0x67, 0xe8, 0x39, + 0x6c, 0x5f, 0x6b, 0xd1, 0x31, 0x5c, 0xc3, 0xb3, 0x0f, 0x77, 0xfd, 0x32, 0xc8, 0x3f, 0x27, 0xa1, + 0xe4, 0xc9, 0xc4, 0xbc, 0xbb, 0x1f, 0x36, 0x70, 0xe9, 0x86, 0x0e, 0xa0, 0x15, 0x91, 0x6b, 0x1a, + 0x12, 0x67, 0xcb, 0x35, 0xbc, 0x2e, 0x2e, 0x10, 0x72, 0x60, 0x9b, 0xb2, 0xeb, 0x79, 0x4c, 0x23, + 0xa7, 0xe9, 0x1a, 0x5e, 0x1b, 0x97, 0x70, 0x74, 0x02, 0x76, 0x51, 0xee, 0x2d, 0x15, 0x12, 0xbd, + 0x80, 0x76, 0x91, 0x4b, 0x38, 0x86, 0xdb, 0xf4, 0xec, 0xc3, 0xff, 0xfc, 0x28, 0xf0, 0x6b, 0x5d, + 0x15, 0x25, 0x2b, 0xb7, 0x57, 0xe6, 0xe7, 0xdb, 0x61, 0x63, 0xf4, 0xc5, 0x84, 0xbd, 0xdc, 0xeb, + 0x94, 0x2d, 0xf8, 0x59, 0x92, 0xb2, 0x70, 0x2e, 0x49, 0x84, 0x10, 0x98, 0x6c, 0xbe, 0x22, 0xaa, + 0xfd, 0x0e, 0x56, 0x32, 0x7a, 0x0a, 0xa6, 0xcc, 0xd6, 0xba, 0xc3, 0x9d, 0xc3, 0x83, 0x87, 0x91, + 0xaa, 0xf0, 0x6c, 0x4d, 0xb0, 0xf2, 0xc9, 0xe3, 0x05, 0xbd, 0x21, 0xaa, 0xe9, 0x26, 0x56, 0x32, + 0x72, 0xc1, 0x5e, 0x93, 0x64, 0x45, 0x85, 0xee, 0xd2, 0x74, 0x0d, 0xaf, 0x87, 0xeb, 0x2a, 0xf4, + 0x3f, 0xc0, 0x8a, 0x47, 0x74, 0x41, 0x49, 0x34, 0x13, 0x8e, 0xa5, 0x62, 0x3b, 0xa5, 0x66, 0x9a, + 0x2f, 0x23, 0x22, 0x31, 0x91, 0x24, 0x72, 0x5a, 0x7a, 0x19, 0x05, 0xac, 0xaf, 0x69, 0xfb, 0xd1, + 0x9a, 0xd0, 0x13, 0xd8, 0x61, 0x7c, 0x56, 0xaf, 0xdb, 0x56, 0x0e, 0x3d, 0xc6, 0x3f, 0xd4, 0x2a, + 0xd7, 0x18, 0xeb, 0xfc, 0x1d, 0x63, 0x7d, 0x68, 0x0b, 0x72, 0x95, 0x12, 0x16, 0x12, 0x07, 0x54, + 0xa7, 0x15, 0x46, 0x43, 0xb0, 0xab, 0x39, 0x98, 0x70, 0x6c, 0xd7, 0xf0, 0x2c, 0x5c, 0x8d, 0xf6, + 0x4e, 0xa0, 0x8f, 0x35, 0x87, 0x20, 0x73, 0xba, 0xae, 0xe1, 0x99, 0x93, 0xd7, 0x79, 0x81, 0xef, + 0xf7, 0xc3, 0xa3, 0x7f, 0xb8, 0x41, 0x7f, 0x7a, 0xc1, 0x13, 0x79, 0x7a, 0xfc, 0x90, 0x7d, 0x92, + 0xa1, 0x31, 0x40, 0x10, 0xf3, 0xf0, 0x72, 0xa6, 0x28, 0xe8, 0xe5, 0xd5, 0x27, 0xbb, 0x9b, 0xfb, + 0x61, 0x17, 0xcf, 0x3f, 0x4d, 0x72, 0xc3, 0x94, 0xde, 0x10, 0xdc, 0x09, 0x4a, 0x31, 0x5f, 0x92, + 0xc8, 0x56, 0x31, 0x65, 0x97, 0x33, 0x39, 0x4f, 0x96, 0x44, 0x3a, 0x7b, 0x8a, 0xf7, 0x5e, 0xa1, + 0x3d, 0x53, 0xca, 0xe2, 0x60, 0xbe, 0x1a, 0xd0, 0x7a, 0xc3, 0x53, 0x26, 0x05, 0xda, 0x07, 0x6b, + 0x41, 0x63, 0x22, 0xd4, 0x99, 0x58, 0x58, 0x83, 0x9c, 0xe7, 0x88, 0x26, 0x6a, 0x69, 0x94, 0x08, + 0x75, 0x2e, 0x16, 0xae, 0xab, 0xd4, 0xee, 0x74, 0x66, 0xa1, 0x2e, 0xc4, 0xc2, 0x15, 0xae, 0x93, + 0x6c, 0x2a, 0x53, 0x45, 0xf2, 0x3e, 0x58, 0x41, 0x26, 0x49, 0x79, 0x18, 0x1a, 0x3c, 0xe2, 0xa1, + 0xf5, 0x07, 0x0f, 0x7d, 0x68, 0xeb, 0x3f, 0x3a, 0x3d, 0x56, 0x13, 0x75, 0x71, 0x85, 0x47, 0xef, + 0xa1, 0xa3, 0xa7, 0x98, 0x12, 0x89, 0x3c, 0x68, 0x85, 0x0a, 0x14, 0xbf, 0x03, 0xf9, 0xef, 0x68, + 0x73, 0xc1, 0x7b, 0x61, 0xcf, 0xdb, 0x0b, 0x13, 0x92, 0xff, 0x88, 0x1a, 0xac, 0x89, 0x4b, 0x38, + 0xd9, 0xbf, 0xfb, 0x39, 0x68, 0xdc, 0x6d, 0x06, 0xc6, 0xb7, 0xcd, 0xc0, 0xf8, 0xb1, 0x19, 0x34, + 0x6e, 0x7f, 0x0d, 0x8c, 0xa0, 0xa5, 0x98, 0x3a, 0xfa, 0x1d, 0x00, 0x00, 0xff, 0xff, 0x28, 0xfa, + 0x1e, 0x63, 0x9b, 0x04, 0x00, 0x00, } diff --git a/lib/db/structs.proto b/lib/db/structs.proto index 5af65cad2..1aaac22d1 100644 --- a/lib/db/structs.proto +++ b/lib/db/structs.proto @@ -35,6 +35,9 @@ message FileInfoTruncated { bool no_permissions = 8; protocol.Vector version = 9 [(gogoproto.nullable) = false]; int64 sequence = 10; + int32 block_size = 13 [(gogoproto.customname) = "RawBlockSize"]; + + // repeated BlockInfo Blocks = 16 string symlink_target = 17; } diff --git a/lib/model/model.go b/lib/model/model.go index 44b0c5b70..0fb8860ed 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -626,7 +626,7 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) FolderComple } // This might might be more than it really is, because some blocks can be of a smaller size. - downloaded = int64(counts[ft.Name] * protocol.BlockSize) + downloaded = int64(counts[ft.Name] * int(ft.BlockSize())) fileNeed = ft.FileSize() - downloaded if fileNeed < 0 { @@ -1968,7 +1968,6 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su Folder: folderCfg.ID, Subs: subDirs, Matcher: ignores, - BlockSize: protocol.BlockSize, TempLifetime: time.Duration(m.cfg.Options().KeepTemporariesH) * time.Hour, CurrentFiler: cFiler{m, folder}, Filesystem: mtimefs, @@ -1978,6 +1977,7 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su ShortID: m.shortID, ProgressTickIntervalS: folderCfg.ScanProgressIntervalS, UseWeakHashes: weakhash.Enabled, + UseLargeBlocks: folderCfg.UseLargeBlocks, }) if err := runner.CheckHealth(); err != nil { @@ -2513,7 +2513,7 @@ func (m *Model) RestoreFolderVersions(folder string, versions map[string]time.Ti return errors, nil } -func (m *Model) Availability(folder, file string, version protocol.Vector, block protocol.BlockInfo) []Availability { +func (m *Model) Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) []Availability { // The slightly unusual locking sequence here is because we need to hold // pmut for the duration (as the value returned from foldersFiles can // get heavily modified on Close()), but also must acquire fmut before @@ -2532,7 +2532,7 @@ func (m *Model) Availability(folder, file string, version protocol.Vector, block var availabilities []Availability next: - for _, device := range fs.Availability(file) { + for _, device := range fs.Availability(file.Name) { for _, pausedFolder := range m.remotePausedFolders[device] { if pausedFolder == folder { continue next @@ -2545,7 +2545,7 @@ next: } for device := range devices { - if m.deviceDownloads[device].Has(folder, file, version, int32(block.Offset/protocol.BlockSize)) { + if m.deviceDownloads[device].Has(folder, file.Name, file.Version, int32(block.Offset/int64(file.BlockSize()))) { availabilities = append(availabilities, Availability{ID: device, FromTemporary: true}) } } diff --git a/lib/model/model_test.go b/lib/model/model_test.go index 493e05ad8..7b93affd0 100644 --- a/lib/model/model_test.go +++ b/lib/model/model_test.go @@ -177,7 +177,7 @@ func TestRequest(t *testing.T) { defer m.Stop() m.ScanFolder("default") - bs := make([]byte, protocol.BlockSize) + bs := make([]byte, protocol.MinBlockSize) // Existing, shared file bs = bs[:6] @@ -409,20 +409,22 @@ func (f *fakeConnection) addFile(name string, flags uint32, ftype protocol.FileI f.mut.Lock() defer f.mut.Unlock() - blocks, _ := scanner.Blocks(context.TODO(), bytes.NewReader(data), protocol.BlockSize, int64(len(data)), nil, true) + blockSize := protocol.BlockSize(int64(len(data))) + blocks, _ := scanner.Blocks(context.TODO(), bytes.NewReader(data), blockSize, int64(len(data)), nil, true) var version protocol.Vector version = version.Update(f.id.Short()) if ftype == protocol.FileInfoTypeFile || ftype == protocol.FileInfoTypeDirectory { f.files = append(f.files, protocol.FileInfo{ - Name: name, - Type: ftype, - Size: int64(len(data)), - ModifiedS: time.Now().Unix(), - Permissions: flags, - Version: version, - Sequence: time.Now().UnixNano(), - Blocks: blocks, + Name: name, + Type: ftype, + Size: int64(len(data)), + ModifiedS: time.Now().Unix(), + Permissions: flags, + Version: version, + Sequence: time.Now().UnixNano(), + RawBlockSize: int32(blockSize), + Blocks: blocks, }) } else { // Symlink @@ -2803,7 +2805,7 @@ func TestNoRequestsFromPausedDevices(t *testing.T) { files.Update(device1, []protocol.FileInfo{file}) files.Update(device2, []protocol.FileInfo{file}) - avail := m.Availability("default", file.Name, file.Version, file.Blocks[0]) + avail := m.Availability("default", file, file.Blocks[0]) if len(avail) != 0 { t.Errorf("should not be available, no connections") } @@ -2813,7 +2815,7 @@ func TestNoRequestsFromPausedDevices(t *testing.T) { // !!! This is not what I'd expect to happen, as we don't even know if the peer has the original index !!! - avail = m.Availability("default", file.Name, file.Version, file.Blocks[0]) + avail = m.Availability("default", file, file.Blocks[0]) if len(avail) != 2 { t.Errorf("should have two available") } @@ -2833,7 +2835,7 @@ func TestNoRequestsFromPausedDevices(t *testing.T) { m.ClusterConfig(device1, cc) m.ClusterConfig(device2, cc) - avail = m.Availability("default", file.Name, file.Version, file.Blocks[0]) + avail = m.Availability("default", file, file.Blocks[0]) if len(avail) != 2 { t.Errorf("should have two available") } @@ -2841,7 +2843,7 @@ func TestNoRequestsFromPausedDevices(t *testing.T) { m.Closed(&fakeConnection{id: device1}, errDeviceUnknown) m.Closed(&fakeConnection{id: device2}, errDeviceUnknown) - avail = m.Availability("default", file.Name, file.Version, file.Blocks[0]) + avail = m.Availability("default", file, file.Blocks[0]) if len(avail) != 0 { t.Errorf("should have no available") } @@ -2856,7 +2858,7 @@ func TestNoRequestsFromPausedDevices(t *testing.T) { ccp.Folders[0].Paused = true m.ClusterConfig(device1, ccp) - avail = m.Availability("default", file.Name, file.Version, file.Blocks[0]) + avail = m.Availability("default", file, file.Blocks[0]) if len(avail) != 1 { t.Errorf("should have one available") } diff --git a/lib/model/rwfolder.go b/lib/model/rwfolder.go index 9d4292ae7..0c69f851b 100644 --- a/lib/model/rwfolder.go +++ b/lib/model/rwfolder.go @@ -126,7 +126,7 @@ func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver vers if f.PullerMaxPendingKiB == 0 { f.PullerMaxPendingKiB = defaultPullerPendingKiB } - if blockSizeKiB := protocol.BlockSize / 1024; f.PullerMaxPendingKiB < blockSizeKiB { + if blockSizeKiB := protocol.MaxBlockSize / 1024; f.PullerMaxPendingKiB < blockSizeKiB { f.PullerMaxPendingKiB = blockSizeKiB } @@ -1010,7 +1010,7 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c // Check for an old temporary file which might have some blocks we could // reuse. - tempBlocks, err := scanner.HashFile(f.ctx, f.fs, tempName, protocol.BlockSize, nil, false) + tempBlocks, err := scanner.HashFile(f.ctx, f.fs, tempName, file.BlockSize(), nil, false) if err == nil { // Check for any reusable blocks in the temp file tempCopyBlocks, _ := blockDiff(tempBlocks, file.Blocks) @@ -1160,7 +1160,7 @@ func (f *sendReceiveFolder) shortcutFile(file protocol.FileInfo) error { // copierRoutine reads copierStates until the in channel closes and performs // the relevant copies when possible, or passes it to the puller routine. func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pullBlockState, out chan<- *sharedPullerState) { - buf := make([]byte, protocol.BlockSize) + buf := make([]byte, protocol.MinBlockSize) for state := range in { dstFd, err := state.tempFile() @@ -1203,7 +1203,7 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch if len(hashesToFind) > 0 { file, err = f.fs.Open(state.file.Name) if err == nil { - weakHashFinder, err = weakhash.NewFinder(file, protocol.BlockSize, hashesToFind) + weakHashFinder, err = weakhash.NewFinder(file, int(state.file.BlockSize()), hashesToFind) if err != nil { l.Debugln("weak hasher", err) } @@ -1268,7 +1268,7 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch return false } - _, err = fd.ReadAt(buf, protocol.BlockSize*int64(index)) + _, err = fd.ReadAt(buf, int64(state.file.BlockSize())*int64(index)) fd.Close() if err != nil { return false @@ -1391,7 +1391,7 @@ func (f *sendReceiveFolder) pullBlock(state pullBlockState, out chan<- *sharedPu } var lastError error - candidates := f.model.Availability(f.folderID, state.file.Name, state.file.Version, state.block) + candidates := f.model.Availability(f.folderID, state.file, state.block) for { // Select the least busy device to pull the block from. If we found no // feasible device at all, fail the block (and in the long run, the diff --git a/lib/model/rwfolder_test.go b/lib/model/rwfolder_test.go index 6415924b0..e555a5855 100644 --- a/lib/model/rwfolder_test.go +++ b/lib/model/rwfolder_test.go @@ -256,7 +256,7 @@ func TestCopierFinder(t *testing.T) { } // Verify that the fetched blocks have actually been written to the temp file - blks, err := scanner.HashFile(context.TODO(), fs.NewFilesystem(fs.FilesystemTypeBasic, "."), tempFile, protocol.BlockSize, nil, false) + blks, err := scanner.HashFile(context.TODO(), fs.NewFilesystem(fs.FilesystemTypeBasic, "."), tempFile, protocol.MinBlockSize, nil, false) if err != nil { t.Log(err) } @@ -273,8 +273,8 @@ func TestWeakHash(t *testing.T) { tempFile := filepath.Join("testdata", fs.TempName("weakhash")) var shift int64 = 10 var size int64 = 1 << 20 - expectBlocks := int(size / protocol.BlockSize) - expectPulls := int(shift / protocol.BlockSize) + expectBlocks := int(size / protocol.MinBlockSize) + expectPulls := int(shift / protocol.MinBlockSize) if shift > 0 { expectPulls++ } @@ -307,7 +307,7 @@ func TestWeakHash(t *testing.T) { // File 1: abcdefgh // File 2: xyabcdef f.Seek(0, os.SEEK_SET) - existing, err := scanner.Blocks(context.TODO(), f, protocol.BlockSize, size, nil, true) + existing, err := scanner.Blocks(context.TODO(), f, protocol.MinBlockSize, size, nil, true) if err != nil { t.Error(err) } @@ -316,7 +316,7 @@ func TestWeakHash(t *testing.T) { remainder := io.LimitReader(f, size-shift) prefix := io.LimitReader(rand.Reader, shift) nf := io.MultiReader(prefix, remainder) - desired, err := scanner.Blocks(context.TODO(), nf, protocol.BlockSize, size, nil, true) + desired, err := scanner.Blocks(context.TODO(), nf, protocol.MinBlockSize, size, nil, true) if err != nil { t.Error(err) } diff --git a/lib/model/sharedpullerstate.go b/lib/model/sharedpullerstate.go index 1c048536c..a09722f47 100644 --- a/lib/model/sharedpullerstate.go +++ b/lib/model/sharedpullerstate.go @@ -217,7 +217,7 @@ func (s *sharedPullerState) copyDone(block protocol.BlockInfo) { s.mut.Lock() s.copyNeeded-- s.updated = time.Now() - s.available = append(s.available, int32(block.Offset/protocol.BlockSize)) + s.available = append(s.available, int32(block.Offset/int64(s.file.BlockSize()))) s.availableUpdated = time.Now() l.Debugln("sharedPullerState", s.folder, s.file.Name, "copyNeeded ->", s.copyNeeded) s.mut.Unlock() @@ -253,7 +253,7 @@ func (s *sharedPullerState) pullDone(block protocol.BlockInfo) { s.mut.Lock() s.pullNeeded-- s.updated = time.Now() - s.available = append(s.available, int32(block.Offset/protocol.BlockSize)) + s.available = append(s.available, int32(block.Offset/int64(s.file.BlockSize()))) s.availableUpdated = time.Now() l.Debugln("sharedPullerState", s.folder, s.file.Name, "pullNeeded done ->", s.pullNeeded) s.mut.Unlock() @@ -314,8 +314,8 @@ func (s *sharedPullerState) Progress() *pullerProgress { CopiedFromElsewhere: s.copyTotal - s.copyNeeded - s.copyOrigin, Pulled: s.pullTotal - s.pullNeeded, Pulling: s.pullNeeded, - BytesTotal: blocksToSize(total), - BytesDone: blocksToSize(done), + BytesTotal: blocksToSize(s.file.BlockSize(), total), + BytesDone: blocksToSize(s.file.BlockSize(), done), } } @@ -343,9 +343,9 @@ func (s *sharedPullerState) Available() []int32 { return blocks } -func blocksToSize(num int) int64 { +func blocksToSize(size int, num int) int64 { if num < 2 { - return protocol.BlockSize / 2 + return int64(size / 2) } - return int64(num-1)*protocol.BlockSize + protocol.BlockSize/2 + return int64(num-1)*int64(size) + int64(size/2) } diff --git a/lib/protocol/bep.pb.go b/lib/protocol/bep.pb.go index cb018572e..a98cef07e 100644 --- a/lib/protocol/bep.pb.go +++ b/lib/protocol/bep.pb.go @@ -306,6 +306,7 @@ type FileInfo struct { NoPermissions bool `protobuf:"varint,8,opt,name=no_permissions,json=noPermissions,proto3" json:"no_permissions,omitempty"` Version Vector `protobuf:"bytes,9,opt,name=version" json:"version"` Sequence int64 `protobuf:"varint,10,opt,name=sequence,proto3" json:"sequence,omitempty"` + RawBlockSize int32 `protobuf:"varint,13,opt,name=block_size,json=blockSize,proto3" json:"block_size,omitempty"` Blocks []BlockInfo `protobuf:"bytes,16,rep,name=Blocks" json:"Blocks"` SymlinkTarget string `protobuf:"bytes,17,opt,name=symlink_target,json=symlinkTarget,proto3" json:"symlink_target,omitempty"` } @@ -876,6 +877,11 @@ func (m *FileInfo) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintBep(dAtA, i, uint64(m.ModifiedBy)) } + if m.RawBlockSize != 0 { + dAtA[i] = 0x68 + i++ + i = encodeVarintBep(dAtA, i, uint64(m.RawBlockSize)) + } if len(m.Blocks) > 0 { for _, msg := range m.Blocks { dAtA[i] = 0x82 @@ -1427,6 +1433,9 @@ func (m *FileInfo) ProtoSize() (n int) { if m.ModifiedBy != 0 { n += 1 + sovBep(uint64(m.ModifiedBy)) } + if m.RawBlockSize != 0 { + n += 1 + sovBep(uint64(m.RawBlockSize)) + } if len(m.Blocks) > 0 { for _, e := range m.Blocks { l = e.ProtoSize() @@ -2904,6 +2913,25 @@ func (m *FileInfo) Unmarshal(dAtA []byte) error { break } } + case 13: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field RawBlockSize", wireType) + } + m.RawBlockSize = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBep + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.RawBlockSize |= (int32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } case 16: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Blocks", wireType) @@ -4164,114 +4192,115 @@ var ( func init() { proto.RegisterFile("bep.proto", fileDescriptorBep) } var fileDescriptorBep = []byte{ - // 1729 bytes of a gzipped FileDescriptorProto + // 1757 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4f, 0x8f, 0xdb, 0xc6, - 0x15, 0x17, 0x25, 0xea, 0xdf, 0x93, 0x76, 0xc3, 0x1d, 0xdb, 0x5b, 0x96, 0xd9, 0x68, 0x69, 0xc5, - 0x8e, 0x37, 0x8b, 0x64, 0xe3, 0x26, 0x69, 0x8b, 0x16, 0x6d, 0x01, 0xfd, 0xe1, 0xae, 0x85, 0xca, - 0x94, 0x3a, 0xd2, 0x3a, 0x75, 0x0e, 0x25, 0x28, 0x71, 0xa4, 0x25, 0x4c, 0x71, 0x54, 0x92, 0x5a, - 0x5b, 0xfd, 0x08, 0xfa, 0x04, 0xbd, 0x08, 0x08, 0xd0, 0x53, 0xef, 0xfd, 0x10, 0x3e, 0x06, 0x3d, - 0xf4, 0xd0, 0x83, 0xd1, 0x6c, 0x2f, 0x3d, 0xf6, 0x5e, 0xa0, 0x28, 0x38, 0x43, 0x52, 0xd4, 0xae, - 0x37, 0xf0, 0xa1, 0x27, 0xce, 0xbc, 0xf7, 0x9b, 0x3f, 0xef, 0xf7, 0xde, 0xef, 0x0d, 0xa1, 0x3c, - 0x22, 0xf3, 0x93, 0xb9, 0x47, 0x03, 0x8a, 0x4a, 0xec, 0x33, 0xa6, 0x8e, 0xf2, 0xe9, 0xd4, 0x0e, - 0x2e, 0x16, 0xa3, 0x93, 0x31, 0x9d, 0x7d, 0x36, 0xa5, 0x53, 0xfa, 0x19, 0xf3, 0x8c, 0x16, 0x13, - 0x36, 0x63, 0x13, 0x36, 0xe2, 0x0b, 0xeb, 0x73, 0xc8, 0x3f, 0x21, 0x8e, 0x43, 0xd1, 0x21, 0x54, - 0x2c, 0x72, 0x69, 0x8f, 0x89, 0xe1, 0x9a, 0x33, 0x22, 0x0b, 0xaa, 0x70, 0x54, 0xc6, 0xc0, 0x4d, - 0xba, 0x39, 0x23, 0x21, 0x60, 0xec, 0xd8, 0xc4, 0x0d, 0x38, 0x20, 0xcb, 0x01, 0xdc, 0xc4, 0x00, - 0x0f, 0x61, 0x37, 0x02, 0x5c, 0x12, 0xcf, 0xb7, 0xa9, 0x2b, 0xe7, 0x18, 0x66, 0x87, 0x5b, 0x9f, - 0x71, 0x63, 0xdd, 0x87, 0xc2, 0x13, 0x62, 0x5a, 0xc4, 0x43, 0x1f, 0x83, 0x18, 0x2c, 0xe7, 0xfc, - 0xac, 0xdd, 0xcf, 0xef, 0x9d, 0xc4, 0x31, 0x9c, 0x3c, 0x25, 0xbe, 0x6f, 0x4e, 0xc9, 0x70, 0x39, - 0x27, 0x98, 0x41, 0xd0, 0xaf, 0xa0, 0x32, 0xa6, 0xb3, 0xb9, 0x47, 0x7c, 0xb6, 0x71, 0x96, 0xad, - 0x38, 0xb8, 0xb1, 0xa2, 0xb5, 0xc1, 0xe0, 0xf4, 0x82, 0x7a, 0x03, 0x76, 0x5a, 0xce, 0xc2, 0x0f, - 0x88, 0xd7, 0xa2, 0xee, 0xc4, 0x9e, 0xa2, 0xc7, 0x50, 0x9c, 0x50, 0xc7, 0x22, 0x9e, 0x2f, 0x0b, - 0x6a, 0xee, 0xa8, 0xf2, 0xb9, 0xb4, 0xd9, 0xec, 0x94, 0x39, 0x9a, 0xe2, 0xeb, 0x37, 0x87, 0x19, - 0x1c, 0xc3, 0xea, 0x7f, 0xca, 0x42, 0x81, 0x7b, 0xd0, 0x3e, 0x64, 0x6d, 0x8b, 0x53, 0xd4, 0x2c, - 0x5c, 0xbd, 0x39, 0xcc, 0x76, 0xda, 0x38, 0x6b, 0x5b, 0xe8, 0x2e, 0xe4, 0x1d, 0x73, 0x44, 0x9c, - 0x88, 0x1c, 0x3e, 0x41, 0xef, 0x43, 0xd9, 0x23, 0xa6, 0x65, 0x50, 0xd7, 0x59, 0x32, 0x4a, 0x4a, - 0xb8, 0x14, 0x1a, 0x7a, 0xae, 0xb3, 0x44, 0x9f, 0x02, 0xb2, 0xa7, 0x2e, 0xf5, 0x88, 0x31, 0x27, - 0xde, 0xcc, 0x66, 0xb7, 0xf5, 0x65, 0x91, 0xa1, 0xf6, 0xb8, 0xa7, 0xbf, 0x71, 0xa0, 0x0f, 0x61, - 0x27, 0x82, 0x5b, 0xc4, 0x21, 0x01, 0x91, 0xf3, 0x0c, 0x59, 0xe5, 0xc6, 0x36, 0xb3, 0xa1, 0xc7, - 0x70, 0xd7, 0xb2, 0x7d, 0x73, 0xe4, 0x10, 0x23, 0x20, 0xb3, 0xb9, 0x61, 0xbb, 0x16, 0x79, 0x45, - 0x7c, 0xb9, 0xc0, 0xb0, 0x28, 0xf2, 0x0d, 0xc9, 0x6c, 0xde, 0xe1, 0x1e, 0xb4, 0x0f, 0x85, 0xb9, - 0xb9, 0xf0, 0x89, 0x25, 0x17, 0x19, 0x26, 0x9a, 0x85, 0x2c, 0xf1, 0x0a, 0xf0, 0x65, 0xe9, 0x3a, - 0x4b, 0x6d, 0xe6, 0x88, 0x59, 0x8a, 0x60, 0xf5, 0x7f, 0x67, 0xa1, 0xc0, 0x3d, 0xe8, 0xa3, 0x84, - 0xa5, 0x6a, 0x73, 0x3f, 0x44, 0xfd, 0xfd, 0xcd, 0x61, 0x89, 0xfb, 0x3a, 0xed, 0x14, 0x6b, 0x08, - 0xc4, 0x54, 0x45, 0xb1, 0x31, 0x3a, 0x80, 0xb2, 0x69, 0x59, 0x61, 0xf6, 0x88, 0x2f, 0xe7, 0xd4, - 0xdc, 0x51, 0x19, 0x6f, 0x0c, 0xe8, 0xa7, 0xdb, 0xd5, 0x20, 0x5e, 0xaf, 0x9f, 0xdb, 0xca, 0x20, - 0x4c, 0xc5, 0x98, 0x78, 0x51, 0x05, 0xe7, 0xd9, 0x79, 0xa5, 0xd0, 0xc0, 0xea, 0xf7, 0x3e, 0x54, - 0x67, 0xe6, 0x2b, 0xc3, 0x27, 0xbf, 0x5f, 0x10, 0x77, 0x4c, 0x18, 0x5d, 0x39, 0x5c, 0x99, 0x99, - 0xaf, 0x06, 0x91, 0x09, 0xd5, 0x00, 0x6c, 0x37, 0xf0, 0xa8, 0xb5, 0x18, 0x13, 0x2f, 0xe2, 0x2a, - 0x65, 0x41, 0x3f, 0x86, 0x12, 0x23, 0xdb, 0xb0, 0x2d, 0xb9, 0xa4, 0x0a, 0x47, 0x62, 0x53, 0x89, - 0x02, 0x2f, 0x32, 0xaa, 0x59, 0xdc, 0xf1, 0x10, 0x17, 0x19, 0xb6, 0x63, 0xa1, 0x5f, 0x80, 0xe2, - 0xbf, 0xb0, 0xc3, 0x44, 0xf1, 0x9d, 0x02, 0x9b, 0xba, 0x86, 0x47, 0x66, 0xf4, 0xd2, 0x74, 0x7c, - 0xb9, 0xcc, 0x8e, 0x91, 0x43, 0x44, 0x27, 0x05, 0xc0, 0x91, 0xbf, 0xde, 0x83, 0x3c, 0xdb, 0x31, - 0xcc, 0x22, 0x2f, 0xd6, 0x48, 0xbd, 0xd1, 0x0c, 0x9d, 0x40, 0x7e, 0x62, 0x3b, 0xc4, 0x97, 0xb3, - 0x2c, 0x87, 0x28, 0x55, 0xe9, 0xb6, 0x43, 0x3a, 0xee, 0x84, 0x46, 0x59, 0xe4, 0xb0, 0xfa, 0x39, - 0x54, 0xd8, 0x86, 0xe7, 0x73, 0xcb, 0x0c, 0xc8, 0xff, 0x6d, 0xdb, 0xff, 0xe4, 0xa0, 0x14, 0x7b, - 0x92, 0xa4, 0x0b, 0xa9, 0xa4, 0x1f, 0x47, 0xfd, 0x80, 0xab, 0x7b, 0xff, 0xe6, 0x7e, 0xa9, 0x86, - 0x80, 0x40, 0xf4, 0xed, 0x3f, 0x10, 0xa6, 0xa7, 0x1c, 0x66, 0x63, 0xa4, 0x42, 0xe5, 0xba, 0x88, - 0x76, 0x70, 0xda, 0x84, 0x3e, 0x00, 0x98, 0x51, 0xcb, 0x9e, 0xd8, 0xc4, 0x32, 0x7c, 0x56, 0x00, - 0x39, 0x5c, 0x8e, 0x2d, 0x03, 0x24, 0x87, 0xe5, 0x1e, 0x4a, 0xc8, 0x8a, 0xb4, 0x12, 0x4f, 0x43, - 0x8f, 0xed, 0x5e, 0x9a, 0x8e, 0x1d, 0x2b, 0x24, 0x9e, 0x86, 0x5d, 0xcf, 0xa5, 0x5b, 0xe2, 0x2d, - 0x31, 0xc0, 0x8e, 0x4b, 0xd3, 0xc2, 0x7d, 0x0c, 0xc5, 0xb8, 0x2b, 0x86, 0xf9, 0xdc, 0x52, 0xd2, - 0x33, 0x32, 0x0e, 0x68, 0xd2, 0x6f, 0x22, 0x18, 0x52, 0xa0, 0x94, 0x94, 0x22, 0xb0, 0x9b, 0x26, - 0xf3, 0xb0, 0x17, 0x27, 0x71, 0xb8, 0xbe, 0x5c, 0x51, 0x85, 0xa3, 0x3c, 0x4e, 0x42, 0xd3, 0xc3, - 0xe3, 0x36, 0x80, 0xd1, 0x52, 0xae, 0xb2, 0x5a, 0x7c, 0x2f, 0xae, 0xc5, 0xc1, 0x05, 0xf5, 0x82, - 0x4e, 0x7b, 0xb3, 0xa2, 0xb9, 0x44, 0x3f, 0x82, 0x42, 0xd3, 0xa1, 0xe3, 0x17, 0xb1, 0xd2, 0xef, - 0x6c, 0xee, 0xc7, 0xec, 0xa9, 0x7c, 0x46, 0xc0, 0x30, 0x74, 0x7f, 0x39, 0x73, 0x6c, 0xf7, 0x85, - 0x11, 0x98, 0xde, 0x94, 0x04, 0xf2, 0x1e, 0x6f, 0xf8, 0x91, 0x75, 0xc8, 0x8c, 0x3f, 0x17, 0xff, - 0xf8, 0xcd, 0x61, 0xa6, 0xee, 0x42, 0x39, 0xd9, 0x27, 0x2c, 0x29, 0x3a, 0x99, 0xf8, 0x24, 0x60, - 0xf9, 0xcf, 0xe1, 0x68, 0x96, 0x64, 0x35, 0xcb, 0x02, 0xe2, 0x59, 0x45, 0x20, 0x5e, 0x98, 0xfe, - 0x05, 0xcb, 0x74, 0x15, 0xb3, 0x71, 0xa8, 0xe3, 0x97, 0xc4, 0x7c, 0x61, 0x30, 0x07, 0xcf, 0x73, - 0x29, 0x34, 0x3c, 0x31, 0xfd, 0x8b, 0xe8, 0xbc, 0x5f, 0x42, 0x81, 0xf3, 0x8a, 0xbe, 0x80, 0xd2, - 0x98, 0x2e, 0xdc, 0x60, 0xd3, 0xeb, 0xf7, 0xd2, 0xad, 0x82, 0x79, 0xa2, 0xc8, 0x12, 0x60, 0xfd, - 0x14, 0x8a, 0x91, 0x0b, 0x3d, 0x4c, 0xfa, 0x98, 0xd8, 0xbc, 0x77, 0x8d, 0xc2, 0xed, 0xe6, 0x7f, - 0x69, 0x3a, 0x0b, 0x7e, 0x79, 0x11, 0xf3, 0x49, 0xfd, 0x2f, 0x02, 0x14, 0x71, 0x98, 0x36, 0x3f, - 0x48, 0x3d, 0x1b, 0xf9, 0xad, 0x67, 0x63, 0x23, 0xb0, 0xec, 0x96, 0xc0, 0x62, 0x8d, 0xe4, 0x52, - 0x1a, 0xd9, 0x30, 0x27, 0xbe, 0x95, 0xb9, 0xfc, 0x5b, 0x98, 0x2b, 0xa4, 0x98, 0x7b, 0x08, 0xbb, - 0x13, 0x8f, 0xce, 0xd8, 0xc3, 0x40, 0x3d, 0xd3, 0x5b, 0x46, 0xf5, 0xbc, 0x13, 0x5a, 0x87, 0xb1, - 0xb1, 0x6e, 0x40, 0x09, 0x13, 0x7f, 0x4e, 0x5d, 0x9f, 0xdc, 0x7a, 0x6d, 0x04, 0xa2, 0x65, 0x06, - 0x26, 0xbb, 0x74, 0x15, 0xb3, 0x31, 0x7a, 0x04, 0xe2, 0x98, 0x5a, 0xfc, 0xca, 0xbb, 0xe9, 0x1a, - 0xd2, 0x3c, 0x8f, 0x7a, 0x2d, 0x6a, 0x11, 0xcc, 0x00, 0xf5, 0x39, 0x48, 0x6d, 0xfa, 0xd2, 0x75, - 0xa8, 0x69, 0xf5, 0x3d, 0x3a, 0x0d, 0x1b, 0xf4, 0xad, 0x8d, 0xa6, 0x0d, 0xc5, 0x05, 0x6b, 0x45, - 0x71, 0xab, 0x79, 0xb0, 0xdd, 0x1a, 0xae, 0x6f, 0xc4, 0xfb, 0x56, 0xac, 0xa7, 0x68, 0x69, 0xfd, - 0x6f, 0x02, 0x28, 0xb7, 0xa3, 0x51, 0x07, 0x2a, 0x1c, 0x69, 0xa4, 0xfe, 0x49, 0x8e, 0xde, 0xe5, - 0x20, 0xd6, 0x95, 0x60, 0x91, 0x8c, 0xdf, 0xfa, 0xa0, 0xa5, 0xf4, 0x9f, 0x7b, 0x37, 0xfd, 0x3f, - 0x82, 0x9d, 0x51, 0x28, 0x98, 0xe4, 0xf9, 0x16, 0xd5, 0xdc, 0x51, 0xbe, 0x99, 0x95, 0x32, 0xb8, - 0x3a, 0xe2, 0x4a, 0x62, 0xf6, 0x7a, 0x01, 0xc4, 0xbe, 0xed, 0x4e, 0xeb, 0x87, 0x90, 0x6f, 0x39, - 0x94, 0x25, 0xac, 0xe0, 0x11, 0xd3, 0xa7, 0x6e, 0xcc, 0x23, 0x9f, 0x1d, 0xff, 0x35, 0x0b, 0x95, - 0xd4, 0xaf, 0x15, 0x7a, 0x0c, 0xbb, 0xad, 0xee, 0xf9, 0x60, 0xa8, 0x61, 0xa3, 0xd5, 0xd3, 0x4f, - 0x3b, 0x67, 0x52, 0x46, 0x39, 0x58, 0xad, 0x55, 0x79, 0xb6, 0x01, 0x6d, 0xff, 0x35, 0x1d, 0x42, - 0xbe, 0xa3, 0xb7, 0xb5, 0xdf, 0x4a, 0x82, 0x72, 0x77, 0xb5, 0x56, 0xa5, 0x14, 0x90, 0x3f, 0x41, - 0x9f, 0x40, 0x95, 0x01, 0x8c, 0xf3, 0x7e, 0xbb, 0x31, 0xd4, 0xa4, 0xac, 0xa2, 0xac, 0xd6, 0xea, - 0xfe, 0x75, 0x5c, 0xc4, 0xf9, 0x87, 0x50, 0xc4, 0xda, 0x6f, 0xce, 0xb5, 0xc1, 0x50, 0xca, 0x29, - 0xfb, 0xab, 0xb5, 0x8a, 0x52, 0xc0, 0x58, 0x35, 0x0f, 0xa1, 0x84, 0xb5, 0x41, 0xbf, 0xa7, 0x0f, - 0x34, 0x49, 0x54, 0x7e, 0xb0, 0x5a, 0xab, 0x77, 0xb6, 0x50, 0x51, 0x95, 0xfe, 0x04, 0xf6, 0xda, - 0xbd, 0xaf, 0xf4, 0x6e, 0xaf, 0xd1, 0x36, 0xfa, 0xb8, 0x77, 0x86, 0xb5, 0xc1, 0x40, 0xca, 0x2b, - 0x87, 0xab, 0xb5, 0xfa, 0x7e, 0x0a, 0x7f, 0xa3, 0xe8, 0x3e, 0x00, 0xb1, 0xdf, 0xd1, 0xcf, 0xa4, - 0x82, 0x72, 0x67, 0xb5, 0x56, 0xdf, 0x4b, 0x41, 0x43, 0x52, 0xc3, 0x88, 0x5b, 0xdd, 0xde, 0x40, - 0x93, 0x8a, 0x37, 0x22, 0x66, 0x64, 0x1f, 0xff, 0x0e, 0xd0, 0xcd, 0x9f, 0x4f, 0xf4, 0x00, 0x44, - 0xbd, 0xa7, 0x6b, 0x52, 0x86, 0xc7, 0x7f, 0x13, 0xa1, 0x53, 0x97, 0xa0, 0x3a, 0xe4, 0xba, 0x5f, - 0x7f, 0x29, 0x09, 0xca, 0x0f, 0x57, 0x6b, 0xf5, 0xde, 0x4d, 0x50, 0xf7, 0xeb, 0x2f, 0x8f, 0x29, - 0x54, 0xd2, 0x1b, 0xd7, 0xa1, 0xf4, 0x54, 0x1b, 0x36, 0xda, 0x8d, 0x61, 0x43, 0xca, 0xf0, 0x2b, - 0xc5, 0xee, 0xa7, 0x24, 0x30, 0x99, 0x08, 0x0f, 0x20, 0xaf, 0x6b, 0xcf, 0x34, 0x2c, 0x09, 0xca, - 0xde, 0x6a, 0xad, 0xee, 0xc4, 0x00, 0x9d, 0x5c, 0x12, 0x0f, 0xd5, 0xa0, 0xd0, 0xe8, 0x7e, 0xd5, - 0x78, 0x3e, 0x90, 0xb2, 0x0a, 0x5a, 0xad, 0xd5, 0xdd, 0xd8, 0xdd, 0x70, 0x5e, 0x9a, 0x4b, 0xff, - 0xf8, 0xbf, 0x02, 0x54, 0xd3, 0x0f, 0x2e, 0xaa, 0x81, 0x78, 0xda, 0xe9, 0x6a, 0xf1, 0x71, 0x69, - 0x5f, 0x38, 0x46, 0x47, 0x50, 0x6e, 0x77, 0xb0, 0xd6, 0x1a, 0xf6, 0xf0, 0xf3, 0x38, 0x96, 0x34, - 0xa8, 0x6d, 0x7b, 0xac, 0xc0, 0x97, 0xe8, 0x67, 0x50, 0x1d, 0x3c, 0x7f, 0xda, 0xed, 0xe8, 0xbf, - 0x36, 0xd8, 0x8e, 0x59, 0xe5, 0xd1, 0x6a, 0xad, 0xde, 0xdf, 0x02, 0x93, 0xb9, 0x47, 0xc6, 0x66, - 0x40, 0xac, 0x01, 0x7f, 0x44, 0x42, 0x67, 0x49, 0x40, 0x2d, 0xd8, 0x8b, 0x97, 0x6e, 0x0e, 0xcb, - 0x29, 0x9f, 0xac, 0xd6, 0xea, 0x47, 0xdf, 0xbb, 0x3e, 0x39, 0xbd, 0x24, 0xa0, 0x07, 0x50, 0x8c, - 0x36, 0x89, 0x2b, 0x29, 0xbd, 0x34, 0x5a, 0x70, 0xfc, 0x67, 0x01, 0xca, 0x49, 0xbb, 0x0a, 0x09, - 0xd7, 0x7b, 0x86, 0x86, 0x71, 0x0f, 0xc7, 0x0c, 0x24, 0x4e, 0x9d, 0xb2, 0x21, 0xba, 0x0f, 0xc5, - 0x33, 0x4d, 0xd7, 0x70, 0xa7, 0x15, 0x0b, 0x23, 0x81, 0x9c, 0x11, 0x97, 0x78, 0xf6, 0x18, 0x7d, - 0x0c, 0x55, 0xbd, 0x67, 0x0c, 0xce, 0x5b, 0x4f, 0xe2, 0xd0, 0xd9, 0xf9, 0xa9, 0xad, 0x06, 0x8b, - 0xf1, 0x05, 0xe3, 0xf3, 0x38, 0xd4, 0xd0, 0xb3, 0x46, 0xb7, 0xd3, 0xe6, 0xd0, 0x9c, 0x22, 0xaf, - 0xd6, 0xea, 0xdd, 0x04, 0xda, 0xe1, 0x7f, 0x1e, 0x21, 0xf6, 0xd8, 0x82, 0xda, 0xf7, 0x37, 0x26, - 0xa4, 0x42, 0xa1, 0xd1, 0xef, 0x6b, 0x7a, 0x3b, 0xbe, 0xfd, 0xc6, 0xd7, 0x98, 0xcf, 0x89, 0x6b, - 0x85, 0x88, 0xd3, 0x1e, 0x3e, 0xd3, 0x86, 0xf1, 0xe5, 0x37, 0x88, 0x53, 0x1a, 0xbe, 0xe0, 0xcd, - 0x83, 0xd7, 0xdf, 0xd5, 0x32, 0xdf, 0x7e, 0x57, 0xcb, 0xbc, 0xbe, 0xaa, 0x09, 0xdf, 0x5e, 0xd5, - 0x84, 0x7f, 0x5c, 0xd5, 0x32, 0xff, 0xba, 0xaa, 0x09, 0xdf, 0xfc, 0xb3, 0x26, 0x8c, 0x0a, 0xac, - 0x91, 0x7d, 0xf1, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xad, 0x8a, 0xef, 0x7f, 0x8f, 0x0e, 0x00, - 0x00, + 0x15, 0x17, 0x25, 0xea, 0xdf, 0x93, 0xb4, 0xe1, 0x8e, 0xed, 0x2d, 0xcb, 0x6c, 0x24, 0x5a, 0xb1, + 0xe3, 0xcd, 0x22, 0x59, 0xbb, 0x49, 0xda, 0xa2, 0x45, 0x5b, 0x40, 0x7f, 0xb8, 0x6b, 0xa1, 0x32, + 0xa5, 0x8e, 0xb4, 0x4e, 0x9d, 0x43, 0x09, 0x4a, 0x1c, 0x69, 0x09, 0x53, 0x1c, 0x95, 0xa4, 0xd6, + 0x56, 0x3e, 0x82, 0x3e, 0x41, 0x2f, 0x02, 0x02, 0xf4, 0x54, 0xa0, 0xc7, 0x7e, 0x08, 0x1f, 0x83, + 0x1e, 0x7a, 0xe8, 0xc1, 0x68, 0xb6, 0x97, 0x1e, 0xfb, 0x09, 0x8a, 0x82, 0x33, 0xa4, 0x44, 0xed, + 0xda, 0x81, 0x0f, 0x39, 0x71, 0xe6, 0xbd, 0xdf, 0xbc, 0x99, 0xf9, 0xcd, 0xef, 0xbd, 0x47, 0x28, + 0x8e, 0xc8, 0xfc, 0x64, 0xee, 0xd1, 0x80, 0xa2, 0x02, 0xfb, 0x8c, 0xa9, 0xa3, 0x7c, 0x3a, 0xb5, + 0x83, 0x8b, 0xc5, 0xe8, 0x64, 0x4c, 0x67, 0x0f, 0xa7, 0x74, 0x4a, 0x1f, 0x32, 0xcf, 0x68, 0x31, + 0x61, 0x33, 0x36, 0x61, 0x23, 0xbe, 0xb0, 0x3e, 0x87, 0xec, 0x63, 0xe2, 0x38, 0x14, 0xd5, 0xa0, + 0x64, 0x91, 0x4b, 0x7b, 0x4c, 0x0c, 0xd7, 0x9c, 0x11, 0x59, 0x50, 0x85, 0xa3, 0x22, 0x06, 0x6e, + 0xd2, 0xcd, 0x19, 0x09, 0x01, 0x63, 0xc7, 0x26, 0x6e, 0xc0, 0x01, 0x69, 0x0e, 0xe0, 0x26, 0x06, + 0xb8, 0x0f, 0x7b, 0x11, 0xe0, 0x92, 0x78, 0xbe, 0x4d, 0x5d, 0x39, 0xc3, 0x30, 0x15, 0x6e, 0x7d, + 0xca, 0x8d, 0x75, 0x1f, 0x72, 0x8f, 0x89, 0x69, 0x11, 0x0f, 0x7d, 0x0c, 0x62, 0xb0, 0x9c, 0xf3, + 0xbd, 0xf6, 0x3e, 0xbb, 0x73, 0x12, 0xdf, 0xe1, 0xe4, 0x09, 0xf1, 0x7d, 0x73, 0x4a, 0x86, 0xcb, + 0x39, 0xc1, 0x0c, 0x82, 0x7e, 0x03, 0xa5, 0x31, 0x9d, 0xcd, 0x3d, 0xe2, 0xb3, 0xc0, 0x69, 0xb6, + 0xe2, 0xf0, 0xc6, 0x8a, 0xd6, 0x16, 0x83, 0x93, 0x0b, 0xea, 0x0d, 0xa8, 0xb4, 0x9c, 0x85, 0x1f, + 0x10, 0xaf, 0x45, 0xdd, 0x89, 0x3d, 0x45, 0x8f, 0x20, 0x3f, 0xa1, 0x8e, 0x45, 0x3c, 0x5f, 0x16, + 0xd4, 0xcc, 0x51, 0xe9, 0x33, 0x69, 0x1b, 0xec, 0x94, 0x39, 0x9a, 0xe2, 0xab, 0xd7, 0xb5, 0x14, + 0x8e, 0x61, 0xf5, 0x3f, 0xa7, 0x21, 0xc7, 0x3d, 0xe8, 0x00, 0xd2, 0xb6, 0xc5, 0x29, 0x6a, 0xe6, + 0xae, 0x5e, 0xd7, 0xd2, 0x9d, 0x36, 0x4e, 0xdb, 0x16, 0xba, 0x0d, 0x59, 0xc7, 0x1c, 0x11, 0x27, + 0x22, 0x87, 0x4f, 0xd0, 0xfb, 0x50, 0xf4, 0x88, 0x69, 0x19, 0xd4, 0x75, 0x96, 0x8c, 0x92, 0x02, + 0x2e, 0x84, 0x86, 0x9e, 0xeb, 0x2c, 0xd1, 0xa7, 0x80, 0xec, 0xa9, 0x4b, 0x3d, 0x62, 0xcc, 0x89, + 0x37, 0xb3, 0xd9, 0x69, 0x7d, 0x59, 0x64, 0xa8, 0x7d, 0xee, 0xe9, 0x6f, 0x1d, 0xe8, 0x43, 0xa8, + 0x44, 0x70, 0x8b, 0x38, 0x24, 0x20, 0x72, 0x96, 0x21, 0xcb, 0xdc, 0xd8, 0x66, 0x36, 0xf4, 0x08, + 0x6e, 0x5b, 0xb6, 0x6f, 0x8e, 0x1c, 0x62, 0x04, 0x64, 0x36, 0x37, 0x6c, 0xd7, 0x22, 0x2f, 0x89, + 0x2f, 0xe7, 0x18, 0x16, 0x45, 0xbe, 0x21, 0x99, 0xcd, 0x3b, 0xdc, 0x83, 0x0e, 0x20, 0x37, 0x37, + 0x17, 0x3e, 0xb1, 0xe4, 0x3c, 0xc3, 0x44, 0xb3, 0x90, 0x25, 0xae, 0x00, 0x5f, 0x96, 0xae, 0xb3, + 0xd4, 0x66, 0x8e, 0x98, 0xa5, 0x08, 0x56, 0xff, 0x6f, 0x1a, 0x72, 0xdc, 0x83, 0x3e, 0xda, 0xb0, + 0x54, 0x6e, 0x1e, 0x84, 0xa8, 0x7f, 0xbe, 0xae, 0x15, 0xb8, 0xaf, 0xd3, 0x4e, 0xb0, 0x86, 0x40, + 0x4c, 0x28, 0x8a, 0x8d, 0xd1, 0x21, 0x14, 0x4d, 0xcb, 0x0a, 0x5f, 0x8f, 0xf8, 0x72, 0x46, 0xcd, + 0x1c, 0x15, 0xf1, 0xd6, 0x80, 0x7e, 0xbe, 0xab, 0x06, 0xf1, 0xba, 0x7e, 0xde, 0x26, 0x83, 0xf0, + 0x29, 0xc6, 0xc4, 0x8b, 0x14, 0x9c, 0x65, 0xfb, 0x15, 0x42, 0x03, 0xd3, 0xef, 0x5d, 0x28, 0xcf, + 0xcc, 0x97, 0x86, 0x4f, 0xfe, 0xb8, 0x20, 0xee, 0x98, 0x30, 0xba, 0x32, 0xb8, 0x34, 0x33, 0x5f, + 0x0e, 0x22, 0x13, 0xaa, 0x02, 0xd8, 0x6e, 0xe0, 0x51, 0x6b, 0x31, 0x26, 0x5e, 0xc4, 0x55, 0xc2, + 0x82, 0x7e, 0x0a, 0x05, 0x46, 0xb6, 0x61, 0x5b, 0x72, 0x41, 0x15, 0x8e, 0xc4, 0xa6, 0x12, 0x5d, + 0x3c, 0xcf, 0xa8, 0x66, 0xf7, 0x8e, 0x87, 0x38, 0xcf, 0xb0, 0x1d, 0x0b, 0xfd, 0x0a, 0x14, 0xff, + 0xb9, 0x1d, 0x3e, 0x14, 0x8f, 0x14, 0xd8, 0xd4, 0x35, 0x3c, 0x32, 0xa3, 0x97, 0xa6, 0xe3, 0xcb, + 0x45, 0xb6, 0x8d, 0x1c, 0x22, 0x3a, 0x09, 0x00, 0x8e, 0xfc, 0xf5, 0x1e, 0x64, 0x59, 0xc4, 0xf0, + 0x15, 0xb9, 0x58, 0xa3, 0xec, 0x8d, 0x66, 0xe8, 0x04, 0xb2, 0x13, 0xdb, 0x21, 0xbe, 0x9c, 0x66, + 0x6f, 0x88, 0x12, 0x4a, 0xb7, 0x1d, 0xd2, 0x71, 0x27, 0x34, 0x7a, 0x45, 0x0e, 0xab, 0x9f, 0x43, + 0x89, 0x05, 0x3c, 0x9f, 0x5b, 0x66, 0x40, 0x7e, 0xb0, 0xb0, 0x7f, 0x15, 0xa1, 0x10, 0x7b, 0x36, + 0x8f, 0x2e, 0x24, 0x1e, 0xfd, 0x38, 0xaa, 0x07, 0x3c, 0xbb, 0x0f, 0x6e, 0xc6, 0x4b, 0x14, 0x04, + 0x04, 0xa2, 0x6f, 0x7f, 0x4d, 0x58, 0x3e, 0x65, 0x30, 0x1b, 0x23, 0x15, 0x4a, 0xd7, 0x93, 0xa8, + 0x82, 0x93, 0x26, 0xf4, 0x01, 0xc0, 0x8c, 0x5a, 0xf6, 0xc4, 0x26, 0x96, 0xe1, 0x33, 0x01, 0x64, + 0x70, 0x31, 0xb6, 0x0c, 0x90, 0x1c, 0xca, 0x3d, 0x4c, 0x21, 0x2b, 0xca, 0x95, 0x78, 0x1a, 0x7a, + 0x6c, 0xf7, 0xd2, 0x74, 0xec, 0x38, 0x43, 0xe2, 0x69, 0x58, 0xf5, 0x5c, 0xba, 0x93, 0xbc, 0x05, + 0x06, 0xa8, 0xb8, 0x34, 0x99, 0xb8, 0x8f, 0x20, 0x1f, 0x57, 0xc5, 0xf0, 0x3d, 0x77, 0x32, 0xe9, + 0x29, 0x19, 0x07, 0x74, 0x53, 0x6f, 0x22, 0x18, 0x52, 0xa0, 0xb0, 0x91, 0x22, 0xb0, 0x93, 0x6e, + 0xe6, 0x61, 0x2d, 0xde, 0xdc, 0xc3, 0xf5, 0xe5, 0x92, 0x2a, 0x1c, 0x65, 0xf1, 0xe6, 0x6a, 0x7a, + 0xb8, 0xdd, 0x16, 0x30, 0x5a, 0xca, 0x65, 0xa6, 0xc5, 0xf7, 0x62, 0x2d, 0x0e, 0x2e, 0xa8, 0x17, + 0x74, 0xda, 0xdb, 0x15, 0xcd, 0x25, 0x7a, 0x08, 0x30, 0x72, 0xe8, 0xf8, 0xb9, 0xc1, 0x68, 0xad, + 0x84, 0x11, 0x9b, 0xd2, 0xd5, 0xeb, 0x5a, 0x19, 0x9b, 0x2f, 0x9a, 0xa1, 0x63, 0x60, 0x7f, 0x4d, + 0x70, 0x71, 0x14, 0x0f, 0xd1, 0x4f, 0x20, 0xc7, 0xec, 0x71, 0x69, 0xb8, 0xb5, 0xbd, 0x10, 0xb3, + 0x27, 0x04, 0x10, 0x01, 0x43, 0xae, 0xfc, 0xe5, 0xcc, 0xb1, 0xdd, 0xe7, 0x46, 0x60, 0x7a, 0x53, + 0x12, 0xc8, 0xfb, 0xbc, 0x43, 0x44, 0xd6, 0x21, 0x33, 0xfe, 0x52, 0xfc, 0xd3, 0x37, 0xb5, 0x54, + 0xdd, 0x85, 0xe2, 0x26, 0x4e, 0xa8, 0x41, 0x3a, 0x99, 0xf8, 0x24, 0x60, 0x82, 0xc9, 0xe0, 0x68, + 0xb6, 0x91, 0x41, 0x9a, 0x31, 0xc0, 0x65, 0x80, 0x40, 0xbc, 0x30, 0xfd, 0x0b, 0x26, 0x8d, 0x32, + 0x66, 0xe3, 0x30, 0xf1, 0x5f, 0x10, 0xf3, 0xb9, 0xc1, 0x1c, 0x5c, 0x18, 0x85, 0xd0, 0xf0, 0xd8, + 0xf4, 0x2f, 0xa2, 0xfd, 0x7e, 0x0d, 0x39, 0xfe, 0x10, 0xe8, 0x73, 0x28, 0x8c, 0xe9, 0xc2, 0x0d, + 0xb6, 0xcd, 0x61, 0x3f, 0x59, 0x5b, 0x98, 0x27, 0xba, 0xd9, 0x06, 0x58, 0x3f, 0x85, 0x7c, 0xe4, + 0x42, 0xf7, 0x37, 0x85, 0x4f, 0x6c, 0xde, 0xb9, 0xc6, 0xf9, 0x6e, 0xb7, 0xb8, 0x34, 0x9d, 0x05, + 0x3f, 0xbc, 0x88, 0xf9, 0xa4, 0xfe, 0x37, 0x01, 0xf2, 0x38, 0x7c, 0x67, 0x3f, 0x48, 0xf4, 0x99, + 0xec, 0x4e, 0x9f, 0xd9, 0x66, 0x64, 0x7a, 0x27, 0x23, 0xe3, 0xa4, 0xca, 0x24, 0x92, 0x6a, 0xcb, + 0x9c, 0xf8, 0x46, 0xe6, 0xb2, 0x6f, 0x60, 0x2e, 0x97, 0x60, 0xee, 0x3e, 0xec, 0x4d, 0x3c, 0x3a, + 0x63, 0x9d, 0x84, 0x7a, 0xa6, 0xb7, 0x8c, 0x12, 0xa0, 0x12, 0x5a, 0x87, 0xb1, 0xb1, 0x6e, 0x40, + 0x01, 0x13, 0x7f, 0x4e, 0x5d, 0x9f, 0xbc, 0xf5, 0xd8, 0x08, 0x44, 0xcb, 0x0c, 0x4c, 0x76, 0xe8, + 0x32, 0x66, 0x63, 0xf4, 0x00, 0xc4, 0x31, 0xb5, 0xf8, 0x91, 0xf7, 0x92, 0x1a, 0xd2, 0x3c, 0x8f, + 0x7a, 0x2d, 0x6a, 0x11, 0xcc, 0x00, 0xf5, 0x39, 0x48, 0x6d, 0xfa, 0xc2, 0x75, 0xa8, 0x69, 0xf5, + 0x3d, 0x3a, 0x0d, 0x2b, 0xfa, 0x5b, 0x2b, 0x53, 0x1b, 0xf2, 0x0b, 0x56, 0xbb, 0xe2, 0xda, 0x74, + 0x6f, 0xb7, 0x96, 0x5c, 0x0f, 0xc4, 0x0b, 0x5d, 0x9c, 0x80, 0xd1, 0xd2, 0xfa, 0x3f, 0x04, 0x50, + 0xde, 0x8e, 0x46, 0x1d, 0x28, 0x71, 0xa4, 0x91, 0xf8, 0x89, 0x39, 0x7a, 0x97, 0x8d, 0x58, 0x19, + 0x83, 0xc5, 0x66, 0xfc, 0xc6, 0x0e, 0x98, 0x28, 0x18, 0x99, 0x77, 0x2b, 0x18, 0x0f, 0xa0, 0xc2, + 0x33, 0x38, 0xee, 0xf7, 0xa2, 0x9a, 0x39, 0xca, 0x36, 0xd3, 0x52, 0x0a, 0x97, 0x47, 0x3c, 0x93, + 0x98, 0xbd, 0x9e, 0x03, 0xb1, 0x6f, 0xbb, 0xd3, 0x7a, 0x0d, 0xb2, 0x2d, 0x87, 0xb2, 0x07, 0xcb, + 0x79, 0xc4, 0xf4, 0xa9, 0x1b, 0xf3, 0xc8, 0x67, 0xc7, 0x7f, 0x4f, 0x43, 0x29, 0xf1, 0x2f, 0x86, + 0x1e, 0xc1, 0x5e, 0xab, 0x7b, 0x3e, 0x18, 0x6a, 0xd8, 0x68, 0xf5, 0xf4, 0xd3, 0xce, 0x99, 0x94, + 0x52, 0x0e, 0x57, 0x6b, 0x55, 0x9e, 0x6d, 0x41, 0xbb, 0xbf, 0x59, 0x35, 0xc8, 0x76, 0xf4, 0xb6, + 0xf6, 0x7b, 0x49, 0x50, 0x6e, 0xaf, 0xd6, 0xaa, 0x94, 0x00, 0xf2, 0x9e, 0xf5, 0x09, 0x94, 0x19, + 0xc0, 0x38, 0xef, 0xb7, 0x1b, 0x43, 0x4d, 0x4a, 0x2b, 0xca, 0x6a, 0xad, 0x1e, 0x5c, 0xc7, 0x45, + 0x9c, 0x7f, 0x08, 0x79, 0xac, 0xfd, 0xee, 0x5c, 0x1b, 0x0c, 0xa5, 0x8c, 0x72, 0xb0, 0x5a, 0xab, + 0x28, 0x01, 0x8c, 0xb3, 0xe6, 0x3e, 0x14, 0xb0, 0x36, 0xe8, 0xf7, 0xf4, 0x81, 0x26, 0x89, 0xca, + 0x8f, 0x56, 0x6b, 0xf5, 0xd6, 0x0e, 0x2a, 0x52, 0xe9, 0xcf, 0x60, 0xbf, 0xdd, 0xfb, 0x52, 0xef, + 0xf6, 0x1a, 0x6d, 0xa3, 0x8f, 0x7b, 0x67, 0x58, 0x1b, 0x0c, 0xa4, 0xac, 0x52, 0x5b, 0xad, 0xd5, + 0xf7, 0x13, 0xf8, 0x1b, 0xa2, 0xfb, 0x00, 0xc4, 0x7e, 0x47, 0x3f, 0x93, 0x72, 0xca, 0xad, 0xd5, + 0x5a, 0x7d, 0x2f, 0x01, 0x0d, 0x49, 0x0d, 0x6f, 0xdc, 0xea, 0xf6, 0x06, 0x9a, 0x94, 0xbf, 0x71, + 0x63, 0x46, 0xf6, 0xf1, 0x1f, 0x00, 0xdd, 0xfc, 0x5b, 0x45, 0xf7, 0x40, 0xd4, 0x7b, 0xba, 0x26, + 0xa5, 0xf8, 0xfd, 0x6f, 0x22, 0x74, 0xea, 0x12, 0x54, 0x87, 0x4c, 0xf7, 0xab, 0x2f, 0x24, 0x41, + 0xf9, 0xf1, 0x6a, 0xad, 0xde, 0xb9, 0x09, 0xea, 0x7e, 0xf5, 0xc5, 0x31, 0x85, 0x52, 0x32, 0x70, + 0x1d, 0x0a, 0x4f, 0xb4, 0x61, 0xa3, 0xdd, 0x18, 0x36, 0xa4, 0x14, 0x3f, 0x52, 0xec, 0x7e, 0x42, + 0x02, 0x93, 0x25, 0xe1, 0x21, 0x64, 0x75, 0xed, 0xa9, 0x86, 0x25, 0x41, 0xd9, 0x5f, 0xad, 0xd5, + 0x4a, 0x0c, 0xd0, 0xc9, 0x25, 0xf1, 0x50, 0x15, 0x72, 0x8d, 0xee, 0x97, 0x8d, 0x67, 0x03, 0x29, + 0xad, 0xa0, 0xd5, 0x5a, 0xdd, 0x8b, 0xdd, 0x0d, 0xe7, 0x85, 0xb9, 0xf4, 0x8f, 0xff, 0x27, 0x40, + 0x39, 0xd9, 0xa1, 0x51, 0x15, 0xc4, 0xd3, 0x4e, 0x57, 0x8b, 0xb7, 0x4b, 0xfa, 0xc2, 0x31, 0x3a, + 0x82, 0x62, 0xbb, 0x83, 0xb5, 0xd6, 0xb0, 0x87, 0x9f, 0xc5, 0x77, 0x49, 0x82, 0xda, 0xb6, 0xc7, + 0x04, 0xbe, 0x44, 0xbf, 0x80, 0xf2, 0xe0, 0xd9, 0x93, 0x6e, 0x47, 0xff, 0xad, 0xc1, 0x22, 0xa6, + 0x95, 0x07, 0xab, 0xb5, 0x7a, 0x77, 0x07, 0x4c, 0xe6, 0x1e, 0x19, 0x9b, 0x01, 0xb1, 0x06, 0xbc, + 0x89, 0x84, 0xce, 0x82, 0x80, 0x5a, 0xb0, 0x1f, 0x2f, 0xdd, 0x6e, 0x96, 0x51, 0x3e, 0x59, 0xad, + 0xd5, 0x8f, 0xbe, 0x77, 0xfd, 0x66, 0xf7, 0x82, 0x80, 0xee, 0x41, 0x3e, 0x0a, 0x12, 0x2b, 0x29, + 0xb9, 0x34, 0x5a, 0x70, 0xfc, 0x17, 0x01, 0x8a, 0x9b, 0x72, 0x15, 0x12, 0xae, 0xf7, 0x0c, 0x0d, + 0xe3, 0x1e, 0x8e, 0x19, 0xd8, 0x38, 0x75, 0xca, 0x86, 0xe8, 0x2e, 0xe4, 0xcf, 0x34, 0x5d, 0xc3, + 0x9d, 0x56, 0x9c, 0x18, 0x1b, 0xc8, 0x19, 0x71, 0x89, 0x67, 0x8f, 0xd1, 0xc7, 0x50, 0xd6, 0x7b, + 0xc6, 0xe0, 0xbc, 0xf5, 0x38, 0xbe, 0x3a, 0xdb, 0x3f, 0x11, 0x6a, 0xb0, 0x18, 0x5f, 0x30, 0x3e, + 0x8f, 0xc3, 0x1c, 0x7a, 0xda, 0xe8, 0x76, 0xda, 0x1c, 0x9a, 0x51, 0xe4, 0xd5, 0x5a, 0xbd, 0xbd, + 0x81, 0x76, 0xf8, 0xaf, 0x4a, 0x88, 0x3d, 0xb6, 0xa0, 0xfa, 0xfd, 0x85, 0x09, 0xa9, 0x90, 0x6b, + 0xf4, 0xfb, 0x9a, 0xde, 0x8e, 0x4f, 0xbf, 0xf5, 0x35, 0xe6, 0x73, 0xe2, 0x5a, 0x21, 0xe2, 0xb4, + 0x87, 0xcf, 0xb4, 0x61, 0x7c, 0xf8, 0x2d, 0xe2, 0x94, 0x86, 0x1d, 0xbc, 0x79, 0xf8, 0xea, 0xbb, + 0x6a, 0xea, 0xdb, 0xef, 0xaa, 0xa9, 0x57, 0x57, 0x55, 0xe1, 0xdb, 0xab, 0xaa, 0xf0, 0xaf, 0xab, + 0x6a, 0xea, 0x3f, 0x57, 0x55, 0xe1, 0x9b, 0x7f, 0x57, 0x85, 0x51, 0x8e, 0x15, 0xb2, 0xcf, 0xff, + 0x1f, 0x00, 0x00, 0xff, 0xff, 0x0d, 0x36, 0x7c, 0x9b, 0xc0, 0x0e, 0x00, 0x00, } diff --git a/lib/protocol/bep.proto b/lib/protocol/bep.proto index fc6c9d9c7..008801e57 100644 --- a/lib/protocol/bep.proto +++ b/lib/protocol/bep.proto @@ -105,6 +105,7 @@ message FileInfo { bool no_permissions = 8; Vector version = 9 [(gogoproto.nullable) = false]; int64 sequence = 10; + int32 block_size = 13 [(gogoproto.customname) = "RawBlockSize"]; repeated BlockInfo Blocks = 16 [(gogoproto.nullable) = false]; string symlink_target = 17; diff --git a/lib/protocol/bep_extensions.go b/lib/protocol/bep_extensions.go index db31f4522..5dde3ea9c 100644 --- a/lib/protocol/bep_extensions.go +++ b/lib/protocol/bep_extensions.go @@ -14,16 +14,11 @@ import ( "time" "github.com/syncthing/syncthing/lib/rand" - "github.com/syncthing/syncthing/lib/sha256" ) const ( SyntheticDirectorySize = 128 -) - -var ( - sha256OfEmptyBlock = sha256.Sum256(make([]byte, BlockSize)) - HelloMessageMagic = uint32(0x2EA7D90B) + HelloMessageMagic = uint32(0x2EA7D90B) ) func (m Hello) Magic() uint32 { @@ -36,8 +31,8 @@ func (f FileInfo) String() string { return fmt.Sprintf("Directory{Name:%q, Sequence:%d, Permissions:0%o, ModTime:%v, Version:%v, Deleted:%v, Invalid:%v, NoPermissions:%v}", f.Name, f.Sequence, f.Permissions, f.ModTime(), f.Version, f.Deleted, f.Invalid, f.NoPermissions) case FileInfoTypeFile: - return fmt.Sprintf("File{Name:%q, Sequence:%d, Permissions:0%o, ModTime:%v, Version:%v, Length:%d, Deleted:%v, Invalid:%v, NoPermissions:%v, Blocks:%v}", - f.Name, f.Sequence, f.Permissions, f.ModTime(), f.Version, f.Size, f.Deleted, f.Invalid, f.NoPermissions, f.Blocks) + return fmt.Sprintf("File{Name:%q, Sequence:%d, Permissions:0%o, ModTime:%v, Version:%v, Length:%d, Deleted:%v, Invalid:%v, NoPermissions:%v, BlockSize:%d, Blocks:%v}", + f.Name, f.Sequence, f.Permissions, f.ModTime(), f.Version, f.Size, f.Deleted, f.Invalid, f.NoPermissions, f.RawBlockSize, f.Blocks) case FileInfoTypeSymlink, FileInfoTypeDeprecatedSymlinkDirectory, FileInfoTypeDeprecatedSymlinkFile: return fmt.Sprintf("Symlink{Name:%q, Type:%v, Sequence:%d, Version:%v, Deleted:%v, Invalid:%v, NoPermissions:%v, SymlinkTarget:%q}", f.Name, f.Type, f.Sequence, f.Version, f.Deleted, f.Invalid, f.NoPermissions, f.SymlinkTarget) @@ -81,6 +76,13 @@ func (f FileInfo) FileSize() int64 { return f.Size } +func (f FileInfo) BlockSize() int { + if f.RawBlockSize == 0 { + return MinBlockSize + } + return int(f.RawBlockSize) +} + func (f FileInfo) FileName() string { return f.Name } @@ -204,7 +206,10 @@ func (b BlockInfo) String() string { // IsEmpty returns true if the block is a full block of zeroes. func (b BlockInfo) IsEmpty() bool { - return b.Size == BlockSize && bytes.Equal(b.Hash, sha256OfEmptyBlock[:]) + if v, ok := sha256OfEmptyBlock[int(b.Size)]; ok { + return bytes.Equal(b.Hash, v[:]) + } + return false } type IndexID uint64 diff --git a/lib/protocol/protocol.go b/lib/protocol/protocol.go index 22275a57c..870fe2102 100644 --- a/lib/protocol/protocol.go +++ b/lib/protocol/protocol.go @@ -3,6 +3,7 @@ package protocol import ( + "crypto/sha256" "encoding/binary" "errors" "fmt" @@ -16,13 +17,51 @@ import ( ) const ( - // BlockSize is the standard data block size (128 KiB) - BlockSize = 128 << 10 + // Shifts + KiB = 10 + MiB = 20 + GiB = 30 +) +const ( // MaxMessageLen is the largest message size allowed on the wire. (500 MB) MaxMessageLen = 500 * 1000 * 1000 + + // MinBlockSize is the minimum block size allowed + MinBlockSize = 128 << KiB + + // MaxBlockSize is the maximum block size allowed + MaxBlockSize = 16 << MiB + + // DesiredPerFileBlocks is the number of blocks we aim for per file + DesiredPerFileBlocks = 2000 ) +// BlockSizes is the list of valid block sizes, from min to max +var BlockSizes []int + +// For each block size, the hash of a block of all zeroes +var sha256OfEmptyBlock = make(map[int][sha256.Size]byte) + +func init() { + for blockSize := MinBlockSize; blockSize <= MaxBlockSize; blockSize *= 2 { + BlockSizes = append(BlockSizes, blockSize) + sha256OfEmptyBlock[blockSize] = sha256.Sum256(make([]byte, blockSize)) + } +} + +// BlockSize returns the block size to use for the given file size +func BlockSize(fileSize int64) int { + var blockSize int + for _, blockSize = range BlockSizes { + if fileSize < int64(DesiredPerFileBlocks*blockSize) { + break + } + } + + return blockSize +} + const ( stateInitial = iota stateReady @@ -158,7 +197,7 @@ func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiv awaiting: make(map[int32]chan asyncResult), outbox: make(chan asyncMessage), closed: make(chan struct{}), - pool: bufferPool{minSize: BlockSize}, + pool: bufferPool{minSize: MinBlockSize}, compression: compress, } @@ -533,7 +572,7 @@ func checkFilename(name string) error { func (c *rawConnection) handleRequest(req Request) { size := int(req.Size) - usePool := size <= BlockSize + usePool := size <= MaxBlockSize var buf []byte var done chan struct{} diff --git a/lib/protocol/protocol_test.go b/lib/protocol/protocol_test.go index f19710ba9..bccff8d52 100644 --- a/lib/protocol/protocol_test.go +++ b/lib/protocol/protocol_test.go @@ -11,7 +11,6 @@ import ( "strings" "testing" "testing/quick" - "time" "encoding/hex" @@ -234,48 +233,6 @@ func testMarshal(t *testing.T, prefix string, m1, m2 message) bool { return true } -func TestMarshalledIndexMessageSize(t *testing.T) { - // We should be able to handle a 1 TiB file without - // blowing the default max message size. - - if testing.Short() { - t.Skip("this test requires a lot of memory") - return - } - - const ( - maxMessageSize = MaxMessageLen - fileSize = 1 << 40 - blockSize = BlockSize - ) - - f := FileInfo{ - Name: "a normal length file name withoout any weird stuff.txt", - Type: FileInfoTypeFile, - Size: fileSize, - Permissions: 0666, - ModifiedS: time.Now().Unix(), - Version: Vector{Counters: []Counter{{ID: 1 << 60, Value: 1}, {ID: 2 << 60, Value: 1}}}, - Blocks: make([]BlockInfo, fileSize/blockSize), - } - - for i := 0; i < fileSize/blockSize; i++ { - f.Blocks[i].Offset = int64(i) * blockSize - f.Blocks[i].Size = blockSize - f.Blocks[i].Hash = []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 20, 1, 2, 3, 4, 5, 6, 7, 8, 9, 30, 1, 2} - } - - idx := Index{ - Folder: "some folder ID", - Files: []FileInfo{f}, - } - - msgSize := idx.ProtoSize() - if msgSize > maxMessageSize { - t.Errorf("Message size %d bytes is larger than max %d", msgSize, maxMessageSize) - } -} - func TestLZ4Compression(t *testing.T) { c := new(rawConnection) @@ -400,3 +357,36 @@ func TestCheckConsistency(t *testing.T) { } } } + +func TestBlockSize(t *testing.T) { + cases := []struct { + fileSize int64 + blockSize int + }{ + {1 << KiB, 128 << KiB}, + {1 << MiB, 128 << KiB}, + {499 << MiB, 256 << KiB}, + {500 << MiB, 512 << KiB}, + {501 << MiB, 512 << KiB}, + {1 << GiB, 1 << MiB}, + {2 << GiB, 2 << MiB}, + {3 << GiB, 2 << MiB}, + {500 << GiB, 16 << MiB}, + {50000 << GiB, 16 << MiB}, + } + + for _, tc := range cases { + size := BlockSize(tc.fileSize) + if size != tc.blockSize { + t.Errorf("BlockSize(%d), size=%d, expected %d", tc.fileSize, size, tc.blockSize) + } + } +} + +var blockSize int + +func BenchmarkBlockSize(b *testing.B) { + for i := 0; i < b.N; i++ { + blockSize = BlockSize(16 << 30) + } +} diff --git a/lib/scanner/blockqueue.go b/lib/scanner/blockqueue.go index e1ef4be1f..2cfeec84f 100644 --- a/lib/scanner/blockqueue.go +++ b/lib/scanner/blockqueue.go @@ -63,7 +63,6 @@ func HashFile(ctx context.Context, fs fs.Filesystem, path string, blockSize int, // is closed and all items handled. type parallelHasher struct { fs fs.Filesystem - blockSize int workers int outbox chan<- protocol.FileInfo inbox <-chan protocol.FileInfo @@ -73,10 +72,9 @@ type parallelHasher struct { wg sync.WaitGroup } -func newParallelHasher(ctx context.Context, fs fs.Filesystem, blockSize, workers int, outbox chan<- protocol.FileInfo, inbox <-chan protocol.FileInfo, counter Counter, done chan<- struct{}, useWeakHashes bool) { +func newParallelHasher(ctx context.Context, fs fs.Filesystem, workers int, outbox chan<- protocol.FileInfo, inbox <-chan protocol.FileInfo, counter Counter, done chan<- struct{}, useWeakHashes bool) { ph := ¶llelHasher{ fs: fs, - blockSize: blockSize, workers: workers, outbox: outbox, inbox: inbox, @@ -108,7 +106,7 @@ func (ph *parallelHasher) hashFiles(ctx context.Context) { panic("Bug. Asked to hash a directory or a deleted file.") } - blocks, err := HashFile(ctx, ph.fs, f.Name, ph.blockSize, ph.counter, ph.useWeakHashes) + blocks, err := HashFile(ctx, ph.fs, f.Name, f.BlockSize(), ph.counter, ph.useWeakHashes) if err != nil { l.Debugln("hash error:", f.Name, err) continue diff --git a/lib/scanner/blocks_test.go b/lib/scanner/blocks_test.go index 8d706dc00..311dbdd72 100644 --- a/lib/scanner/blocks_test.go +++ b/lib/scanner/blocks_test.go @@ -125,7 +125,7 @@ func TestAdler32Variants(t *testing.T) { } // protocol block sized data - data := make([]byte, protocol.BlockSize) + data := make([]byte, protocol.MinBlockSize) for i := 0; i < 5; i++ { rand.Read(data) if !checkFn(data) { diff --git a/lib/scanner/infinitefs_test.go b/lib/scanner/virtualfs_test.go similarity index 77% rename from lib/scanner/infinitefs_test.go rename to lib/scanner/virtualfs_test.go index 6d850fefe..8445d8d61 100644 --- a/lib/scanner/infinitefs_test.go +++ b/lib/scanner/virtualfs_test.go @@ -55,6 +55,48 @@ func (i infiniteFS) Open(name string) (fs.File, error) { return &fakeFile{name, i.filesize, 0}, nil } +type singleFileFS struct { + fs.Filesystem + name string + filesize int64 +} + +func (s singleFileFS) Lstat(name string) (fs.FileInfo, error) { + switch name { + case ".": + return fakeInfo{".", 0}, nil + case s.name: + return fakeInfo{s.name, s.filesize}, nil + default: + return nil, errors.New("no such file") + } +} + +func (s singleFileFS) Stat(name string) (fs.FileInfo, error) { + switch name { + case ".": + return fakeInfo{".", 0}, nil + case s.name: + return fakeInfo{s.name, s.filesize}, nil + default: + return nil, errors.New("no such file") + } +} + +func (s singleFileFS) DirNames(name string) ([]string, error) { + if name != "." { + return nil, errors.New("no such file") + } + return []string{s.name}, nil +} + +func (s singleFileFS) Open(name string) (fs.File, error) { + if name != s.name { + return nil, errors.New("no such file") + } + return &fakeFile{s.name, s.filesize, 0}, nil +} + type fakeInfo struct { name string size int64 diff --git a/lib/scanner/walk.go b/lib/scanner/walk.go index 6f958e9dc..0be3fde05 100644 --- a/lib/scanner/walk.go +++ b/lib/scanner/walk.go @@ -42,8 +42,6 @@ type Config struct { Folder string // Limit walking to these paths within Dir, or no limit if Sub is empty Subs []string - // BlockSize controls the size of the block used when hashing. - BlockSize int // If Matcher is not nil, it is used to identify files to ignore which were specified by the user. Matcher *ignore.Matcher // Number of hours to keep temporary files for @@ -68,6 +66,8 @@ type Config struct { ProgressTickIntervalS int // Whether or not we should also compute weak hashes UseWeakHashes bool + // Whether to use large blocks for large files or the old standard of 128KiB for everything. + UseLargeBlocks bool } type CurrentFiler interface { @@ -98,7 +98,7 @@ type walker struct { // Walk returns the list of files found in the local folder by scanning the // file system. Files are blockwise hashed. func (w *walker) walk(ctx context.Context) chan protocol.FileInfo { - l.Debugln("Walk", w.Subs, w.BlockSize, w.Matcher) + l.Debugln("Walk", w.Subs, w.Matcher) toHashChan := make(chan protocol.FileInfo) finishedChan := make(chan protocol.FileInfo) @@ -120,7 +120,7 @@ func (w *walker) walk(ctx context.Context) chan protocol.FileInfo { // We're not required to emit scan progress events, just kick off hashers, // and feed inputs directly from the walker. if w.ProgressTickIntervalS < 0 { - newParallelHasher(ctx, w.Filesystem, w.BlockSize, w.Hashers, finishedChan, toHashChan, nil, nil, w.UseWeakHashes) + newParallelHasher(ctx, w.Filesystem, w.Hashers, finishedChan, toHashChan, nil, nil, w.UseWeakHashes) return finishedChan } @@ -151,7 +151,7 @@ func (w *walker) walk(ctx context.Context) chan protocol.FileInfo { done := make(chan struct{}) progress := newByteCounter() - newParallelHasher(ctx, w.Filesystem, w.BlockSize, w.Hashers, finishedChan, realToHashChan, progress, done, w.UseWeakHashes) + newParallelHasher(ctx, w.Filesystem, w.Hashers, finishedChan, realToHashChan, progress, done, w.UseWeakHashes) // A routine which actually emits the FolderScanProgress events // every w.ProgressTicker ticks, until the hasher routines terminate. @@ -161,7 +161,7 @@ func (w *walker) walk(ctx context.Context) chan protocol.FileInfo { for { select { case <-done: - l.Debugln("Walk progress done", w.Folder, w.Subs, w.BlockSize, w.Matcher) + l.Debugln("Walk progress done", w.Folder, w.Subs, w.Matcher) ticker.Stop() return case <-ticker.C: @@ -285,32 +285,52 @@ func (w *walker) walkRegular(ctx context.Context, relPath string, info fs.FileIn curMode |= 0111 } - cf, ok := w.CurrentFiler.CurrentFile(relPath) + blockSize := protocol.MinBlockSize + curFile, hasCurFile := w.CurrentFiler.CurrentFile(relPath) + + if w.UseLargeBlocks { + blockSize = protocol.BlockSize(info.Size()) + + if hasCurFile { + // Check if we should retain current block size. + curBlockSize := curFile.BlockSize() + if blockSize > curBlockSize && blockSize/curBlockSize <= 2 { + // New block size is larger, but not more than twice larger. + // Retain. + blockSize = curBlockSize + } else if curBlockSize > blockSize && curBlockSize/blockSize <= 2 { + // Old block size is larger, but not more than twice larger. + // Retain. + blockSize = curBlockSize + } + } + } f := protocol.FileInfo{ Name: relPath, Type: protocol.FileInfoTypeFile, - Version: cf.Version.Update(w.ShortID), + Version: curFile.Version.Update(w.ShortID), Permissions: curMode & uint32(maskModePerm), NoPermissions: w.IgnorePerms, ModifiedS: info.ModTime().Unix(), ModifiedNs: int32(info.ModTime().Nanosecond()), ModifiedBy: w.ShortID, Size: info.Size(), + RawBlockSize: int32(blockSize), } - if ok { - if cf.IsEquivalent(f, w.IgnorePerms, true) { + if hasCurFile { + if curFile.IsEquivalent(f, w.IgnorePerms, true) { return nil } - if cf.Invalid { + if curFile.Invalid { // We do not want to override the global version with the file we // currently have. Keeping only our local counter makes sure we are in // conflict with any other existing versions, which will be resolved by // the normal pulling mechanisms. f.Version = f.Version.DropOthers(w.ShortID) } - l.Debugln("rescan:", cf, info.ModTime().Unix(), info.Mode()&fs.ModePerm) + l.Debugln("rescan:", curFile, info.ModTime().Unix(), info.Mode()&fs.ModePerm) } l.Debugln("to hash:", relPath, f) diff --git a/lib/scanner/walk_test.go b/lib/scanner/walk_test.go index 33e2308f3..80ad3f5d6 100644 --- a/lib/scanner/walk_test.go +++ b/lib/scanner/walk_test.go @@ -65,7 +65,6 @@ func TestWalkSub(t *testing.T) { fchan := Walk(context.TODO(), Config{ Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"), Subs: []string{"dir2"}, - BlockSize: 128 * 1024, Matcher: ignores, Hashers: 2, }) @@ -98,7 +97,6 @@ func TestWalk(t *testing.T) { fchan := Walk(context.TODO(), Config{ Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"), - BlockSize: 128 * 1024, Matcher: ignores, Hashers: 2, }) @@ -221,11 +219,11 @@ func TestNormalization(t *testing.T) { // make sure it all gets done. In production, things will be correct // eventually... - _, err := walkDir(fs, "testdata/normalization") + _, err := walkDir(fs, "testdata/normalization", nil) if err != nil { t.Fatal(err) } - tmp, err := walkDir(fs, "testdata/normalization") + tmp, err := walkDir(fs, "testdata/normalization", nil) if err != nil { t.Fatal(err) } @@ -272,7 +270,7 @@ func TestWalkSymlinkUnix(t *testing.T) { for _, path := range []string{".", "link"} { // Scan it - files, _ := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks"), path) + files, _ := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks"), path, nil) // Verify that we got one symlink and with the correct attributes if len(files) != 1 { @@ -303,7 +301,7 @@ func TestWalkSymlinkWindows(t *testing.T) { for _, path := range []string{".", "link"} { // Scan it - files, _ := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks"), path) + files, _ := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks"), path, nil) // Verify that we got zero symlinks if len(files) != 0 { @@ -332,7 +330,7 @@ func TestWalkRootSymlink(t *testing.T) { } // Scan it - files, err := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, link), ".") + files, err := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, link), ".", nil) if err != nil { t.Fatal("Expected no error when root folder path is provided via a symlink: " + err.Error()) } @@ -342,13 +340,83 @@ func TestWalkRootSymlink(t *testing.T) { } } -func walkDir(fs fs.Filesystem, dir string) ([]protocol.FileInfo, error) { +func TestBlocksizeHysteresis(t *testing.T) { + // Verify that we select the right block size in the presence of old + // file information. + + sf := fs.NewWalkFilesystem(&singleFileFS{ + name: "testfile.dat", + filesize: 500 << 20, // 500 MiB + }) + + current := make(fakeCurrentFiler) + + runTest := func(expectedBlockSize int) { + files, err := walkDir(sf, ".", current) + if err != nil { + t.Fatal(err) + } + if len(files) != 1 { + t.Fatalf("expected one file, not %d", len(files)) + } + if s := files[0].BlockSize(); s != expectedBlockSize { + t.Fatalf("incorrect block size %d != expected %d", s, expectedBlockSize) + } + } + + // Scan with no previous knowledge. We should get a 512 KiB block size. + + runTest(512 << 10) + + // Scan on the assumption that previous size was 256 KiB. Retain 256 KiB + // block size. + + current["testfile.dat"] = protocol.FileInfo{ + Name: "testfile.dat", + Size: 500 << 20, + RawBlockSize: 256 << 10, + } + runTest(256 << 10) + + // Scan on the assumption that previous size was 1 MiB. Retain 1 MiB + // block size. + + current["testfile.dat"] = protocol.FileInfo{ + Name: "testfile.dat", + Size: 500 << 20, + RawBlockSize: 1 << 20, + } + runTest(1 << 20) + + // Scan on the assumption that previous size was 128 KiB. Move to 512 + // KiB because the difference is large. + + current["testfile.dat"] = protocol.FileInfo{ + Name: "testfile.dat", + Size: 500 << 20, + RawBlockSize: 128 << 10, + } + runTest(512 << 10) + + // Scan on the assumption that previous size was 2 MiB. Move to 512 + // KiB because the difference is large. + + current["testfile.dat"] = protocol.FileInfo{ + Name: "testfile.dat", + Size: 500 << 20, + RawBlockSize: 2 << 20, + } + runTest(512 << 10) +} + +func walkDir(fs fs.Filesystem, dir string, cfiler CurrentFiler) ([]protocol.FileInfo, error) { fchan := Walk(context.TODO(), Config{ - Filesystem: fs, - Subs: []string{dir}, - BlockSize: 128 * 1024, - AutoNormalize: true, - Hashers: 2, + Filesystem: fs, + Subs: []string{dir}, + AutoNormalize: true, + Hashers: 2, + UseLargeBlocks: true, + CurrentFiler: cfiler, }) var tmp []protocol.FileInfo @@ -410,7 +478,7 @@ func BenchmarkHashFile(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - if _, err := HashFile(context.TODO(), fs.NewFilesystem(fs.FilesystemTypeBasic, ""), testdataName, protocol.BlockSize, nil, true); err != nil { + if _, err := HashFile(context.TODO(), fs.NewFilesystem(fs.FilesystemTypeBasic, ""), testdataName, protocol.MinBlockSize, nil, true); err != nil { b.Fatal(err) } } @@ -451,7 +519,6 @@ func TestStopWalk(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) fchan := Walk(ctx, Config{ Filesystem: fs, - BlockSize: 128 * 1024, Hashers: numHashers, ProgressTickIntervalS: -1, // Don't attempt to build the full list of files before starting to scan... }) @@ -513,7 +580,7 @@ func TestIssue4799(t *testing.T) { } fd.Close() - files, err := walkDir(fs, "/foo") + files, err := walkDir(fs, "/foo", nil) if err != nil { t.Fatal(err) } @@ -540,7 +607,6 @@ func TestIssue4841(t *testing.T) { fchan := Walk(context.TODO(), Config{ Filesystem: fs, Subs: nil, - BlockSize: 128 * 1024, AutoNormalize: true, Hashers: 2, CurrentFiler: fakeCurrentFiler{ diff --git a/test/h1/config.xml b/test/h1/config.xml index 80ccec9c3..418d253a5 100644 --- a/test/h1/config.xml +++ b/test/h1/config.xml @@ -19,6 +19,7 @@ false 25 .stfolder + true basic @@ -38,6 +39,7 @@ false 25 .stfolder + true
tcp://127.0.0.1:22004
diff --git a/test/h2/config.xml b/test/h2/config.xml index cf25c4c7d..a5d21e274 100644 --- a/test/h2/config.xml +++ b/test/h2/config.xml @@ -18,6 +18,7 @@ false 25 .stfolder + true basic @@ -37,6 +38,7 @@ false 25 .stfolder + true basic @@ -56,6 +58,7 @@ false 25 .stfolder + true
tcp://127.0.0.1:22001
diff --git a/test/h3/config.xml b/test/h3/config.xml index 6eca4321a..a204db7d4 100644 --- a/test/h3/config.xml +++ b/test/h3/config.xml @@ -20,6 +20,7 @@ false 25 .stfolder + true basic @@ -39,6 +40,7 @@ false 25 .stfolder + true
tcp://127.0.0.1:22001