From 05828368200d519e3ddc18f22ebf05c4a6c5e836 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Wed, 14 Dec 2016 23:30:29 +0000 Subject: [PATCH] lib/model, lib/scanner: Efficient inserts/deletes in the middle of the file GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3527 --- lib/config/folderconfiguration.go | 1 + lib/db/structs.pb.go | 58 +++---- lib/discover/local.pb.go | 33 ++-- lib/model/rwfolder.go | 104 +++++++++---- lib/model/rwfolder_test.go | 133 +++++++++++++++- lib/model/sharedpullerstate.go | 50 +++--- lib/protocol/bep.pb.go | 248 +++++++++++++++++------------- lib/protocol/bep.proto | 7 +- lib/protocol/bep_extensions.go | 2 +- lib/protocol/deviceid_test.pb.go | 16 +- lib/scanner/blocks.go | 62 ++------ lib/weakhash/benchmark_test.go | 30 ++++ lib/weakhash/weakhash.go | 169 ++++++++++++++++++++ lib/weakhash/weakhash_test.go | 188 ++++++++++++++++++++++ test/h2/config.xml | 75 ++++----- test/h3/config.xml | 8 +- 16 files changed, 878 insertions(+), 306 deletions(-) create mode 100644 lib/weakhash/benchmark_test.go create mode 100644 lib/weakhash/weakhash.go create mode 100644 lib/weakhash/weakhash_test.go diff --git a/lib/config/folderconfiguration.go b/lib/config/folderconfiguration.go index a1d8aed7c..b36ca60bd 100644 --- a/lib/config/folderconfiguration.go +++ b/lib/config/folderconfiguration.go @@ -40,6 +40,7 @@ type FolderConfiguration struct { DisableSparseFiles bool `xml:"disableSparseFiles" json:"disableSparseFiles"` DisableTempIndexes bool `xml:"disableTempIndexes" json:"disableTempIndexes"` Fsync bool `xml:"fsync" json:"fsync"` + DisableWeakHash bool `xml:"disableWeakHash" json:"disableWeakHash"` cachedPath string diff --git a/lib/db/structs.pb.go b/lib/db/structs.pb.go index 1ba6e4194..c63f3b029 100644 --- a/lib/db/structs.pb.go +++ b/lib/db/structs.pb.go @@ -954,33 +954,33 @@ var ( ) var fileDescriptorStructs = []byte{ - // 442 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x51, 0x5d, 0x8b, 0xd3, 0x40, - 0x14, 0x6d, 0xb7, 0xd9, 0x7e, 0xdc, 0xd8, 0xd5, 0x1d, 0x64, 0x19, 0x0a, 0xb6, 0x4b, 0x41, 0x10, - 0xc1, 0x54, 0x57, 0x7c, 0xf1, 0x71, 0x1f, 0x16, 0x04, 0x11, 0x19, 0x97, 0xf5, 0xb1, 0x34, 0x99, - 0xdb, 0xec, 0x60, 0x32, 0x53, 0x33, 0x93, 0x42, 0xfd, 0x25, 0xbe, 0xb9, 0x3f, 0xa7, 0x8f, 0xfe, - 0x02, 0xd1, 0xfa, 0x47, 0x9c, 0xce, 0xa4, 0x31, 0x8f, 0xfb, 0x10, 0xb8, 0xe7, 0x9e, 0x73, 0xee, - 0x3d, 0x93, 0x0b, 0x43, 0x6d, 0x8a, 0x32, 0x31, 0x3a, 0x5a, 0x15, 0xca, 0x28, 0x72, 0xc4, 0xe3, - 0xd1, 0x8b, 0x54, 0x98, 0xdb, 0x32, 0x8e, 0x12, 0x95, 0xcf, 0x52, 0x95, 0xaa, 0x99, 0xa3, 0xe2, - 0x72, 0xe9, 0x90, 0x03, 0xae, 0xf2, 0x96, 0xd1, 0x9b, 0x86, 0x5c, 0x6f, 0x64, 0x62, 0x6e, 0x85, - 0x4c, 0x1b, 0x55, 0x26, 0x62, 0x3f, 0x21, 0x51, 0xd9, 0x2c, 0xc6, 0x95, 0xb7, 0x4d, 0x3f, 0x43, - 0x78, 0x25, 0x32, 0xbc, 0xc1, 0x42, 0x0b, 0x25, 0xc9, 0x4b, 0xe8, 0xad, 0x7d, 0x49, 0xdb, 0xe7, - 0xed, 0x67, 0xe1, 0xc5, 0xa3, 0xe8, 0x60, 0x8a, 0x6e, 0x30, 0x31, 0xaa, 0xb8, 0x0c, 0xb6, 0xbf, - 0x26, 0x2d, 0x76, 0x90, 0x91, 0x33, 0xe8, 0x72, 0x5c, 0x8b, 0x04, 0xe9, 0x91, 0x35, 0x3c, 0x60, - 0x15, 0x9a, 0x5e, 0x41, 0x58, 0x0d, 0x7d, 0x2f, 0xb4, 0x21, 0xaf, 0xa0, 0x5f, 0x39, 0xb4, 0x9d, - 0xdc, 0xb1, 0x93, 0x1f, 0x46, 0x3c, 0x8e, 0x1a, 0xbb, 0xab, 0xc1, 0xb5, 0xec, 0x6d, 0xf0, 0xfd, - 0x6e, 0xd2, 0x9a, 0xfe, 0xe8, 0xc0, 0xe9, 0x5e, 0xf5, 0x4e, 0x2e, 0xd5, 0x75, 0x51, 0xca, 0x64, - 0x61, 0x90, 0x13, 0x02, 0x81, 0x5c, 0xe4, 0xe8, 0x42, 0x0e, 0x98, 0xab, 0xc9, 0x73, 0x08, 0xcc, - 0x66, 0xe5, 0x73, 0x9c, 0x5c, 0x9c, 0xfd, 0x0f, 0x5e, 0xdb, 0x2d, 0xcb, 0x9c, 0x66, 0xef, 0xd7, - 0xe2, 0x1b, 0xd2, 0x8e, 0xd5, 0x76, 0x98, 0xab, 0xc9, 0x39, 0x84, 0x2b, 0x2c, 0x72, 0xa1, 0x7d, - 0xca, 0xc0, 0x52, 0x43, 0xd6, 0x6c, 0x91, 0x27, 0x00, 0xb9, 0xe2, 0x62, 0x29, 0x90, 0xcf, 0x35, - 0x3d, 0x76, 0xde, 0xc1, 0xa1, 0xf3, 0x89, 0x50, 0xe8, 0x71, 0xcc, 0xd0, 0xe6, 0xa3, 0x5d, 0xcb, - 0xf5, 0xd9, 0x01, 0xee, 0x19, 0x21, 0xd7, 0x8b, 0x4c, 0x70, 0xda, 0xf3, 0x4c, 0x05, 0xc9, 0x53, - 0x38, 0x91, 0x6a, 0xde, 0xdc, 0xdb, 0x77, 0x82, 0xa1, 0x54, 0x1f, 0x1b, 0x9b, 0x1b, 0x77, 0x19, - 0xdc, 0xef, 0x2e, 0x23, 0xe8, 0x6b, 0xfc, 0x5a, 0xa2, 0xb4, 0x97, 0x01, 0x97, 0xb4, 0xc6, 0x64, - 0x02, 0x61, 0xfd, 0x0e, 0xbb, 0x31, 0xb4, 0xf4, 0x31, 0xab, 0x9f, 0xf6, 0x41, 0xef, 0x53, 0xe9, - 0x4d, 0x9e, 0x09, 0xf9, 0x65, 0x6e, 0x16, 0x45, 0x8a, 0x86, 0x9e, 0xba, 0x1f, 0x3d, 0xac, 0xba, - 0xd7, 0xae, 0xe9, 0x2f, 0x74, 0xf9, 0x78, 0xfb, 0x67, 0xdc, 0xda, 0xee, 0xc6, 0xed, 0x9f, 0xf6, - 0xfb, 0xbd, 0x1b, 0xb7, 0xee, 0xfe, 0x8e, 0xdb, 0x71, 0xd7, 0xe5, 0x7b, 0xfd, 0x2f, 0x00, 0x00, - 0xff, 0xff, 0xb1, 0x2f, 0x12, 0xb6, 0xda, 0x02, 0x00, 0x00, + // 446 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x51, 0xd1, 0x8a, 0xd3, 0x40, + 0x14, 0xed, 0x6c, 0xb3, 0xdb, 0xf6, 0xc6, 0xae, 0xee, 0x20, 0xcb, 0x50, 0x30, 0x0d, 0x05, 0x21, + 0x08, 0xa6, 0x5a, 0xf1, 0xc5, 0xc7, 0x7d, 0x58, 0x10, 0x44, 0x24, 0x2e, 0xeb, 0x63, 0x69, 0x32, + 0xb7, 0xd9, 0xc1, 0x64, 0x26, 0x66, 0x26, 0x85, 0xfa, 0x25, 0xbe, 0xb9, 0x9f, 0xd3, 0x47, 0xbf, + 0x40, 0xb4, 0xfe, 0x88, 0x74, 0x92, 0xc6, 0x3c, 0xfa, 0x76, 0xce, 0xbd, 0xe7, 0xdc, 0x7b, 0x66, + 0x2e, 0x8c, 0xb5, 0x29, 0xab, 0xc4, 0xe8, 0xb0, 0x28, 0x95, 0x51, 0xf4, 0x84, 0xc7, 0x93, 0xe7, + 0xa9, 0x30, 0x77, 0x55, 0x1c, 0x26, 0x2a, 0x9f, 0xa7, 0x2a, 0x55, 0x73, 0xdb, 0x8a, 0xab, 0xb5, + 0x65, 0x96, 0x58, 0x54, 0x5b, 0x26, 0xaf, 0x3b, 0x72, 0xbd, 0x95, 0x89, 0xb9, 0x13, 0x32, 0xed, + 0xa0, 0x4c, 0xc4, 0xf5, 0x84, 0x44, 0x65, 0xf3, 0x18, 0x8b, 0xda, 0x36, 0xfb, 0x04, 0xee, 0xb5, + 0xc8, 0xf0, 0x16, 0x4b, 0x2d, 0x94, 0xa4, 0x2f, 0x60, 0xb0, 0xa9, 0x21, 0x23, 0x3e, 0x09, 0xdc, + 0xc5, 0xa3, 0xf0, 0x68, 0x0a, 0x6f, 0x31, 0x31, 0xaa, 0xbc, 0x72, 0x76, 0x3f, 0xa7, 0xbd, 0xe8, + 0x28, 0xa3, 0x97, 0x70, 0xc6, 0x71, 0x23, 0x12, 0x64, 0x27, 0x3e, 0x09, 0x1e, 0x44, 0x0d, 0x9b, + 0x5d, 0x83, 0xdb, 0x0c, 0x7d, 0x27, 0xb4, 0xa1, 0x2f, 0x61, 0xd8, 0x38, 0x34, 0x23, 0x7e, 0x3f, + 0x70, 0x17, 0x0f, 0x43, 0x1e, 0x87, 0x9d, 0xdd, 0xcd, 0xe0, 0x56, 0xf6, 0xc6, 0xf9, 0x76, 0x3f, + 0xed, 0xcd, 0xbe, 0xf7, 0xe1, 0xe2, 0xa0, 0x7a, 0x2b, 0xd7, 0xea, 0xa6, 0xac, 0x64, 0xb2, 0x32, + 0xc8, 0x29, 0x05, 0x47, 0xae, 0x72, 0xb4, 0x21, 0x47, 0x91, 0xc5, 0xf4, 0x19, 0x38, 0x66, 0x5b, + 0xd4, 0x39, 0xce, 0x17, 0x97, 0xff, 0x82, 0xb7, 0xf6, 0x6d, 0x81, 0x91, 0xd5, 0x1c, 0xfc, 0x5a, + 0x7c, 0x45, 0xd6, 0xf7, 0x49, 0xd0, 0x8f, 0x2c, 0xa6, 0x3e, 0xb8, 0x05, 0x96, 0xb9, 0xd0, 0x75, + 0x4a, 0xc7, 0x27, 0xc1, 0x38, 0xea, 0x96, 0xe8, 0x13, 0x80, 0x5c, 0x71, 0xb1, 0x16, 0xc8, 0x97, + 0x9a, 0x9d, 0x5a, 0xef, 0xe8, 0x58, 0xf9, 0x48, 0x19, 0x0c, 0x38, 0x66, 0x68, 0x90, 0xb3, 0x33, + 0x9f, 0x04, 0xc3, 0xe8, 0x48, 0x0f, 0x1d, 0x21, 0x37, 0xab, 0x4c, 0x70, 0x36, 0xa8, 0x3b, 0x0d, + 0xa5, 0x4f, 0xe1, 0x5c, 0xaa, 0x65, 0x77, 0xef, 0xd0, 0x0a, 0xc6, 0x52, 0x7d, 0xe8, 0x6c, 0xee, + 0xdc, 0x65, 0xf4, 0x7f, 0x77, 0x99, 0xc0, 0x50, 0xe3, 0x97, 0x0a, 0x65, 0x82, 0x0c, 0x6c, 0xd2, + 0x96, 0xd3, 0x29, 0xb8, 0xed, 0x3b, 0xa4, 0x66, 0xae, 0x4f, 0x82, 0xd3, 0xa8, 0x7d, 0xda, 0x7b, + 0x7d, 0x48, 0xa5, 0xb7, 0x79, 0x26, 0xe4, 0xe7, 0xa5, 0x59, 0x95, 0x29, 0x1a, 0x76, 0x61, 0x3f, + 0x7a, 0xdc, 0x54, 0x6f, 0x6c, 0xb1, 0xbe, 0xd0, 0xd5, 0xe3, 0xdd, 0x6f, 0xaf, 0xb7, 0xdb, 0x7b, + 0xe4, 0xc7, 0xde, 0x23, 0xbf, 0xf6, 0x5e, 0xef, 0xfe, 0x8f, 0x47, 0xe2, 0x33, 0x9b, 0xef, 0xd5, + 0xdf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xb1, 0x2f, 0x12, 0xb6, 0xda, 0x02, 0x00, 0x00, } diff --git a/lib/discover/local.pb.go b/lib/discover/local.pb.go index 1137642c9..6be791a3b 100644 --- a/lib/discover/local.pb.go +++ b/lib/discover/local.pb.go @@ -382,20 +382,21 @@ var ( ) var fileDescriptorLocal = []byte{ - // 235 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0xce, 0xc9, 0x4f, 0x4e, - 0xcc, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x48, 0xc9, 0x2c, 0x4e, 0xce, 0x2f, 0x4b, - 0x2d, 0x92, 0xd2, 0x4d, 0xcf, 0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x4f, 0xcf, - 0x4f, 0xcf, 0xd7, 0x07, 0x2b, 0x48, 0x2a, 0x4d, 0x03, 0xf3, 0xc0, 0x1c, 0x30, 0x0b, 0xa2, 0x51, - 0x69, 0x2d, 0x23, 0x17, 0x87, 0x63, 0x5e, 0x5e, 0x7e, 0x69, 0x5e, 0x72, 0xaa, 0x50, 0x10, 0x17, - 0x53, 0x66, 0x8a, 0x04, 0xa3, 0x02, 0xa3, 0x06, 0x8f, 0x93, 0xd3, 0x89, 0x7b, 0xf2, 0x0c, 0xb7, - 0xee, 0xc9, 0x9b, 0x20, 0x99, 0x57, 0x5c, 0x99, 0x97, 0x5c, 0x92, 0x91, 0x99, 0x97, 0x8e, 0xc4, - 0xca, 0xc9, 0x4c, 0x82, 0x58, 0x91, 0x9c, 0x9f, 0xa3, 0xe7, 0x92, 0x5a, 0x96, 0x99, 0x9c, 0xea, - 0xe9, 0xf2, 0xe8, 0x9e, 0x3c, 0x93, 0xa7, 0x4b, 0x10, 0xd0, 0x34, 0x21, 0x19, 0x2e, 0xce, 0xc4, - 0x94, 0x94, 0xa2, 0xd4, 0xe2, 0xe2, 0xd4, 0x62, 0x09, 0x26, 0x05, 0x66, 0x0d, 0xce, 0x20, 0x84, - 0x80, 0x90, 0x3e, 0x17, 0x77, 0x66, 0x5e, 0x71, 0x49, 0x22, 0xd0, 0xf6, 0x78, 0xa0, 0xd5, 0xcc, - 0x40, 0xab, 0x99, 0x9d, 0xf8, 0x80, 0xda, 0xb9, 0x3c, 0xa1, 0xc2, 0x40, 0x63, 0xb8, 0x60, 0x4a, - 0x3c, 0x53, 0x9c, 0x44, 0x4e, 0x3c, 0x94, 0x63, 0x38, 0xf1, 0x48, 0x8e, 0xf1, 0x02, 0x10, 0x3f, - 0x78, 0x24, 0xc7, 0xb0, 0xe0, 0xb1, 0x1c, 0x63, 0x12, 0x1b, 0xd8, 0x05, 0xc6, 0x80, 0x00, 0x00, - 0x00, 0xff, 0xff, 0xa4, 0x46, 0x4f, 0x13, 0x14, 0x01, 0x00, 0x00, + // 241 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x4c, 0x8e, 0x4f, 0x4e, 0x84, 0x30, + 0x14, 0xc6, 0x29, 0x24, 0x66, 0xa6, 0x63, 0x5c, 0x10, 0x17, 0xc4, 0x98, 0x42, 0x5c, 0xb1, 0x11, + 0x16, 0x7a, 0x01, 0x09, 0x9b, 0x6e, 0xb9, 0x80, 0x81, 0xb6, 0x32, 0x2f, 0xc1, 0x3e, 0x43, 0x61, + 0x12, 0x6f, 0xe3, 0x05, 0xbc, 0x07, 0x4b, 0xd7, 0x2e, 0x1a, 0xad, 0x17, 0x31, 0xe9, 0x68, 0x86, + 0xdd, 0xf7, 0xfd, 0xf2, 0x7b, 0x7f, 0xe8, 0x6e, 0x40, 0xd1, 0x0e, 0xc5, 0xcb, 0x88, 0x13, 0xc6, + 0x1b, 0x09, 0x46, 0xe0, 0x41, 0x8d, 0x57, 0xb7, 0x3d, 0x4c, 0xfb, 0xb9, 0x2b, 0x04, 0x3e, 0x97, + 0x3d, 0xf6, 0x58, 0x7a, 0xa1, 0x9b, 0x9f, 0x7c, 0xf3, 0xc5, 0xa7, 0xe3, 0xe0, 0xcd, 0x3b, 0xa1, + 0x9b, 0x07, 0xad, 0x71, 0xd6, 0x42, 0xc5, 0x0d, 0x0d, 0x41, 0x26, 0x24, 0x23, 0xf9, 0x79, 0x55, + 0x2d, 0x36, 0x0d, 0x3e, 0x6d, 0x7a, 0xbf, 0xda, 0x67, 0x5e, 0xb5, 0x98, 0xf6, 0xa0, 0xfb, 0x55, + 0x1a, 0xa0, 0x3b, 0x9e, 0x10, 0x38, 0x14, 0xb5, 0x3a, 0x80, 0x50, 0xbc, 0x76, 0x36, 0x0d, 0x79, + 0xdd, 0x84, 0x20, 0xe3, 0x6b, 0xba, 0x6d, 0xa5, 0x1c, 0x95, 0x31, 0xca, 0x24, 0x61, 0x16, 0xe5, + 0xdb, 0xe6, 0x04, 0xe2, 0x92, 0xee, 0x40, 0x9b, 0xa9, 0xd5, 0x42, 0x3d, 0x82, 0x4c, 0xa2, 0x8c, + 0xe4, 0x51, 0x75, 0xe1, 0x6c, 0x4a, 0xf9, 0x1f, 0xe6, 0x75, 0x43, 0xff, 0x15, 0x2e, 0xab, 0xcb, + 0xe5, 0x9b, 0x05, 0x8b, 0x63, 0xe4, 0xc3, 0x31, 0xf2, 0xe5, 0x58, 0xf0, 0xf6, 0xc3, 0x48, 0x77, + 0xe6, 0x3f, 0xb8, 0xfb, 0x0d, 0x00, 0x00, 0xff, 0xff, 0xa4, 0x46, 0x4f, 0x13, 0x14, 0x01, 0x00, + 0x00, } diff --git a/lib/model/rwfolder.go b/lib/model/rwfolder.go index 532e38159..230f2a69c 100644 --- a/lib/model/rwfolder.go +++ b/lib/model/rwfolder.go @@ -28,6 +28,7 @@ import ( "github.com/syncthing/syncthing/lib/symlinks" "github.com/syncthing/syncthing/lib/sync" "github.com/syncthing/syncthing/lib/versioner" + "github.com/syncthing/syncthing/lib/weakhash" ) func init() { @@ -92,6 +93,7 @@ type rwFolder struct { checkFreeSpace bool ignoreDelete bool fsync bool + useWeakHash bool copiers int pullers int @@ -128,6 +130,7 @@ func newRWFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Ver checkFreeSpace: cfg.MinDiskFreePct != 0, ignoreDelete: cfg.IgnoreDelete, fsync: cfg.Fsync, + useWeakHash: !cfg.DisableWeakHash, queue: newJobQueue(), pullTimer: time.NewTimer(time.Second), @@ -1169,7 +1172,7 @@ func (f *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks created: time.Now(), } - l.Debugf("%v need file %s; copy %d, reused %v", f, file.Name, len(blocks), reused) + l.Debugf("%v need file %s; copy %d, reused %v", f, file.Name, len(blocks), len(reused)) cs := copyBlocksState{ sharedPullerState: &s, @@ -1231,6 +1234,21 @@ func (f *rwFolder) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pull } f.model.fmut.RUnlock() + var weakHashFinder *weakhash.Finder + if f.useWeakHash { + hashesToFind := make([]uint32, 0, len(state.blocks)) + for _, block := range state.blocks { + if block.WeakHash != 0 { + hashesToFind = append(hashesToFind, block.WeakHash) + } + } + + weakHashFinder, err = weakhash.NewFinder(state.realName, protocol.BlockSize, hashesToFind) + if err != nil { + l.Debugln("weak hasher", err) + } + } + for _, block := range state.blocks { if f.allowSparse && state.reused == 0 && block.IsEmpty() { // The block is a block of all zeroes, and we are not reusing @@ -1245,45 +1263,70 @@ func (f *rwFolder) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pull } buf = buf[:int(block.Size)] - found := f.model.finder.Iterate(folders, block.Hash, func(folder, file string, index int32) bool { - inFile, err := rootedJoinedPath(folderRoots[folder], file) - if err != nil { - return false - } - fd, err := os.Open(inFile) - if err != nil { - return false - } - _, err = fd.ReadAt(buf, protocol.BlockSize*int64(index)) - fd.Close() - if err != nil { - return false - } - - hash, err := scanner.VerifyBuffer(buf, block) - if err != nil { - if hash != nil { - l.Debugf("Finder block mismatch in %s:%s:%d expected %q got %q", folder, file, index, block.Hash, hash) - err = f.model.finder.Fix(folder, file, index, block.Hash, hash) - if err != nil { - l.Warnln("finder fix:", err) - } - } else { - l.Debugln("Finder failed to verify buffer", err) - } - return false + found, err := weakHashFinder.Iterate(block.WeakHash, buf, func(offset int64) bool { + if _, err := scanner.VerifyBuffer(buf, block); err != nil { + return true } _, err = dstFd.WriteAt(buf, block.Offset) if err != nil { state.fail("dst write", err) + } - if file == state.file.Name { + if offset == block.Offset { state.copiedFromOrigin() + } else { + state.copiedFromOriginShifted() } - return true + + return false }) + if err != nil { + l.Debugln("weak hasher iter", err) + } + + if !found { + found = f.model.finder.Iterate(folders, block.Hash, func(folder, file string, index int32) bool { + inFile, err := rootedJoinedPath(folderRoots[folder], file) + if err != nil { + return false + } + fd, err := os.Open(inFile) + if err != nil { + return false + } + + _, err = fd.ReadAt(buf, protocol.BlockSize*int64(index)) + fd.Close() + if err != nil { + return false + } + + hash, err := scanner.VerifyBuffer(buf, block) + if err != nil { + if hash != nil { + l.Debugf("Finder block mismatch in %s:%s:%d expected %q got %q", folder, file, index, block.Hash, hash) + err = f.model.finder.Fix(folder, file, index, block.Hash, hash) + if err != nil { + l.Warnln("finder fix:", err) + } + } else { + l.Debugln("Finder failed to verify buffer", err) + } + return false + } + + _, err = dstFd.WriteAt(buf, block.Offset) + if err != nil { + state.fail("dst write", err) + } + if file == state.file.Name { + state.copiedFromOrigin() + } + return true + }) + } if state.failed() != nil { break @@ -1300,6 +1343,7 @@ func (f *rwFolder) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pull state.copyDone(block) } } + weakHashFinder.Close() out <- state.sharedPullerState } } diff --git a/lib/model/rwfolder_test.go b/lib/model/rwfolder_test.go index d560606ed..50c58fedb 100644 --- a/lib/model/rwfolder_test.go +++ b/lib/model/rwfolder_test.go @@ -7,12 +7,15 @@ package model import ( + "crypto/rand" + "io" "os" "path/filepath" "testing" "time" "github.com/syncthing/syncthing/lib/db" + "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/scanner" "github.com/syncthing/syncthing/lib/sync" @@ -72,6 +75,8 @@ func setUpRwFolder(model *Model) rwFolder { stateTracker: newStateTracker("default"), model: model, }, + + mtimeFS: fs.NewMtimeFS(db.NewNamespacedKV(model.db, "mtime")), dir: "testdata", queue: newJobQueue(), errors: make(map[string]string), @@ -199,7 +204,7 @@ func TestCopierFinder(t *testing.T) { select { case <-pullChan: - t.Fatal("Finisher channel has data to be read") + t.Fatal("Pull channel has data to be read") case <-finisherChan: t.Fatal("Finisher channel has data to be read") default: @@ -240,6 +245,132 @@ func TestCopierFinder(t *testing.T) { os.Remove(tempFile) } +func TestWeakHash(t *testing.T) { + tempFile := filepath.Join("testdata", defTempNamer.TempName("weakhash")) + var shift int64 = 10 + var size int64 = 1 << 20 + expectBlocks := int(size / protocol.BlockSize) + expectPulls := int(shift / protocol.BlockSize) + if shift > 0 { + expectPulls++ + } + + cleanup := func() { + for _, path := range []string{tempFile, "testdata/weakhash"} { + os.Remove(path) + } + } + + cleanup() + defer cleanup() + + f, err := os.Create("testdata/weakhash") + if err != nil { + t.Error(err) + } + defer f.Close() + _, err = io.CopyN(f, rand.Reader, size) + if err != nil { + t.Error(err) + } + info, err := f.Stat() + if err != nil { + t.Error(err) + } + + // Create two files, second file has `shifted` bytes random prefix, yet + // both are of the same length, for example: + // File 1: abcdefgh + // File 2: xyabcdef + f.Seek(0, os.SEEK_SET) + existing, err := scanner.Blocks(f, protocol.BlockSize, size, nil) + if err != nil { + t.Error(err) + } + + f.Seek(0, os.SEEK_SET) + remainder := io.LimitReader(f, size-shift) + prefix := io.LimitReader(rand.Reader, shift) + nf := io.MultiReader(prefix, remainder) + desired, err := scanner.Blocks(nf, protocol.BlockSize, size, nil) + if err != nil { + t.Error(err) + } + + existingFile := protocol.FileInfo{ + Name: "weakhash", + Blocks: existing, + Size: size, + ModifiedS: info.ModTime().Unix(), + ModifiedNs: int32(info.ModTime().Nanosecond()), + } + desiredFile := protocol.FileInfo{ + Name: "weakhash", + Size: size, + Blocks: desired, + ModifiedS: info.ModTime().Unix() + 1, + } + + // Setup the model/pull environment + m := setUpModel(existingFile) + fo := setUpRwFolder(m) + copyChan := make(chan copyBlocksState) + pullChan := make(chan pullBlockState, expectBlocks) + finisherChan := make(chan *sharedPullerState, 1) + + // Run a single fetcher routine + go fo.copierRoutine(copyChan, pullChan, finisherChan) + + // Test 1 - no weak hashing, file gets fully repulled (`expectBlocks` pulls). + fo.handleFile(desiredFile, copyChan, finisherChan) + + var pulls []pullBlockState + for len(pulls) < expectBlocks { + select { + case pull := <-pullChan: + pulls = append(pulls, pull) + case <-time.After(time.Second): + t.Error("timed out") + } + } + finish := <-finisherChan + + select { + case <-pullChan: + t.Fatal("Pull channel has data to be read") + case <-finisherChan: + t.Fatal("Finisher channel has data to be read") + default: + } + + finish.fd.Close() + if err := os.Remove(tempFile); err != nil && !os.IsNotExist(err) { + t.Error(err) + } + + // Test 2 - using weak hash, expectPulls blocks pulled. + fo.useWeakHash = true + fo.handleFile(desiredFile, copyChan, finisherChan) + + pulls = pulls[:0] + for len(pulls) < expectPulls { + select { + case pull := <-pullChan: + pulls = append(pulls, pull) + case <-time.After(time.Second): + t.Error("timed out") + } + } + + finish = <-finisherChan + finish.fd.Close() + + expectShifted := expectBlocks - expectPulls + if finish.copyOriginShifted != expectShifted { + t.Errorf("did not copy %d shifted", expectShifted) + } +} + // Test that updating a file removes it's old blocks from the blockmap func TestCopierCleanup(t *testing.T) { iterFn := func(folder, file string, index int32) bool { diff --git a/lib/model/sharedpullerstate.go b/lib/model/sharedpullerstate.go index 35692b406..9dab8b2d1 100644 --- a/lib/model/sharedpullerstate.go +++ b/lib/model/sharedpullerstate.go @@ -31,30 +31,32 @@ type sharedPullerState struct { created time.Time // Mutable, must be locked for access - err error // The first error we hit - fd *os.File // The fd of the temp file - copyTotal int // Total number of copy actions for the whole job - pullTotal int // Total number of pull actions for the whole job - copyOrigin int // Number of blocks copied from the original file - copyNeeded int // Number of copy actions still pending - pullNeeded int // Number of block pulls still pending - updated time.Time // Time when any of the counters above were last updated - closed bool // True if the file has been finalClosed. - available []int32 // Indexes of the blocks that are available in the temporary file - availableUpdated time.Time // Time when list of available blocks was last updated - mut sync.RWMutex // Protects the above + err error // The first error we hit + fd *os.File // The fd of the temp file + copyTotal int // Total number of copy actions for the whole job + pullTotal int // Total number of pull actions for the whole job + copyOrigin int // Number of blocks copied from the original file + copyOriginShifted int // Number of blocks copied from the original file but shifted + copyNeeded int // Number of copy actions still pending + pullNeeded int // Number of block pulls still pending + updated time.Time // Time when any of the counters above were last updated + closed bool // True if the file has been finalClosed. + available []int32 // Indexes of the blocks that are available in the temporary file + availableUpdated time.Time // Time when list of available blocks was last updated + mut sync.RWMutex // Protects the above } // A momentary state representing the progress of the puller type pullerProgress struct { - Total int `json:"total"` - Reused int `json:"reused"` - CopiedFromOrigin int `json:"copiedFromOrigin"` - CopiedFromElsewhere int `json:"copiedFromElsewhere"` - Pulled int `json:"pulled"` - Pulling int `json:"pulling"` - BytesDone int64 `json:"bytesDone"` - BytesTotal int64 `json:"bytesTotal"` + Total int `json:"total"` + Reused int `json:"reused"` + CopiedFromOrigin int `json:"copiedFromOrigin"` + CopiedFromOriginShifted int `json:"copiedFromOriginShifted"` + CopiedFromElsewhere int `json:"copiedFromElsewhere"` + Pulled int `json:"pulled"` + Pulling int `json:"pulling"` + BytesDone int64 `json:"bytesDone"` + BytesTotal int64 `json:"bytesTotal"` } // A lockedWriterAt synchronizes WriteAt calls with an external mutex. @@ -241,6 +243,14 @@ func (s *sharedPullerState) copiedFromOrigin() { s.mut.Unlock() } +func (s *sharedPullerState) copiedFromOriginShifted() { + s.mut.Lock() + s.copyOrigin++ + s.copyOriginShifted++ + s.updated = time.Now() + s.mut.Unlock() +} + func (s *sharedPullerState) pullStarted() { s.mut.Lock() s.copyTotal-- diff --git a/lib/protocol/bep.pb.go b/lib/protocol/bep.pb.go index d533efa42..494fa1696 100644 --- a/lib/protocol/bep.pb.go +++ b/lib/protocol/bep.pb.go @@ -303,7 +303,7 @@ type FileInfo struct { NoPermissions bool `protobuf:"varint,8,opt,name=no_permissions,json=noPermissions,proto3" json:"no_permissions,omitempty"` Version Vector `protobuf:"bytes,9,opt,name=version" json:"version"` Sequence int64 `protobuf:"varint,10,opt,name=sequence,proto3" json:"sequence,omitempty"` - Blocks []BlockInfo `protobuf:"bytes,16,rep,name=Blocks" json:"Blocks"` + Blocks []BlockInfo `protobuf:"bytes,16,rep,name=Blocks,json=blocks" json:"Blocks"` SymlinkTarget string `protobuf:"bytes,17,opt,name=symlink_target,json=symlinkTarget,proto3" json:"symlink_target,omitempty"` } @@ -312,9 +312,10 @@ func (*FileInfo) ProtoMessage() {} func (*FileInfo) Descriptor() ([]byte, []int) { return fileDescriptorBep, []int{7} } type BlockInfo struct { - Offset int64 `protobuf:"varint,1,opt,name=offset,proto3" json:"offset,omitempty"` - Size int32 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` - Hash []byte `protobuf:"bytes,3,opt,name=hash,proto3" json:"hash,omitempty"` + Offset int64 `protobuf:"varint,1,opt,name=offset,proto3" json:"offset,omitempty"` + Size int32 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` + Hash []byte `protobuf:"bytes,3,opt,name=hash,proto3" json:"hash,omitempty"` + WeakHash uint32 `protobuf:"varint,4,opt,name=weak_hash,json=weakHash,proto3" json:"weak_hash,omitempty"` } func (m *BlockInfo) Reset() { *m = BlockInfo{} } @@ -913,6 +914,11 @@ func (m *BlockInfo) MarshalTo(data []byte) (int, error) { i = encodeVarintBep(data, i, uint64(len(m.Hash))) i += copy(data[i:], m.Hash) } + if m.WeakHash != 0 { + data[i] = 0x20 + i++ + i = encodeVarintBep(data, i, uint64(m.WeakHash)) + } return i, nil } @@ -1423,6 +1429,9 @@ func (m *BlockInfo) ProtoSize() (n int) { if l > 0 { n += 1 + l + sovBep(uint64(l)) } + if m.WeakHash != 0 { + n += 1 + sovBep(uint64(m.WeakHash)) + } return n } @@ -3011,6 +3020,25 @@ func (m *BlockInfo) Unmarshal(data []byte) error { m.Hash = []byte{} } iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field WeakHash", wireType) + } + m.WeakHash = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBep + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + m.WeakHash |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipBep(data[iNdEx:]) @@ -4029,108 +4057,112 @@ var ( ) var fileDescriptorBep = []byte{ - // 1645 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x57, 0x4f, 0x73, 0xdb, 0x5a, - 0x15, 0x8f, 0x6d, 0xf9, 0xdf, 0xb5, 0x93, 0xe7, 0xdc, 0xa6, 0x79, 0x46, 0xcd, 0x4b, 0x82, 0xde, - 0x2b, 0x04, 0x0f, 0x4d, 0xa1, 0x05, 0x3a, 0xc3, 0x00, 0x33, 0x8e, 0xad, 0xa4, 0x9a, 0x3a, 0xb2, - 0x2b, 0xdb, 0x29, 0x65, 0x81, 0x46, 0xb6, 0xae, 0x1d, 0x4d, 0x64, 0x5d, 0x23, 0xc9, 0x6d, 0xc3, - 0x47, 0x80, 0x2f, 0xc0, 0x86, 0x99, 0x6e, 0xd9, 0xf3, 0x21, 0xca, 0xae, 0xd3, 0x25, 0x8b, 0x0e, - 0x94, 0x0d, 0x4b, 0x36, 0xec, 0x39, 0xf7, 0x5e, 0x49, 0x96, 0xf3, 0x87, 0xe9, 0xe2, 0x2d, 0x32, - 0xd6, 0x3d, 0xe7, 0x77, 0xcf, 0xb9, 0xe7, 0x77, 0xfe, 0xdc, 0x1b, 0x54, 0x1e, 0x91, 0xf9, 0xe1, - 0xdc, 0xa7, 0x21, 0xc5, 0x25, 0xfe, 0x33, 0xa6, 0xae, 0xfc, 0x60, 0xea, 0x84, 0xe7, 0x8b, 0xd1, - 0xe1, 0x98, 0xce, 0x1e, 0x4e, 0xe9, 0x94, 0x3e, 0xe4, 0x9a, 0xd1, 0x62, 0xc2, 0x57, 0x7c, 0xc1, - 0xbf, 0xc4, 0x46, 0x65, 0x8e, 0xf2, 0x4f, 0x89, 0xeb, 0x52, 0xbc, 0x87, 0x2a, 0x36, 0x79, 0xe5, - 0x8c, 0x89, 0xe9, 0x59, 0x33, 0x52, 0xcf, 0xec, 0x67, 0x0e, 0xca, 0x06, 0x12, 0x22, 0x1d, 0x24, - 0x0c, 0x30, 0x76, 0x1d, 0xe2, 0x85, 0x02, 0x90, 0x15, 0x00, 0x21, 0xe2, 0x80, 0xfb, 0x68, 0x23, - 0x02, 0xbc, 0x22, 0x7e, 0xe0, 0x50, 0xaf, 0x9e, 0xe3, 0x98, 0x75, 0x21, 0x3d, 0x13, 0x42, 0x25, - 0x40, 0x85, 0xa7, 0xc4, 0xb2, 0x89, 0x8f, 0x7f, 0x80, 0xa4, 0xf0, 0x72, 0x2e, 0x7c, 0x6d, 0x3c, - 0xba, 0x7b, 0x18, 0xc7, 0x70, 0x78, 0x4a, 0x82, 0xc0, 0x9a, 0x92, 0x01, 0x28, 0x0d, 0x0e, 0xc1, - 0xbf, 0x02, 0xe7, 0x74, 0x36, 0xf7, 0x41, 0xc1, 0x0c, 0x67, 0xf9, 0x8e, 0x9d, 0x6b, 0x3b, 0x5a, - 0x4b, 0x8c, 0x91, 0xde, 0xa0, 0x34, 0xd1, 0x7a, 0xcb, 0x5d, 0x04, 0x21, 0xf1, 0x5b, 0xd4, 0x9b, - 0x38, 0x53, 0xfc, 0x23, 0x54, 0x9c, 0x50, 0x17, 0x4e, 0x11, 0x80, 0xfb, 0xdc, 0x41, 0xe5, 0x51, - 0x6d, 0x69, 0xec, 0x98, 0x2b, 0x8e, 0xa4, 0x77, 0x1f, 0xf7, 0xd6, 0x8c, 0x18, 0xa6, 0xfc, 0x31, - 0x8b, 0x0a, 0x42, 0x83, 0xb7, 0x51, 0xd6, 0xb1, 0x05, 0x45, 0x47, 0x85, 0x4f, 0x1f, 0xf7, 0xb2, - 0x5a, 0xdb, 0x00, 0x09, 0xde, 0x42, 0x79, 0xd7, 0x1a, 0x11, 0x37, 0x22, 0x47, 0x2c, 0xf0, 0x3d, - 0x54, 0xf6, 0x21, 0x60, 0x93, 0x7a, 0xee, 0x25, 0xa7, 0xa4, 0x64, 0x94, 0x98, 0xa0, 0x0b, 0x6b, - 0xfc, 0x00, 0x61, 0x67, 0xea, 0x51, 0x9f, 0x98, 0x73, 0xe2, 0xcf, 0x1c, 0x7e, 0xda, 0xa0, 0x2e, - 0x71, 0xd4, 0xa6, 0xd0, 0xf4, 0x96, 0x0a, 0xfc, 0x35, 0x5a, 0x8f, 0xe0, 0x36, 0x71, 0x49, 0x48, - 0xea, 0x79, 0x8e, 0xac, 0x0a, 0x61, 0x9b, 0xcb, 0x20, 0xb6, 0x2d, 0xdb, 0x09, 0xac, 0x91, 0x4b, - 0xcc, 0x90, 0xcc, 0xe6, 0xa6, 0xe3, 0xd9, 0xe4, 0x0d, 0x09, 0xea, 0x05, 0x8e, 0xc5, 0x91, 0x6e, - 0x00, 0x2a, 0x4d, 0x68, 0x18, 0x1b, 0x22, 0xd3, 0x41, 0xbd, 0x76, 0x95, 0x8d, 0x36, 0x57, 0xc4, - 0x6c, 0x44, 0x30, 0xe5, 0x3f, 0xc0, 0x86, 0xd0, 0xe0, 0xef, 0x25, 0x6c, 0x54, 0x8f, 0xb6, 0x19, - 0xea, 0xef, 0x1f, 0xf7, 0x4a, 0x42, 0xa7, 0xb5, 0x53, 0xec, 0x60, 0x24, 0xa5, 0x2a, 0x87, 0x7f, - 0xe3, 0x1d, 0x54, 0xb6, 0x6c, 0x9b, 0x65, 0x09, 0x5c, 0xe7, 0xc0, 0x75, 0xd9, 0x58, 0x0a, 0xf0, - 0x93, 0xd5, 0xac, 0x4b, 0x57, 0xeb, 0xe4, 0xb6, 0x74, 0x33, 0xca, 0xc7, 0xc4, 0x8f, 0x2a, 0x35, - 0xcf, 0xfd, 0x95, 0x98, 0x80, 0xd7, 0xe9, 0x77, 0x51, 0x75, 0x66, 0xbd, 0x31, 0x03, 0xf2, 0xbb, - 0x05, 0xf1, 0xc6, 0x84, 0xd3, 0x92, 0x33, 0x2a, 0x20, 0xeb, 0x47, 0x22, 0xbc, 0x8b, 0x90, 0xe3, - 0x85, 0x3e, 0xb5, 0x17, 0xb0, 0xab, 0x5e, 0xe4, 0xbc, 0xa5, 0x24, 0xf8, 0xa7, 0xa8, 0xc4, 0x49, - 0x35, 0x21, 0xf0, 0x12, 0x68, 0xa5, 0x23, 0x39, 0x0a, 0xbc, 0xc8, 0x29, 0xe5, 0x71, 0xc7, 0x9f, - 0x46, 0x91, 0x63, 0x35, 0x1b, 0xff, 0x02, 0xc9, 0xc1, 0x85, 0xc3, 0x12, 0x22, 0x2c, 0x85, 0x70, - 0x56, 0xd3, 0x27, 0x33, 0xfa, 0xca, 0x72, 0x83, 0x7a, 0x99, 0xbb, 0xa9, 0x33, 0x84, 0x96, 0x02, - 0x18, 0x91, 0x5e, 0xe9, 0xa2, 0x3c, 0xb7, 0x08, 0xe5, 0x57, 0x10, 0x45, 0x19, 0x75, 0x69, 0xb4, - 0xc2, 0x87, 0x28, 0x3f, 0x71, 0x5c, 0x20, 0x32, 0xcb, 0x73, 0x88, 0x53, 0x15, 0x0d, 0x62, 0xcd, - 0x9b, 0xd0, 0x28, 0x8b, 0x02, 0xa6, 0x0c, 0x51, 0x85, 0x1b, 0x1c, 0xce, 0x6d, 0x0b, 0xca, 0xe6, - 0xdb, 0x32, 0xfb, 0xb7, 0x1c, 0x2a, 0xc5, 0x9a, 0x24, 0xe9, 0x99, 0x54, 0xd2, 0x1b, 0x51, 0xdf, - 0x8b, 0x2e, 0xde, 0xbe, 0x6e, 0x2f, 0xd5, 0xf8, 0xb0, 0x3f, 0x70, 0x7e, 0x4f, 0x78, 0xdf, 0xe4, - 0x0c, 0xfe, 0x8d, 0xf7, 0x51, 0xe5, 0x6a, 0xb3, 0xac, 0x1b, 0x69, 0x11, 0xfe, 0x0a, 0xa1, 0x19, - 0xb5, 0x9d, 0x89, 0x43, 0x6c, 0x33, 0xe0, 0x05, 0x90, 0x33, 0xca, 0xb1, 0xa4, 0x8f, 0xeb, 0xac, - 0xdc, 0x59, 0xab, 0xd8, 0x51, 0x4f, 0xc4, 0x4b, 0xa6, 0x71, 0x3c, 0x60, 0x1b, 0xf2, 0x2a, 0xb2, - 0x1e, 0x2f, 0xd9, 0x74, 0xf3, 0xe8, 0x4a, 0x93, 0x96, 0x38, 0x60, 0xdd, 0xa3, 0xe9, 0x06, 0x85, - 0x4e, 0x8a, 0xa7, 0x1f, 0xcb, 0xe7, 0x4a, 0x27, 0x9d, 0x91, 0x71, 0x48, 0x93, 0xb9, 0x12, 0xc1, - 0xb0, 0x8c, 0x4a, 0x49, 0x29, 0x22, 0x7e, 0xd2, 0x64, 0xcd, 0x66, 0x6e, 0x12, 0x07, 0x78, 0xac, - 0x80, 0x3a, 0x6f, 0x24, 0xa1, 0xe9, 0x01, 0xfe, 0x31, 0x2a, 0x1c, 0xb9, 0x74, 0x7c, 0x11, 0xf7, - 0xed, 0x9d, 0xa5, 0x37, 0x2e, 0x4f, 0x65, 0x27, 0x02, 0xb2, 0x40, 0x82, 0xcb, 0x99, 0xeb, 0x78, - 0x17, 0x66, 0x68, 0xf9, 0x53, 0x12, 0xd6, 0x37, 0xc5, 0x98, 0x8e, 0xa4, 0x03, 0x2e, 0xfc, 0xb9, - 0xf4, 0xa7, 0xb7, 0x7b, 0x6b, 0xca, 0x73, 0x54, 0x4e, 0xec, 0xb0, 0x02, 0xa1, 0x93, 0x49, 0x00, - 0x3b, 0x32, 0xfc, 0x9c, 0xd1, 0x2a, 0xc9, 0x51, 0x96, 0x1f, 0x4f, 0xe4, 0x08, 0x64, 0xe7, 0x56, - 0x70, 0xce, 0xf3, 0x56, 0x35, 0xf8, 0x77, 0x64, 0xf2, 0x97, 0xa8, 0x20, 0x88, 0xc0, 0x8f, 0x51, - 0x69, 0x4c, 0x17, 0x5e, 0xb8, 0x1c, 0xc2, 0x9b, 0xe9, 0xde, 0xe6, 0x9a, 0xe8, 0xf0, 0x09, 0x50, - 0x39, 0x46, 0xc5, 0x48, 0x05, 0x91, 0xc4, 0x83, 0x47, 0x3a, 0xba, 0x1b, 0xf7, 0x5f, 0xff, 0x9c, - 0xfa, 0xe1, 0xca, 0xdc, 0x81, 0xa9, 0x0c, 0x29, 0x5c, 0x88, 0xf3, 0x49, 0x86, 0x58, 0x28, 0x7f, - 0xcd, 0xa0, 0xa2, 0xc1, 0x78, 0x0e, 0xc2, 0xd4, 0x3c, 0xcf, 0xaf, 0xcc, 0xf3, 0x65, 0x47, 0x64, - 0x57, 0x3a, 0x22, 0x2e, 0xea, 0x5c, 0xaa, 0xa8, 0x97, 0xe4, 0x48, 0x37, 0x92, 0x93, 0xbf, 0x81, - 0x9c, 0xc2, 0x92, 0x1c, 0x96, 0x96, 0x89, 0x4f, 0x67, 0x7c, 0x62, 0x53, 0xdf, 0xf2, 0x2f, 0xa3, - 0x02, 0x5c, 0x67, 0xd2, 0x41, 0x2c, 0x54, 0x4c, 0x54, 0x32, 0x48, 0x30, 0x87, 0x52, 0x23, 0xb7, - 0x1e, 0x1b, 0xcc, 0x43, 0x43, 0x5b, 0xfc, 0xd0, 0x60, 0x9e, 0x7d, 0xe3, 0xef, 0x23, 0x69, 0x4c, - 0x6d, 0x71, 0xe4, 0x8d, 0x74, 0x99, 0xa8, 0xbe, 0x4f, 0xe1, 0x52, 0xb4, 0xa1, 0xe1, 0x18, 0x00, - 0x1e, 0x04, 0xb5, 0x36, 0x7d, 0xed, 0xb9, 0xd4, 0xb2, 0x7b, 0x3e, 0x9d, 0xb2, 0x89, 0x7a, 0xeb, - 0x64, 0x68, 0xa3, 0xe2, 0x82, 0xcf, 0x8e, 0x78, 0x36, 0x7c, 0xb3, 0xda, 0xcb, 0x57, 0x0d, 0x89, - 0x41, 0x13, 0x37, 0x40, 0xb4, 0x55, 0xf9, 0x90, 0x41, 0xf2, 0xed, 0x68, 0xac, 0xa1, 0x8a, 0x40, - 0x9a, 0xa9, 0xc7, 0xc2, 0xc1, 0xe7, 0x38, 0xe2, 0x63, 0x04, 0x2d, 0x92, 0xef, 0x1b, 0x6f, 0xa0, - 0x54, 0xc3, 0xe6, 0x3e, 0xaf, 0x61, 0xe1, 0x0e, 0x1e, 0xb1, 0x9e, 0x48, 0xee, 0x55, 0x09, 0x62, - 0xcf, 0x1b, 0xd5, 0x91, 0x68, 0x14, 0x2e, 0x53, 0x0a, 0x48, 0xea, 0x39, 0xde, 0x54, 0xd9, 0x43, - 0xf9, 0x96, 0x4b, 0x79, 0xb2, 0x0a, 0x70, 0xe9, 0x07, 0xe0, 0x26, 0xe2, 0x50, 0xac, 0x1a, 0x1f, - 0xb2, 0xa8, 0x92, 0x7a, 0xef, 0xc0, 0x79, 0x36, 0x5a, 0x9d, 0x61, 0x7f, 0xa0, 0x1a, 0x66, 0xab, - 0xab, 0x1f, 0x6b, 0x27, 0xb5, 0x35, 0x79, 0xe7, 0x0f, 0x7f, 0xde, 0xaf, 0xcf, 0x96, 0xa0, 0xd5, - 0xa7, 0x0c, 0xb8, 0xd0, 0xf4, 0xb6, 0xfa, 0xeb, 0x5a, 0x46, 0xde, 0x02, 0x60, 0x2d, 0x05, 0x14, - 0xf7, 0xc5, 0x0f, 0x51, 0x95, 0x03, 0xcc, 0x61, 0xaf, 0xdd, 0x1c, 0xa8, 0xb5, 0xac, 0x2c, 0x03, - 0x6e, 0xfb, 0x2a, 0x2e, 0xe2, 0xfb, 0x6b, 0xe8, 0x0b, 0xf5, 0xf9, 0x50, 0xed, 0x0f, 0x6a, 0x39, - 0x79, 0x1b, 0x80, 0x38, 0x05, 0x8c, 0x3b, 0xe6, 0x3e, 0x94, 0xa1, 0xda, 0xef, 0x75, 0xf5, 0xbe, - 0x5a, 0x93, 0xe4, 0x2f, 0x01, 0x75, 0x67, 0x05, 0x15, 0x55, 0xe8, 0xcf, 0xd0, 0x66, 0xbb, 0xfb, - 0x42, 0xef, 0x74, 0x9b, 0x6d, 0xb3, 0x67, 0x74, 0x4f, 0x60, 0x4f, 0xbf, 0x96, 0x97, 0xf7, 0x00, - 0x7f, 0x2f, 0x85, 0xbf, 0x56, 0x70, 0x5f, 0x01, 0x7b, 0x9a, 0x7e, 0x52, 0x2b, 0xc8, 0x77, 0x00, - 0xfa, 0x45, 0x0a, 0xca, 0x48, 0x65, 0x11, 0xb7, 0x3a, 0x5d, 0x70, 0x5d, 0xbc, 0x16, 0x31, 0x27, - 0xbb, 0xf1, 0x5b, 0x84, 0xaf, 0xbf, 0x08, 0xf1, 0x37, 0x48, 0xd2, 0xbb, 0xba, 0x0a, 0x84, 0xf2, - 0xf8, 0xaf, 0x23, 0x74, 0xea, 0x11, 0xac, 0xa0, 0x5c, 0xe7, 0x37, 0x3f, 0x01, 0x32, 0xbf, 0x03, - 0xa0, 0xbb, 0xd7, 0x41, 0xa0, 0x6c, 0x50, 0x54, 0x49, 0x1b, 0x56, 0x50, 0xe9, 0x54, 0x1d, 0x34, - 0x81, 0xdc, 0x26, 0x18, 0xe7, 0x47, 0x8a, 0xd5, 0xa7, 0x24, 0xb4, 0x78, 0x03, 0xee, 0xa0, 0xbc, - 0xae, 0x9e, 0xa9, 0x06, 0x18, 0xde, 0x04, 0xc0, 0x7a, 0x0c, 0xd0, 0x09, 0xd4, 0x15, 0x3c, 0x38, - 0x0a, 0xcd, 0xce, 0x8b, 0xe6, 0xcb, 0x3e, 0x24, 0x07, 0x83, 0x7a, 0x23, 0x56, 0x37, 0xdd, 0xd7, - 0xd6, 0x65, 0xd0, 0xf8, 0x6f, 0x06, 0x55, 0xd3, 0xb7, 0x23, 0x6c, 0x90, 0x8e, 0xb5, 0x8e, 0x1a, - 0xbb, 0x4b, 0xeb, 0xd8, 0x37, 0x3e, 0x40, 0xe5, 0xb6, 0x66, 0xa8, 0xad, 0x41, 0xd7, 0x78, 0x19, - 0xc7, 0x92, 0x06, 0xb5, 0x1d, 0x9f, 0x17, 0x37, 0x7b, 0x81, 0x56, 0xfb, 0x2f, 0x4f, 0x3b, 0x9a, - 0xfe, 0xcc, 0xe4, 0x16, 0xb3, 0xf2, 0x3d, 0x00, 0x7f, 0x99, 0x06, 0xf7, 0xc5, 0xcd, 0xc0, 0x0d, - 0x3f, 0x41, 0x9b, 0x31, 0x7c, 0xe9, 0x20, 0x27, 0xef, 0xc3, 0x9e, 0x9d, 0x1b, 0xf6, 0x2c, 0xfd, - 0x3c, 0x46, 0x5f, 0xc4, 0x1b, 0x87, 0xfa, 0x33, 0x1d, 0xca, 0x02, 0x2a, 0x67, 0x17, 0xb6, 0xc9, - 0x37, 0x6c, 0x1b, 0x7a, 0x17, 0x1e, 0x14, 0x45, 0xe3, 0x2f, 0x19, 0x54, 0x4e, 0x26, 0x14, 0xe3, - 0x59, 0xef, 0x9a, 0xaa, 0x61, 0x74, 0x8d, 0x38, 0xf0, 0x44, 0xa9, 0x53, 0xfe, 0x09, 0xaf, 0xbb, - 0xe2, 0x89, 0xaa, 0xab, 0x86, 0xd6, 0x8a, 0xfb, 0x21, 0x81, 0x9c, 0x10, 0x8f, 0xf8, 0xce, 0x18, - 0xfe, 0xef, 0xa8, 0x82, 0x99, 0xfe, 0xb0, 0xf5, 0x34, 0x8e, 0x98, 0x17, 0x70, 0xca, 0x54, 0x7f, - 0x31, 0x3e, 0xe7, 0xd1, 0x36, 0x58, 0xeb, 0x9c, 0x35, 0x3b, 0x5a, 0x5b, 0x40, 0x73, 0x72, 0x1d, - 0xa0, 0x5b, 0x09, 0x54, 0x13, 0xaf, 0x03, 0x86, 0x6d, 0xd8, 0x68, 0xf7, 0xff, 0xcf, 0x22, 0x78, - 0xb8, 0x14, 0x9a, 0xbd, 0x9e, 0xaa, 0xb7, 0xe3, 0xd3, 0x2f, 0x75, 0xcd, 0xf9, 0x9c, 0x78, 0x36, - 0x43, 0x1c, 0x77, 0x8d, 0x13, 0x75, 0x10, 0x1f, 0x7e, 0x89, 0x38, 0xa6, 0xec, 0x5e, 0x3e, 0xda, - 0x79, 0xf7, 0xcf, 0xdd, 0xb5, 0xf7, 0xf0, 0xf7, 0xee, 0xd3, 0x6e, 0xe6, 0x3d, 0xfc, 0xfd, 0xe3, - 0xd3, 0xee, 0xda, 0xbf, 0xe1, 0xf7, 0xed, 0xbf, 0x76, 0x33, 0xa3, 0x02, 0x9f, 0x5d, 0x8f, 0xff, - 0x17, 0x00, 0x00, 0xff, 0xff, 0x20, 0x74, 0xd7, 0x8f, 0x1b, 0x0e, 0x00, 0x00, + // 1700 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x56, 0x4f, 0x73, 0xdb, 0xc6, + 0x15, 0x17, 0x48, 0xf0, 0xdf, 0x23, 0xa5, 0x40, 0x6b, 0x59, 0x41, 0x61, 0x85, 0x42, 0xe0, 0xb8, + 0x55, 0x34, 0x8d, 0xe2, 0xc6, 0x69, 0x33, 0xd3, 0x69, 0x3b, 0x43, 0x91, 0x90, 0xcc, 0x09, 0x0d, + 0xb2, 0x4b, 0xca, 0xae, 0x73, 0x28, 0x06, 0x24, 0x96, 0x14, 0x46, 0x20, 0x96, 0x05, 0x40, 0xd9, + 0xea, 0x47, 0x60, 0xbf, 0x40, 0x2f, 0x9c, 0xc9, 0xb5, 0xf7, 0x7e, 0x08, 0xf7, 0x96, 0xc9, 0xb1, + 0x07, 0x4f, 0xa3, 0x5e, 0x7a, 0xec, 0xa5, 0xf7, 0x0e, 0x76, 0x01, 0x10, 0x94, 0xe4, 0x8e, 0x0f, + 0x3d, 0x71, 0xf7, 0xbd, 0xdf, 0xbe, 0xdd, 0xf7, 0x7b, 0xef, 0xf7, 0x08, 0xa8, 0x0c, 0xc9, 0xec, + 0x68, 0xe6, 0xd3, 0x90, 0xa2, 0x32, 0xfb, 0x19, 0x51, 0x57, 0xf9, 0x6c, 0xe2, 0x84, 0xe7, 0xf3, + 0xe1, 0xd1, 0x88, 0x4e, 0x3f, 0x9f, 0xd0, 0x09, 0xfd, 0x9c, 0x79, 0x86, 0xf3, 0x31, 0xdb, 0xb1, + 0x0d, 0x5b, 0xf1, 0x83, 0xda, 0x0c, 0x0a, 0x4f, 0x89, 0xeb, 0x52, 0xb4, 0x0f, 0x55, 0x9b, 0x5c, + 0x3a, 0x23, 0x62, 0x7a, 0xd6, 0x94, 0xc8, 0x82, 0x2a, 0x1c, 0x54, 0x30, 0x70, 0x93, 0x61, 0x4d, + 0x49, 0x04, 0x18, 0xb9, 0x0e, 0xf1, 0x42, 0x0e, 0xc8, 0x71, 0x00, 0x37, 0x31, 0xc0, 0x23, 0xd8, + 0x8a, 0x01, 0x97, 0xc4, 0x0f, 0x1c, 0xea, 0xc9, 0x79, 0x86, 0xd9, 0xe4, 0xd6, 0xe7, 0xdc, 0xa8, + 0x05, 0x50, 0x7c, 0x4a, 0x2c, 0x9b, 0xf8, 0xe8, 0x53, 0x10, 0xc3, 0xab, 0x19, 0xbf, 0x6b, 0xeb, + 0x8b, 0xfb, 0x47, 0x49, 0x0e, 0x47, 0xcf, 0x48, 0x10, 0x58, 0x13, 0x32, 0xb8, 0x9a, 0x11, 0xcc, + 0x20, 0xe8, 0x37, 0x50, 0x1d, 0xd1, 0xe9, 0xcc, 0x27, 0x01, 0x0b, 0x9c, 0x63, 0x27, 0xf6, 0x6e, + 0x9d, 0x68, 0xae, 0x30, 0x38, 0x7b, 0x40, 0x6b, 0xc0, 0x66, 0xd3, 0x9d, 0x07, 0x21, 0xf1, 0x9b, + 0xd4, 0x1b, 0x3b, 0x13, 0xf4, 0x18, 0x4a, 0x63, 0xea, 0xda, 0xc4, 0x0f, 0x64, 0x41, 0xcd, 0x1f, + 0x54, 0xbf, 0x90, 0x56, 0xc1, 0x4e, 0x98, 0xe3, 0x58, 0x7c, 0xf3, 0x76, 0x7f, 0x03, 0x27, 0x30, + 0xed, 0x4f, 0x39, 0x28, 0x72, 0x0f, 0xda, 0x85, 0x9c, 0x63, 0x73, 0x8a, 0x8e, 0x8b, 0xd7, 0x6f, + 0xf7, 0x73, 0xed, 0x16, 0xce, 0x39, 0x36, 0xda, 0x81, 0x82, 0x6b, 0x0d, 0x89, 0x1b, 0x93, 0xc3, + 0x37, 0xe8, 0x01, 0x54, 0x7c, 0x62, 0xd9, 0x26, 0xf5, 0xdc, 0x2b, 0x46, 0x49, 0x19, 0x97, 0x23, + 0x43, 0xd7, 0x73, 0xaf, 0xd0, 0x67, 0x80, 0x9c, 0x89, 0x47, 0x7d, 0x62, 0xce, 0x88, 0x3f, 0x75, + 0xd8, 0x6b, 0x03, 0x59, 0x64, 0xa8, 0x6d, 0xee, 0xe9, 0xad, 0x1c, 0xe8, 0x21, 0x6c, 0xc6, 0x70, + 0x9b, 0xb8, 0x24, 0x24, 0x72, 0x81, 0x21, 0x6b, 0xdc, 0xd8, 0x62, 0x36, 0xf4, 0x18, 0x76, 0x6c, + 0x27, 0xb0, 0x86, 0x2e, 0x31, 0x43, 0x32, 0x9d, 0x99, 0x8e, 0x67, 0x93, 0xd7, 0x24, 0x90, 0x8b, + 0x0c, 0x8b, 0x62, 0xdf, 0x80, 0x4c, 0x67, 0x6d, 0xee, 0x89, 0xd8, 0xe0, 0x95, 0x0e, 0x64, 0xe9, + 0x26, 0x1b, 0x2d, 0xe6, 0x48, 0xd8, 0x88, 0x61, 0xda, 0xbf, 0x73, 0x50, 0xe4, 0x1e, 0xf4, 0xe3, + 0x94, 0x8d, 0xda, 0xf1, 0x6e, 0x84, 0xfa, 0xfb, 0xdb, 0xfd, 0x32, 0xf7, 0xb5, 0x5b, 0x19, 0x76, + 0x10, 0x88, 0x99, 0xce, 0x61, 0x6b, 0xb4, 0x07, 0x15, 0xcb, 0xb6, 0xa3, 0x2a, 0x91, 0x40, 0xce, + 0xab, 0xf9, 0x83, 0x0a, 0x5e, 0x19, 0xd0, 0x57, 0xeb, 0x55, 0x17, 0x6f, 0xf6, 0xc9, 0xbb, 0xca, + 0x1d, 0x51, 0x3e, 0x22, 0x7e, 0xdc, 0xa9, 0x05, 0x76, 0x5f, 0x39, 0x32, 0xb0, 0x3e, 0xfd, 0x18, + 0x6a, 0x53, 0xeb, 0xb5, 0x19, 0x90, 0x3f, 0xcc, 0x89, 0x37, 0x22, 0x8c, 0x96, 0x3c, 0xae, 0x4e, + 0xad, 0xd7, 0xfd, 0xd8, 0x84, 0xea, 0x00, 0x8e, 0x17, 0xfa, 0xd4, 0x9e, 0x8f, 0x88, 0x2f, 0x97, + 0x18, 0x6f, 0x19, 0x0b, 0xfa, 0x39, 0x94, 0x19, 0xa9, 0xa6, 0x63, 0xcb, 0x65, 0x55, 0x38, 0x10, + 0x8f, 0x95, 0x38, 0xf1, 0x12, 0xa3, 0x94, 0xe5, 0x9d, 0x2c, 0x71, 0x89, 0x61, 0xdb, 0x36, 0xfa, + 0x15, 0x28, 0xc1, 0x85, 0x13, 0x15, 0x84, 0x47, 0x0a, 0x1d, 0xea, 0x99, 0x3e, 0x99, 0xd2, 0x4b, + 0xcb, 0x0d, 0xe4, 0x0a, 0xbb, 0x46, 0x8e, 0x10, 0xed, 0x0c, 0x00, 0xc7, 0x7e, 0xad, 0x0b, 0x05, + 0x16, 0x11, 0xed, 0x42, 0x91, 0x37, 0x65, 0xac, 0xd2, 0x78, 0x87, 0x8e, 0xa0, 0x30, 0x76, 0x5c, + 0x12, 0xc8, 0x39, 0x56, 0x43, 0x94, 0xe9, 0x68, 0xc7, 0x25, 0x6d, 0x6f, 0x4c, 0xe3, 0x2a, 0x72, + 0x98, 0x76, 0x06, 0x55, 0x16, 0xf0, 0x6c, 0x66, 0x5b, 0x21, 0xf9, 0xbf, 0x85, 0xfd, 0x5b, 0x1e, + 0xca, 0x89, 0x27, 0x2d, 0xba, 0x90, 0x29, 0xfa, 0x61, 0xac, 0x7b, 0xae, 0xe2, 0xdd, 0xdb, 0xf1, + 0x32, 0xc2, 0x47, 0x20, 0x06, 0xce, 0x1f, 0x09, 0xd3, 0x4d, 0x1e, 0xb3, 0x35, 0x52, 0xa1, 0x7a, + 0x53, 0x2c, 0x9b, 0x38, 0x6b, 0x42, 0x1f, 0x01, 0x4c, 0xa9, 0xed, 0x8c, 0x1d, 0x62, 0x9b, 0x01, + 0x6b, 0x80, 0x3c, 0xae, 0x24, 0x96, 0x3e, 0x92, 0xa3, 0x76, 0x8f, 0xa4, 0x62, 0xc7, 0x9a, 0x48, + 0xb6, 0x91, 0xc7, 0xf1, 0x2e, 0x2d, 0xd7, 0xb1, 0xe3, 0xaa, 0x27, 0xdb, 0x68, 0xba, 0x79, 0x74, + 0x4d, 0xa4, 0x65, 0x06, 0xd8, 0xf4, 0x68, 0x56, 0xa0, 0x8f, 0xa1, 0x94, 0x4c, 0xbf, 0xa8, 0x9e, + 0x6b, 0x4a, 0x7a, 0x4e, 0x46, 0x21, 0x4d, 0xe7, 0x4a, 0x0c, 0x43, 0x0a, 0x94, 0xd3, 0x56, 0x04, + 0xf6, 0xd2, 0x74, 0x1f, 0xcd, 0xdc, 0x34, 0x0f, 0x2f, 0x90, 0xab, 0xaa, 0x70, 0x50, 0xc0, 0x69, + 0x6a, 0x46, 0x80, 0x7e, 0x06, 0xc5, 0x63, 0x97, 0x8e, 0x2e, 0x12, 0xdd, 0xde, 0x5b, 0xdd, 0xc6, + 0xec, 0x99, 0xea, 0x14, 0x87, 0x0c, 0x18, 0x25, 0x12, 0x5c, 0x4d, 0x5d, 0xc7, 0xbb, 0x30, 0x43, + 0xcb, 0x9f, 0x90, 0x50, 0xde, 0xe6, 0x63, 0x3a, 0xb6, 0x0e, 0x98, 0xf1, 0x97, 0xe2, 0x9f, 0xbf, + 0xdd, 0xdf, 0xd0, 0x3c, 0xa8, 0xa4, 0x71, 0xa2, 0x06, 0xa1, 0xe3, 0x71, 0x40, 0x42, 0x56, 0xcd, + 0x3c, 0x8e, 0x77, 0x69, 0x8d, 0x72, 0xec, 0x79, 0xbc, 0x46, 0x08, 0xc4, 0x73, 0x2b, 0x38, 0x67, + 0x75, 0xab, 0x61, 0xb6, 0x8e, 0x54, 0xf9, 0x8a, 0x58, 0x17, 0x26, 0x73, 0xf0, 0xaa, 0x95, 0x23, + 0xc3, 0x53, 0x2b, 0x38, 0x8f, 0xef, 0xfb, 0x35, 0x14, 0x39, 0x4b, 0xe8, 0x09, 0x94, 0x47, 0x74, + 0xee, 0x85, 0xab, 0x09, 0xbd, 0x9d, 0x15, 0x3e, 0xf3, 0xc4, 0x99, 0xa5, 0x40, 0xed, 0x04, 0x4a, + 0xb1, 0x0b, 0x3d, 0x4a, 0xa7, 0x92, 0x78, 0x7c, 0x3f, 0x11, 0x67, 0xff, 0x9c, 0xfa, 0xe1, 0xda, + 0x50, 0xda, 0x81, 0xc2, 0xa5, 0xe5, 0xce, 0xf9, 0xe3, 0x45, 0xcc, 0x37, 0xda, 0x5f, 0x05, 0x28, + 0xe1, 0xa8, 0x08, 0x41, 0x98, 0x19, 0xf6, 0x85, 0xb5, 0x61, 0xbf, 0x92, 0x4b, 0x6e, 0x4d, 0x2e, + 0x49, 0xc7, 0xe7, 0x33, 0x1d, 0xbf, 0x62, 0x4e, 0xbc, 0x93, 0xb9, 0xc2, 0x1d, 0xcc, 0x15, 0x33, + 0xcc, 0x3d, 0x82, 0xad, 0xb1, 0x4f, 0xa7, 0x6c, 0x9c, 0x53, 0xdf, 0xf2, 0xaf, 0xe2, 0xee, 0xdc, + 0x8c, 0xac, 0x83, 0xc4, 0xa8, 0x99, 0x50, 0xc6, 0x24, 0x98, 0x51, 0x2f, 0x20, 0xef, 0x7c, 0x36, + 0x02, 0xd1, 0xb6, 0x42, 0x8b, 0x3d, 0xba, 0x86, 0xd9, 0x1a, 0xfd, 0x04, 0xc4, 0x11, 0xb5, 0xf9, + 0x93, 0xb7, 0xb2, 0x3d, 0xa4, 0xfb, 0x3e, 0xf5, 0x9b, 0xd4, 0x26, 0x98, 0x01, 0xb4, 0x19, 0x48, + 0x2d, 0xfa, 0xca, 0x73, 0xa9, 0x65, 0xf7, 0x7c, 0x3a, 0x89, 0xc6, 0xed, 0x3b, 0xc7, 0x46, 0x0b, + 0x4a, 0x73, 0x36, 0x58, 0x92, 0xc1, 0xf1, 0xc9, 0xba, 0xd0, 0x6f, 0x06, 0xe2, 0x53, 0x28, 0x51, + 0x47, 0x7c, 0x54, 0xfb, 0x5e, 0x00, 0xe5, 0xdd, 0x68, 0xd4, 0x86, 0x2a, 0x47, 0x9a, 0x99, 0x2f, + 0x89, 0x83, 0xf7, 0xb9, 0x88, 0xcd, 0x18, 0x98, 0xa7, 0xeb, 0x3b, 0xff, 0x9e, 0x32, 0x6a, 0xce, + 0xbf, 0x9f, 0x9a, 0x1f, 0xc2, 0x26, 0xd3, 0x59, 0xfa, 0xa7, 0x2b, 0xaa, 0xf9, 0x83, 0x02, 0xae, + 0x0d, 0xb9, 0x8a, 0x98, 0x4d, 0x2b, 0x82, 0xd8, 0x73, 0xbc, 0x89, 0xb6, 0x0f, 0x85, 0xa6, 0x4b, + 0x59, 0xb1, 0x8a, 0x3e, 0xb1, 0x02, 0xea, 0x25, 0x1c, 0xf2, 0xdd, 0xe1, 0xf7, 0x39, 0xa8, 0x66, + 0x3e, 0x86, 0xd0, 0x63, 0xd8, 0x6a, 0x76, 0xce, 0xfa, 0x03, 0x1d, 0x9b, 0xcd, 0xae, 0x71, 0xd2, + 0x3e, 0x95, 0x36, 0x94, 0xbd, 0xc5, 0x52, 0x95, 0xa7, 0x2b, 0xd0, 0xfa, 0x77, 0xce, 0x3e, 0x14, + 0xda, 0x46, 0x4b, 0xff, 0x9d, 0x24, 0x28, 0x3b, 0x8b, 0xa5, 0x2a, 0x65, 0x80, 0xfc, 0xcf, 0xe4, + 0xa7, 0x50, 0x63, 0x00, 0xf3, 0xac, 0xd7, 0x6a, 0x0c, 0x74, 0x29, 0xa7, 0x28, 0x8b, 0xa5, 0xba, + 0x7b, 0x13, 0x17, 0xf3, 0xfd, 0x10, 0x4a, 0x58, 0xff, 0xed, 0x99, 0xde, 0x1f, 0x48, 0x79, 0x65, + 0x77, 0xb1, 0x54, 0x51, 0x06, 0x98, 0x28, 0xe6, 0x11, 0x94, 0xb1, 0xde, 0xef, 0x75, 0x8d, 0xbe, + 0x2e, 0x89, 0xca, 0x87, 0x8b, 0xa5, 0x7a, 0x6f, 0x0d, 0x15, 0x77, 0xe8, 0x2f, 0x60, 0xbb, 0xd5, + 0x7d, 0x61, 0x74, 0xba, 0x8d, 0x96, 0xd9, 0xc3, 0xdd, 0x53, 0xac, 0xf7, 0xfb, 0x52, 0x41, 0xd9, + 0x5f, 0x2c, 0xd5, 0x07, 0x19, 0xfc, 0xad, 0x86, 0xfb, 0x08, 0xc4, 0x5e, 0xdb, 0x38, 0x95, 0x8a, + 0xca, 0xbd, 0xc5, 0x52, 0xfd, 0x20, 0x03, 0x8d, 0x48, 0x8d, 0x32, 0x6e, 0x76, 0xba, 0x7d, 0x5d, + 0x2a, 0xdd, 0xca, 0x98, 0x91, 0x7d, 0xf8, 0x7b, 0x40, 0xb7, 0x3f, 0x17, 0xd1, 0x27, 0x20, 0x1a, + 0x5d, 0x43, 0x97, 0x36, 0x78, 0xfe, 0xb7, 0x11, 0x06, 0xf5, 0x08, 0xd2, 0x20, 0xdf, 0xf9, 0xe6, + 0x4b, 0x49, 0x50, 0x7e, 0xb4, 0x58, 0xaa, 0xf7, 0x6f, 0x83, 0x3a, 0xdf, 0x7c, 0x79, 0x48, 0xa1, + 0x9a, 0x0d, 0xac, 0x41, 0xf9, 0x99, 0x3e, 0x68, 0xb4, 0x1a, 0x83, 0x86, 0xb4, 0xc1, 0x9f, 0x94, + 0xb8, 0x9f, 0x91, 0xd0, 0x62, 0x02, 0xdc, 0x83, 0x82, 0xa1, 0x3f, 0xd7, 0xb1, 0x24, 0x28, 0xdb, + 0x8b, 0xa5, 0xba, 0x99, 0x00, 0x0c, 0x72, 0x49, 0x7c, 0x54, 0x87, 0x62, 0xa3, 0xf3, 0xa2, 0xf1, + 0xb2, 0x2f, 0xe5, 0x14, 0xb4, 0x58, 0xaa, 0x5b, 0x89, 0xbb, 0xe1, 0xbe, 0xb2, 0xae, 0x82, 0xc3, + 0xff, 0x08, 0x50, 0xcb, 0xfe, 0x75, 0xa2, 0x3a, 0x88, 0x27, 0xed, 0x8e, 0x9e, 0x5c, 0x97, 0xf5, + 0x45, 0x6b, 0x74, 0x00, 0x95, 0x56, 0x1b, 0xeb, 0xcd, 0x41, 0x17, 0xbf, 0x4c, 0x72, 0xc9, 0x82, + 0x5a, 0x8e, 0xcf, 0x9a, 0x3b, 0xfa, 0x3c, 0xad, 0xf5, 0x5f, 0x3e, 0xeb, 0xb4, 0x8d, 0xaf, 0x4d, + 0x16, 0x31, 0xa7, 0x3c, 0x58, 0x2c, 0xd5, 0x0f, 0xb3, 0xe0, 0x3e, 0xff, 0xdb, 0x60, 0x81, 0xbf, + 0x82, 0xed, 0x04, 0xbe, 0xba, 0x20, 0xaf, 0xa8, 0x8b, 0xa5, 0xba, 0x77, 0xc7, 0x99, 0xd5, 0x3d, + 0x4f, 0xe0, 0x83, 0xe4, 0xe0, 0x99, 0xf1, 0xb5, 0xd1, 0x7d, 0x61, 0x48, 0xa2, 0x52, 0x5f, 0x2c, + 0x55, 0xe5, 0x8e, 0x63, 0x67, 0xde, 0x85, 0x47, 0x5f, 0x79, 0x87, 0x7f, 0x11, 0xa0, 0x92, 0x4e, + 0xa8, 0x88, 0x67, 0xa3, 0x6b, 0xea, 0x18, 0x77, 0x71, 0x92, 0x78, 0xea, 0x34, 0x28, 0x5b, 0xa2, + 0x8f, 0xa1, 0x74, 0xaa, 0x1b, 0x3a, 0x6e, 0x37, 0x13, 0x3d, 0xa4, 0x90, 0x53, 0xe2, 0x11, 0xdf, + 0x19, 0xa1, 0x4f, 0xa1, 0x66, 0x74, 0xcd, 0xfe, 0x59, 0xf3, 0x69, 0x92, 0x31, 0x6b, 0xe0, 0x4c, + 0xa8, 0xfe, 0x7c, 0x74, 0xce, 0xb2, 0x3d, 0x8c, 0xa4, 0xf3, 0xbc, 0xd1, 0x69, 0xb7, 0x38, 0x34, + 0xaf, 0xc8, 0x8b, 0xa5, 0xba, 0x93, 0x42, 0xdb, 0xfc, 0xd3, 0x21, 0xc2, 0x1e, 0xda, 0x50, 0xff, + 0xdf, 0xb3, 0x08, 0xa9, 0x50, 0x6c, 0xf4, 0x7a, 0xba, 0xd1, 0x4a, 0x5e, 0xbf, 0xf2, 0x35, 0x66, + 0x33, 0xe2, 0xd9, 0x11, 0xe2, 0xa4, 0x8b, 0x4f, 0xf5, 0x41, 0xf2, 0xf8, 0x15, 0xe2, 0x84, 0x46, + 0x7f, 0xda, 0xc7, 0x7b, 0x6f, 0x7e, 0xa8, 0x6f, 0x7c, 0xf7, 0x43, 0x7d, 0xe3, 0xcd, 0x75, 0x5d, + 0xf8, 0xee, 0xba, 0x2e, 0xfc, 0xe3, 0xba, 0xbe, 0xf1, 0xaf, 0xeb, 0xba, 0xf0, 0xed, 0x3f, 0xeb, + 0xc2, 0xb0, 0xc8, 0x66, 0xd7, 0x93, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xee, 0xe2, 0x0f, 0x00, + 0x38, 0x0e, 0x00, 0x00, } diff --git a/lib/protocol/bep.proto b/lib/protocol/bep.proto index 751a3b5f2..9c97e2a0b 100644 --- a/lib/protocol/bep.proto +++ b/lib/protocol/bep.proto @@ -120,9 +120,10 @@ enum FileInfoType { message BlockInfo { option (gogoproto.goproto_stringer) = false; - int64 offset = 1; - int32 size = 2; - bytes hash = 3; + int64 offset = 1; + int32 size = 2; + bytes hash = 3; + uint32 weak_hash = 4; } message Vector { diff --git a/lib/protocol/bep_extensions.go b/lib/protocol/bep_extensions.go index ece75016b..74ad158ec 100644 --- a/lib/protocol/bep_extensions.go +++ b/lib/protocol/bep_extensions.go @@ -114,7 +114,7 @@ func (f FileInfo) WinsConflict(other FileInfo) bool { } func (b BlockInfo) String() string { - return fmt.Sprintf("Block{%d/%d/%x}", b.Offset, b.Size, b.Hash) + return fmt.Sprintf("Block{%d/%d/%d/%x}", b.Offset, b.Size, b.WeakHash, b.Hash) } // IsEmpty returns true if the block is a full block of zeroes. diff --git a/lib/protocol/deviceid_test.pb.go b/lib/protocol/deviceid_test.pb.go index 95677dd75..06df45425 100644 --- a/lib/protocol/deviceid_test.pb.go +++ b/lib/protocol/deviceid_test.pb.go @@ -427,16 +427,16 @@ var ( ) var fileDescriptorDeviceidTest = []byte{ - // 171 bytes of a gzipped FileDescriptorProto + // 176 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0x12, 0x4e, 0x49, 0x2d, 0xcb, 0x4c, 0x4e, 0xcd, 0x4c, 0x89, 0x2f, 0x49, 0x2d, 0x2e, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x00, 0x53, 0xc9, 0xf9, 0x39, 0x52, 0xba, 0xe9, 0x99, 0x25, 0x19, 0xa5, 0x49, 0x7a, 0xc9, 0xf9, 0xb9, 0xfa, 0xe9, 0xf9, 0xe9, 0xf9, 0xfa, 0x60, 0x99, 0xa4, 0xd2, 0x34, 0x30, 0x0f, 0xcc, - 0x01, 0xb3, 0x20, 0x1a, 0x95, 0x54, 0xb9, 0xf8, 0x43, 0x80, 0xc6, 0xf8, 0xe7, 0xa4, 0xb8, 0x80, - 0x8d, 0xf5, 0x74, 0x11, 0x12, 0xe2, 0x62, 0x01, 0x99, 0x2c, 0xc1, 0xa8, 0xc0, 0xa8, 0xc1, 0x13, - 0x04, 0x66, 0x2b, 0x99, 0x43, 0x94, 0xf9, 0xa5, 0x96, 0xc3, 0x95, 0xa9, 0x20, 0x2b, 0x73, 0x12, - 0x38, 0x71, 0x4f, 0x9e, 0xe1, 0xd6, 0x3d, 0x79, 0x0e, 0x98, 0x3c, 0x44, 0xa3, 0x93, 0xcc, 0x89, - 0x87, 0x72, 0x0c, 0x17, 0x80, 0xf8, 0xc4, 0x23, 0x39, 0xc6, 0x0b, 0x40, 0xfc, 0xe0, 0x91, 0x1c, - 0xc3, 0x0b, 0x20, 0x5e, 0xf0, 0x58, 0x8e, 0x31, 0x89, 0x0d, 0xec, 0x08, 0x63, 0x40, 0x00, 0x00, - 0x00, 0xff, 0xff, 0x35, 0x9c, 0x00, 0x78, 0xd4, 0x00, 0x00, 0x00, + 0x01, 0xb3, 0x20, 0x1a, 0x95, 0x54, 0xb9, 0xf8, 0x43, 0x52, 0x8b, 0x4b, 0xfc, 0x73, 0x52, 0x5c, + 0xc0, 0xc6, 0x7a, 0xba, 0x08, 0x09, 0x71, 0xb1, 0x80, 0x4c, 0x96, 0x60, 0x54, 0x60, 0xd4, 0xe0, + 0x09, 0x02, 0xb3, 0x95, 0xcc, 0x21, 0xca, 0xfc, 0x52, 0xcb, 0xe1, 0xca, 0x54, 0x90, 0x95, 0x39, + 0x09, 0x9c, 0xb8, 0x27, 0xcf, 0x70, 0xeb, 0x9e, 0x3c, 0x07, 0x4c, 0x1e, 0xa2, 0xd1, 0x49, 0xe6, + 0xc4, 0x43, 0x39, 0x86, 0x0b, 0x0f, 0xe5, 0x18, 0x4e, 0x3c, 0x92, 0x63, 0xbc, 0xf0, 0x48, 0x8e, + 0xf1, 0xc1, 0x23, 0x39, 0x86, 0x17, 0x8f, 0xe4, 0x18, 0x16, 0x3c, 0x96, 0x63, 0x4c, 0x62, 0x03, + 0x3b, 0xc2, 0x18, 0x10, 0x00, 0x00, 0xff, 0xff, 0x35, 0x9c, 0x00, 0x78, 0xd4, 0x00, 0x00, 0x00, } diff --git a/lib/scanner/blocks.go b/lib/scanner/blocks.go index 3d9919ff4..eb5ad2bca 100644 --- a/lib/scanner/blocks.go +++ b/lib/scanner/blocks.go @@ -13,6 +13,7 @@ import ( "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/sha256" + "github.com/syncthing/syncthing/lib/weakhash" ) var SHA256OfNothing = []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55} @@ -25,6 +26,7 @@ type Counter interface { func Blocks(r io.Reader, blocksize int, sizehint int64, counter Counter) ([]protocol.BlockInfo, error) { hf := sha256.New() hashLength := hf.Size() + whf := weakhash.NewHash(blocksize) var blocks []protocol.BlockInfo var hashes, thisHash []byte @@ -44,7 +46,7 @@ func Blocks(r io.Reader, blocksize int, sizehint int64, counter Counter) ([]prot var offset int64 for { lr := io.LimitReader(r, int64(blocksize)) - n, err := copyBuffer(hf, lr, buf) + n, err := io.CopyBuffer(hf, io.TeeReader(lr, whf), buf) if err != nil { return nil, err } @@ -63,15 +65,17 @@ func Blocks(r io.Reader, blocksize int, sizehint int64, counter Counter) ([]prot thisHash, hashes = hashes[:hashLength], hashes[hashLength:] b := protocol.BlockInfo{ - Size: int32(n), - Offset: offset, - Hash: thisHash, + Size: int32(n), + Offset: offset, + Hash: thisHash, + WeakHash: whf.Sum32(), } blocks = append(blocks, b) offset += int64(n) hf.Reset() + whf.Reset() } if len(blocks) == 0 { @@ -123,9 +127,12 @@ func BlockDiff(src, tgt []protocol.BlockInfo) (have, need []protocol.BlockInfo) // list and actual reader contents func Verify(r io.Reader, blocksize int, blocks []protocol.BlockInfo) error { hf := sha256.New() + // A 32k buffer is used for copying into the hash function. + buf := make([]byte, 32<<10) + for i, block := range blocks { lr := &io.LimitedReader{R: r, N: int64(blocksize)} - _, err := io.Copy(hf, lr) + _, err := io.CopyBuffer(hf, lr, buf) if err != nil { return err } @@ -180,48 +187,3 @@ func BlocksEqual(src, tgt []protocol.BlockInfo) bool { } return true } - -// This is a copy & paste of io.copyBuffer from the Go 1.5 standard library, -// as we want this but also want to build with Go 1.3+. - -// copyBuffer is the actual implementation of Copy and CopyBuffer. -// if buf is nil, one is allocated. -func copyBuffer(dst io.Writer, src io.Reader, buf []byte) (written int64, err error) { - // If the reader has a WriteTo method, use it to do the copy. - // Avoids an allocation and a copy. - if wt, ok := src.(io.WriterTo); ok { - return wt.WriteTo(dst) - } - // Similarly, if the writer has a ReadFrom method, use it to do the copy. - if rt, ok := dst.(io.ReaderFrom); ok { - return rt.ReadFrom(src) - } - if buf == nil { - buf = make([]byte, 32*1024) - } - for { - nr, er := src.Read(buf) - if nr > 0 { - nw, ew := dst.Write(buf[0:nr]) - if nw > 0 { - written += int64(nw) - } - if ew != nil { - err = ew - break - } - if nr != nw { - err = io.ErrShortWrite - break - } - } - if er == io.EOF { - break - } - if er != nil { - err = er - break - } - } - return written, err -} diff --git a/lib/weakhash/benchmark_test.go b/lib/weakhash/benchmark_test.go new file mode 100644 index 000000000..79f5ce9dd --- /dev/null +++ b/lib/weakhash/benchmark_test.go @@ -0,0 +1,30 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package weakhash + +import ( + "os" + "testing" +) + +const testFile = "../model/testdata/~syncthing~file.tmp" + +func BenchmarkFind1MFile(b *testing.B) { + b.ReportAllocs() + b.SetBytes(1 << 20) + for i := 0; i < b.N; i++ { + fd, err := os.Open(testFile) + if err != nil { + b.Fatal(err) + } + _, err = Find(fd, []uint32{0, 1, 2}, 128<<10) + if err != nil { + b.Fatal(err) + } + fd.Close() + } +} diff --git a/lib/weakhash/weakhash.go b/lib/weakhash/weakhash.go new file mode 100644 index 000000000..5c210a52d --- /dev/null +++ b/lib/weakhash/weakhash.go @@ -0,0 +1,169 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package weakhash + +import ( + "bufio" + "hash" + "io" + "os" +) + +const ( + Size = 4 +) + +func NewHash(size int) hash.Hash32 { + return &digest{ + buf: make([]byte, size), + size: size, + } +} + +// Find finds all the blocks of the given size within io.Reader that matches +// the hashes provided, and returns a hash -> slice of offsets within reader +// map, that produces the same weak hash. +func Find(ir io.Reader, hashesToFind []uint32, size int) (map[uint32][]int64, error) { + if ir == nil { + return nil, nil + } + + r := bufio.NewReader(ir) + hf := NewHash(size) + + n, err := io.CopyN(hf, r, int64(size)) + if err == io.EOF { + return nil, nil + } + if err != nil { + return nil, err + } + if n != int64(size) { + return nil, io.ErrShortBuffer + } + + offsets := make(map[uint32][]int64) + for _, hashToFind := range hashesToFind { + offsets[hashToFind] = nil + } + + var i int64 + var hash uint32 + for { + hash = hf.Sum32() + if existing, ok := offsets[hash]; ok { + offsets[hash] = append(existing, i) + } + i++ + + bt, err := r.ReadByte() + if err == io.EOF { + break + } else if err != nil { + return offsets, err + } + hf.Write([]byte{bt}) + } + return offsets, nil +} + +// Using this: http://tutorials.jenkov.com/rsync/checksums.html +// Example implementations: https://gist.github.com/csabahenk/1096262/revisions +// Alternative that could be used is adler32 http://blog.liw.fi/posts/rsync-in-python/#comment-fee8d5e07794fdba3fe2d76aa2706a13 +type digest struct { + buf []byte + size int + a uint16 + b uint16 + j int +} + +func (d *digest) Write(data []byte) (int, error) { + for _, c := range data { + // TODO: Use this in Go 1.6 + // d.a = d.a - uint16(d.buf[d.j]) + uint16(c) + // d.b = d.b - uint16(d.size)*uint16(d.buf[d.j]) + d.a + d.a -= uint16(d.buf[d.j]) + d.a += uint16(c) + d.b -= uint16(d.size) * uint16(d.buf[d.j]) + d.b += d.a + + d.buf[d.j] = c + d.j = (d.j + 1) % d.size + } + return len(data), nil +} + +func (d *digest) Reset() { + for i := range d.buf { + d.buf[i] = 0x0 + } + d.a = 0 + d.b = 0 + d.j = 0 +} + +func (d *digest) Sum(b []byte) []byte { + r := d.Sum32() + return append(b, byte(r>>24), byte(r>>16), byte(r>>8), byte(r)) +} + +func (d *digest) Sum32() uint32 { return uint32(d.a) | (uint32(d.b) << 16) } +func (digest) Size() int { return Size } +func (digest) BlockSize() int { return 1 } + +func NewFinder(path string, size int, hashesToFind []uint32) (*Finder, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + + offsets, err := Find(file, hashesToFind, size) + if err != nil { + file.Close() + return nil, err + } + + return &Finder{ + file: file, + size: size, + offsets: offsets, + }, nil +} + +type Finder struct { + file *os.File + size int + offsets map[uint32][]int64 +} + +// Iterate iterates all available blocks that matches the provided hash, reads +// them into buf, and calls the iterator function. The iterator function should +// return wether it wishes to continue interating. +func (h *Finder) Iterate(hash uint32, buf []byte, iterFunc func(int64) bool) (bool, error) { + if h == nil || hash == 0 || len(buf) != h.size { + return false, nil + } + + for _, offset := range h.offsets[hash] { + _, err := h.file.ReadAt(buf, offset) + if err != nil { + return false, err + } + if !iterFunc(offset) { + return true, nil + } + } + return false, nil +} + +// Close releases any resource associated with the finder +func (h *Finder) Close() { + if h != nil { + h.file.Close() + } +} diff --git a/lib/weakhash/weakhash_test.go b/lib/weakhash/weakhash_test.go new file mode 100644 index 000000000..525d60dc6 --- /dev/null +++ b/lib/weakhash/weakhash_test.go @@ -0,0 +1,188 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// The existence of this file means we get 0% test coverage rather than no +// test coverage at all. Remove when implementing an actual test. + +package weakhash + +import ( + "bytes" + "io/ioutil" + "os" + "reflect" + "testing" +) + +var payload = []byte("abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz") +var hashes = []uint32{ + 64225674, + 64881038, + 65536402, + 66191766, + 66847130, + 67502494, + 68157858, + 68813222, + 69468586, + 70123950, + 70779314, + 71434678, + 72090042, + 72745406, + 73400770, + 74056134, + 74711498, + 75366862, + 76022226, + 76677590, + 77332954, + 77988318, + 78643682, + 77595084, + 74842550, + 70386080, + 64225674, + 64881038, + 65536402, + 66191766, + 66847130, + 67502494, + 68157858, + 68813222, + 69468586, + 70123950, + 70779314, + 71434678, + 72090042, + 72745406, + 73400770, + 74056134, + 74711498, + 75366862, + 76022226, + 76677590, + 77332954, + 77988318, + 78643682, + 77595084, + 74842550, + 70386080, + 64225674, + 64881038, + 65536402, + 66191766, + 66847130, + 67502494, + 68157858, + 68813222, + 69468586, + 70123950, + 70779314, + 71434678, + 72090042, + 72745406, + 73400770, + 74056134, + 74711498, + 75366862, + 76022226, + 76677590, + 77332954, + 77988318, + 78643682, + 77595084, + 74842550, + 70386080, + 64225674, + 64881038, + 65536402, + 66191766, + 66847130, + 67502494, + 68157858, + 68813222, + 69468586, + 70123950, + 70779314, + 71434678, + 72090042, + 72745406, + 73400770, + 74056134, + 74711498, + 75366862, + 76022226, + 76677590, + 77332954, + 77988318, + 78643682, + 71893365, + 71893365, +} + +// Tested using an alternative C implementation at https://gist.github.com/csabahenk/1096262 +func TestHashCorrect(t *testing.T) { + h := NewHash(Size) + pos := 0 + for pos < Size { + h.Write([]byte{payload[pos]}) + pos++ + } + + for i := 0; pos < len(payload); i++ { + if h.Sum32() != hashes[i] { + t.Errorf("mismatch at %d", i) + } + h.Write([]byte{payload[pos]}) + pos++ + } +} + +func TestFinder(t *testing.T) { + f, err := ioutil.TempFile("", "") + if err != nil { + t.Error(err) + } + defer os.Remove(f.Name()) + defer f.Close() + + if _, err := f.Write(payload); err != nil { + t.Error(err) + } + + hashes := []uint32{64881038, 65536402} + finder, err := NewFinder(f.Name(), 4, hashes) + if err != nil { + t.Error(err) + } + defer finder.Close() + + expected := map[uint32][]int64{ + 64881038: []int64{1, 27, 53, 79}, + 65536402: []int64{2, 28, 54, 80}, + } + actual := make(map[uint32][]int64) + + b := make([]byte, Size) + + for _, hash := range hashes { + _, err := finder.Iterate(hash, b[:4], func(offset int64) bool { + if !bytes.Equal(b, payload[offset:offset+4]) { + t.Errorf("Not equal at %d: %s != %s", offset, string(b), string(payload[offset:offset+4])) + } + actual[hash] = append(actual[hash], offset) + return true + }) + if err != nil { + t.Error(err) + } + } + + if !reflect.DeepEqual(actual, expected) { + t.Errorf("Not equal: %#v != %#v", actual, expected) + } +} diff --git a/test/h2/config.xml b/test/h2/config.xml index 09b0a0dd2..c688583fd 100644 --- a/test/h2/config.xml +++ b/test/h2/config.xml @@ -1,42 +1,6 @@ - + - - - 1 - - - - 1 - 16 - 0 - random - false - 0 - 0 - 0 - -1 - false - false - - - - - 1 - - 1 - 16 - 0 - random - false - 0 - 0 - 0 - -1 - false - false - - 1 @@ -52,6 +16,43 @@ -1 false false + false + + + + + 1 + + 1 + 16 + 0 + random + false + 0 + 0 + 0 + -1 + false + false + false + + + + + 1 + + 1 + 16 + 0 + random + false + 0 + 0 + 0 + -1 + false + false + false
tcp://127.0.0.1:22001
diff --git a/test/h3/config.xml b/test/h3/config.xml index 93977b158..6579d8bd2 100644 --- a/test/h3/config.xml +++ b/test/h3/config.xml @@ -1,5 +1,5 @@ - - + + 1 @@ -15,8 +15,9 @@ -1 false false + false - + @@ -35,6 +36,7 @@ -1 false false + false
tcp://127.0.0.1:22001