lib/model, lib/scanner: Efficient inserts/deletes in the middle of the file
GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3527
This commit is contained in:
Audrius Butkevicius
parent
bb15776ae6
commit
0582836820
|
|
@ -40,6 +40,7 @@ type FolderConfiguration struct {
|
|||
DisableSparseFiles bool `xml:"disableSparseFiles" json:"disableSparseFiles"`
|
||||
DisableTempIndexes bool `xml:"disableTempIndexes" json:"disableTempIndexes"`
|
||||
Fsync bool `xml:"fsync" json:"fsync"`
|
||||
DisableWeakHash bool `xml:"disableWeakHash" json:"disableWeakHash"`
|
||||
|
||||
cachedPath string
|
||||
|
||||
|
|
|
|||
|
|
@ -954,33 +954,33 @@ var (
|
|||
)
|
||||
|
||||
var fileDescriptorStructs = []byte{
|
||||
// 442 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x51, 0x5d, 0x8b, 0xd3, 0x40,
|
||||
0x14, 0x6d, 0xb7, 0xd9, 0x7e, 0xdc, 0xd8, 0xd5, 0x1d, 0x64, 0x19, 0x0a, 0xb6, 0x4b, 0x41, 0x10,
|
||||
0xc1, 0x54, 0x57, 0x7c, 0xf1, 0x71, 0x1f, 0x16, 0x04, 0x11, 0x19, 0x97, 0xf5, 0xb1, 0x34, 0x99,
|
||||
0xdb, 0xec, 0x60, 0x32, 0x53, 0x33, 0x93, 0x42, 0xfd, 0x25, 0xbe, 0xb9, 0x3f, 0xa7, 0x8f, 0xfe,
|
||||
0x02, 0xd1, 0xfa, 0x47, 0x9c, 0xce, 0xa4, 0x31, 0x8f, 0xfb, 0x10, 0xb8, 0xe7, 0x9e, 0x73, 0xee,
|
||||
0x3d, 0x93, 0x0b, 0x43, 0x6d, 0x8a, 0x32, 0x31, 0x3a, 0x5a, 0x15, 0xca, 0x28, 0x72, 0xc4, 0xe3,
|
||||
0xd1, 0x8b, 0x54, 0x98, 0xdb, 0x32, 0x8e, 0x12, 0x95, 0xcf, 0x52, 0x95, 0xaa, 0x99, 0xa3, 0xe2,
|
||||
0x72, 0xe9, 0x90, 0x03, 0xae, 0xf2, 0x96, 0xd1, 0x9b, 0x86, 0x5c, 0x6f, 0x64, 0x62, 0x6e, 0x85,
|
||||
0x4c, 0x1b, 0x55, 0x26, 0x62, 0x3f, 0x21, 0x51, 0xd9, 0x2c, 0xc6, 0x95, 0xb7, 0x4d, 0x3f, 0x43,
|
||||
0x78, 0x25, 0x32, 0xbc, 0xc1, 0x42, 0x0b, 0x25, 0xc9, 0x4b, 0xe8, 0xad, 0x7d, 0x49, 0xdb, 0xe7,
|
||||
0xed, 0x67, 0xe1, 0xc5, 0xa3, 0xe8, 0x60, 0x8a, 0x6e, 0x30, 0x31, 0xaa, 0xb8, 0x0c, 0xb6, 0xbf,
|
||||
0x26, 0x2d, 0x76, 0x90, 0x91, 0x33, 0xe8, 0x72, 0x5c, 0x8b, 0x04, 0xe9, 0x91, 0x35, 0x3c, 0x60,
|
||||
0x15, 0x9a, 0x5e, 0x41, 0x58, 0x0d, 0x7d, 0x2f, 0xb4, 0x21, 0xaf, 0xa0, 0x5f, 0x39, 0xb4, 0x9d,
|
||||
0xdc, 0xb1, 0x93, 0x1f, 0x46, 0x3c, 0x8e, 0x1a, 0xbb, 0xab, 0xc1, 0xb5, 0xec, 0x6d, 0xf0, 0xfd,
|
||||
0x6e, 0xd2, 0x9a, 0xfe, 0xe8, 0xc0, 0xe9, 0x5e, 0xf5, 0x4e, 0x2e, 0xd5, 0x75, 0x51, 0xca, 0x64,
|
||||
0x61, 0x90, 0x13, 0x02, 0x81, 0x5c, 0xe4, 0xe8, 0x42, 0x0e, 0x98, 0xab, 0xc9, 0x73, 0x08, 0xcc,
|
||||
0x66, 0xe5, 0x73, 0x9c, 0x5c, 0x9c, 0xfd, 0x0f, 0x5e, 0xdb, 0x2d, 0xcb, 0x9c, 0x66, 0xef, 0xd7,
|
||||
0xe2, 0x1b, 0xd2, 0x8e, 0xd5, 0x76, 0x98, 0xab, 0xc9, 0x39, 0x84, 0x2b, 0x2c, 0x72, 0xa1, 0x7d,
|
||||
0xca, 0xc0, 0x52, 0x43, 0xd6, 0x6c, 0x91, 0x27, 0x00, 0xb9, 0xe2, 0x62, 0x29, 0x90, 0xcf, 0x35,
|
||||
0x3d, 0x76, 0xde, 0xc1, 0xa1, 0xf3, 0x89, 0x50, 0xe8, 0x71, 0xcc, 0xd0, 0xe6, 0xa3, 0x5d, 0xcb,
|
||||
0xf5, 0xd9, 0x01, 0xee, 0x19, 0x21, 0xd7, 0x8b, 0x4c, 0x70, 0xda, 0xf3, 0x4c, 0x05, 0xc9, 0x53,
|
||||
0x38, 0x91, 0x6a, 0xde, 0xdc, 0xdb, 0x77, 0x82, 0xa1, 0x54, 0x1f, 0x1b, 0x9b, 0x1b, 0x77, 0x19,
|
||||
0xdc, 0xef, 0x2e, 0x23, 0xe8, 0x6b, 0xfc, 0x5a, 0xa2, 0xb4, 0x97, 0x01, 0x97, 0xb4, 0xc6, 0x64,
|
||||
0x02, 0x61, 0xfd, 0x0e, 0xbb, 0x31, 0xb4, 0xf4, 0x31, 0xab, 0x9f, 0xf6, 0x41, 0xef, 0x53, 0xe9,
|
||||
0x4d, 0x9e, 0x09, 0xf9, 0x65, 0x6e, 0x16, 0x45, 0x8a, 0x86, 0x9e, 0xba, 0x1f, 0x3d, 0xac, 0xba,
|
||||
0xd7, 0xae, 0xe9, 0x2f, 0x74, 0xf9, 0x78, 0xfb, 0x67, 0xdc, 0xda, 0xee, 0xc6, 0xed, 0x9f, 0xf6,
|
||||
0xfb, 0xbd, 0x1b, 0xb7, 0xee, 0xfe, 0x8e, 0xdb, 0x71, 0xd7, 0xe5, 0x7b, 0xfd, 0x2f, 0x00, 0x00,
|
||||
0xff, 0xff, 0xb1, 0x2f, 0x12, 0xb6, 0xda, 0x02, 0x00, 0x00,
|
||||
// 446 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x51, 0xd1, 0x8a, 0xd3, 0x40,
|
||||
0x14, 0xed, 0x6c, 0xb3, 0xdb, 0xf6, 0xc6, 0xae, 0xee, 0x20, 0xcb, 0x50, 0x30, 0x0d, 0x05, 0x21,
|
||||
0x08, 0xa6, 0x5a, 0xf1, 0xc5, 0xc7, 0x7d, 0x58, 0x10, 0x44, 0x24, 0x2e, 0xeb, 0x63, 0x69, 0x32,
|
||||
0xb7, 0xd9, 0xc1, 0x64, 0x26, 0x66, 0x26, 0x85, 0xfa, 0x25, 0xbe, 0xb9, 0x9f, 0xd3, 0x47, 0xbf,
|
||||
0x40, 0xb4, 0xfe, 0x88, 0x74, 0x92, 0xc6, 0x3c, 0xfa, 0x76, 0xce, 0xbd, 0xe7, 0xdc, 0x7b, 0x66,
|
||||
0x2e, 0x8c, 0xb5, 0x29, 0xab, 0xc4, 0xe8, 0xb0, 0x28, 0x95, 0x51, 0xf4, 0x84, 0xc7, 0x93, 0xe7,
|
||||
0xa9, 0x30, 0x77, 0x55, 0x1c, 0x26, 0x2a, 0x9f, 0xa7, 0x2a, 0x55, 0x73, 0xdb, 0x8a, 0xab, 0xb5,
|
||||
0x65, 0x96, 0x58, 0x54, 0x5b, 0x26, 0xaf, 0x3b, 0x72, 0xbd, 0x95, 0x89, 0xb9, 0x13, 0x32, 0xed,
|
||||
0xa0, 0x4c, 0xc4, 0xf5, 0x84, 0x44, 0x65, 0xf3, 0x18, 0x8b, 0xda, 0x36, 0xfb, 0x04, 0xee, 0xb5,
|
||||
0xc8, 0xf0, 0x16, 0x4b, 0x2d, 0x94, 0xa4, 0x2f, 0x60, 0xb0, 0xa9, 0x21, 0x23, 0x3e, 0x09, 0xdc,
|
||||
0xc5, 0xa3, 0xf0, 0x68, 0x0a, 0x6f, 0x31, 0x31, 0xaa, 0xbc, 0x72, 0x76, 0x3f, 0xa7, 0xbd, 0xe8,
|
||||
0x28, 0xa3, 0x97, 0x70, 0xc6, 0x71, 0x23, 0x12, 0x64, 0x27, 0x3e, 0x09, 0x1e, 0x44, 0x0d, 0x9b,
|
||||
0x5d, 0x83, 0xdb, 0x0c, 0x7d, 0x27, 0xb4, 0xa1, 0x2f, 0x61, 0xd8, 0x38, 0x34, 0x23, 0x7e, 0x3f,
|
||||
0x70, 0x17, 0x0f, 0x43, 0x1e, 0x87, 0x9d, 0xdd, 0xcd, 0xe0, 0x56, 0xf6, 0xc6, 0xf9, 0x76, 0x3f,
|
||||
0xed, 0xcd, 0xbe, 0xf7, 0xe1, 0xe2, 0xa0, 0x7a, 0x2b, 0xd7, 0xea, 0xa6, 0xac, 0x64, 0xb2, 0x32,
|
||||
0xc8, 0x29, 0x05, 0x47, 0xae, 0x72, 0xb4, 0x21, 0x47, 0x91, 0xc5, 0xf4, 0x19, 0x38, 0x66, 0x5b,
|
||||
0xd4, 0x39, 0xce, 0x17, 0x97, 0xff, 0x82, 0xb7, 0xf6, 0x6d, 0x81, 0x91, 0xd5, 0x1c, 0xfc, 0x5a,
|
||||
0x7c, 0x45, 0xd6, 0xf7, 0x49, 0xd0, 0x8f, 0x2c, 0xa6, 0x3e, 0xb8, 0x05, 0x96, 0xb9, 0xd0, 0x75,
|
||||
0x4a, 0xc7, 0x27, 0xc1, 0x38, 0xea, 0x96, 0xe8, 0x13, 0x80, 0x5c, 0x71, 0xb1, 0x16, 0xc8, 0x97,
|
||||
0x9a, 0x9d, 0x5a, 0xef, 0xe8, 0x58, 0xf9, 0x48, 0x19, 0x0c, 0x38, 0x66, 0x68, 0x90, 0xb3, 0x33,
|
||||
0x9f, 0x04, 0xc3, 0xe8, 0x48, 0x0f, 0x1d, 0x21, 0x37, 0xab, 0x4c, 0x70, 0x36, 0xa8, 0x3b, 0x0d,
|
||||
0xa5, 0x4f, 0xe1, 0x5c, 0xaa, 0x65, 0x77, 0xef, 0xd0, 0x0a, 0xc6, 0x52, 0x7d, 0xe8, 0x6c, 0xee,
|
||||
0xdc, 0x65, 0xf4, 0x7f, 0x77, 0x99, 0xc0, 0x50, 0xe3, 0x97, 0x0a, 0x65, 0x82, 0x0c, 0x6c, 0xd2,
|
||||
0x96, 0xd3, 0x29, 0xb8, 0xed, 0x3b, 0xa4, 0x66, 0xae, 0x4f, 0x82, 0xd3, 0xa8, 0x7d, 0xda, 0x7b,
|
||||
0x7d, 0x48, 0xa5, 0xb7, 0x79, 0x26, 0xe4, 0xe7, 0xa5, 0x59, 0x95, 0x29, 0x1a, 0x76, 0x61, 0x3f,
|
||||
0x7a, 0xdc, 0x54, 0x6f, 0x6c, 0xb1, 0xbe, 0xd0, 0xd5, 0xe3, 0xdd, 0x6f, 0xaf, 0xb7, 0xdb, 0x7b,
|
||||
0xe4, 0xc7, 0xde, 0x23, 0xbf, 0xf6, 0x5e, 0xef, 0xfe, 0x8f, 0x47, 0xe2, 0x33, 0x9b, 0xef, 0xd5,
|
||||
0xdf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xb1, 0x2f, 0x12, 0xb6, 0xda, 0x02, 0x00, 0x00,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -382,20 +382,21 @@ var (
|
|||
)
|
||||
|
||||
var fileDescriptorLocal = []byte{
|
||||
// 235 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0xce, 0xc9, 0x4f, 0x4e,
|
||||
0xcc, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x48, 0xc9, 0x2c, 0x4e, 0xce, 0x2f, 0x4b,
|
||||
0x2d, 0x92, 0xd2, 0x4d, 0xcf, 0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x4f, 0xcf,
|
||||
0x4f, 0xcf, 0xd7, 0x07, 0x2b, 0x48, 0x2a, 0x4d, 0x03, 0xf3, 0xc0, 0x1c, 0x30, 0x0b, 0xa2, 0x51,
|
||||
0x69, 0x2d, 0x23, 0x17, 0x87, 0x63, 0x5e, 0x5e, 0x7e, 0x69, 0x5e, 0x72, 0xaa, 0x50, 0x10, 0x17,
|
||||
0x53, 0x66, 0x8a, 0x04, 0xa3, 0x02, 0xa3, 0x06, 0x8f, 0x93, 0xd3, 0x89, 0x7b, 0xf2, 0x0c, 0xb7,
|
||||
0xee, 0xc9, 0x9b, 0x20, 0x99, 0x57, 0x5c, 0x99, 0x97, 0x5c, 0x92, 0x91, 0x99, 0x97, 0x8e, 0xc4,
|
||||
0xca, 0xc9, 0x4c, 0x82, 0x58, 0x91, 0x9c, 0x9f, 0xa3, 0xe7, 0x92, 0x5a, 0x96, 0x99, 0x9c, 0xea,
|
||||
0xe9, 0xf2, 0xe8, 0x9e, 0x3c, 0x93, 0xa7, 0x4b, 0x10, 0xd0, 0x34, 0x21, 0x19, 0x2e, 0xce, 0xc4,
|
||||
0x94, 0x94, 0xa2, 0xd4, 0xe2, 0xe2, 0xd4, 0x62, 0x09, 0x26, 0x05, 0x66, 0x0d, 0xce, 0x20, 0x84,
|
||||
0x80, 0x90, 0x3e, 0x17, 0x77, 0x66, 0x5e, 0x71, 0x49, 0x22, 0xd0, 0xf6, 0x78, 0xa0, 0xd5, 0xcc,
|
||||
0x40, 0xab, 0x99, 0x9d, 0xf8, 0x80, 0xda, 0xb9, 0x3c, 0xa1, 0xc2, 0x40, 0x63, 0xb8, 0x60, 0x4a,
|
||||
0x3c, 0x53, 0x9c, 0x44, 0x4e, 0x3c, 0x94, 0x63, 0x38, 0xf1, 0x48, 0x8e, 0xf1, 0x02, 0x10, 0x3f,
|
||||
0x78, 0x24, 0xc7, 0xb0, 0xe0, 0xb1, 0x1c, 0x63, 0x12, 0x1b, 0xd8, 0x05, 0xc6, 0x80, 0x00, 0x00,
|
||||
0x00, 0xff, 0xff, 0xa4, 0x46, 0x4f, 0x13, 0x14, 0x01, 0x00, 0x00,
|
||||
// 241 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x4c, 0x8e, 0x4f, 0x4e, 0x84, 0x30,
|
||||
0x14, 0xc6, 0x29, 0x24, 0x66, 0xa6, 0x63, 0x5c, 0x10, 0x17, 0xc4, 0x98, 0x42, 0x5c, 0xb1, 0x11,
|
||||
0x16, 0x7a, 0x01, 0x09, 0x9b, 0x6e, 0xb9, 0x80, 0x81, 0xb6, 0x32, 0x2f, 0xc1, 0x3e, 0x43, 0x61,
|
||||
0x12, 0x6f, 0xe3, 0x05, 0xbc, 0x07, 0x4b, 0xd7, 0x2e, 0x1a, 0xad, 0x17, 0x31, 0xe9, 0x68, 0x86,
|
||||
0xdd, 0xf7, 0xfd, 0xf2, 0x7b, 0x7f, 0xe8, 0x6e, 0x40, 0xd1, 0x0e, 0xc5, 0xcb, 0x88, 0x13, 0xc6,
|
||||
0x1b, 0x09, 0x46, 0xe0, 0x41, 0x8d, 0x57, 0xb7, 0x3d, 0x4c, 0xfb, 0xb9, 0x2b, 0x04, 0x3e, 0x97,
|
||||
0x3d, 0xf6, 0x58, 0x7a, 0xa1, 0x9b, 0x9f, 0x7c, 0xf3, 0xc5, 0xa7, 0xe3, 0xe0, 0xcd, 0x3b, 0xa1,
|
||||
0x9b, 0x07, 0xad, 0x71, 0xd6, 0x42, 0xc5, 0x0d, 0x0d, 0x41, 0x26, 0x24, 0x23, 0xf9, 0x79, 0x55,
|
||||
0x2d, 0x36, 0x0d, 0x3e, 0x6d, 0x7a, 0xbf, 0xda, 0x67, 0x5e, 0xb5, 0x98, 0xf6, 0xa0, 0xfb, 0x55,
|
||||
0x1a, 0xa0, 0x3b, 0x9e, 0x10, 0x38, 0x14, 0xb5, 0x3a, 0x80, 0x50, 0xbc, 0x76, 0x36, 0x0d, 0x79,
|
||||
0xdd, 0x84, 0x20, 0xe3, 0x6b, 0xba, 0x6d, 0xa5, 0x1c, 0x95, 0x31, 0xca, 0x24, 0x61, 0x16, 0xe5,
|
||||
0xdb, 0xe6, 0x04, 0xe2, 0x92, 0xee, 0x40, 0x9b, 0xa9, 0xd5, 0x42, 0x3d, 0x82, 0x4c, 0xa2, 0x8c,
|
||||
0xe4, 0x51, 0x75, 0xe1, 0x6c, 0x4a, 0xf9, 0x1f, 0xe6, 0x75, 0x43, 0xff, 0x15, 0x2e, 0xab, 0xcb,
|
||||
0xe5, 0x9b, 0x05, 0x8b, 0x63, 0xe4, 0xc3, 0x31, 0xf2, 0xe5, 0x58, 0xf0, 0xf6, 0xc3, 0x48, 0x77,
|
||||
0xe6, 0x3f, 0xb8, 0xfb, 0x0d, 0x00, 0x00, 0xff, 0xff, 0xa4, 0x46, 0x4f, 0x13, 0x14, 0x01, 0x00,
|
||||
0x00,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ import (
|
|||
"github.com/syncthing/syncthing/lib/symlinks"
|
||||
"github.com/syncthing/syncthing/lib/sync"
|
||||
"github.com/syncthing/syncthing/lib/versioner"
|
||||
"github.com/syncthing/syncthing/lib/weakhash"
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
|
@ -92,6 +93,7 @@ type rwFolder struct {
|
|||
checkFreeSpace bool
|
||||
ignoreDelete bool
|
||||
fsync bool
|
||||
useWeakHash bool
|
||||
|
||||
copiers int
|
||||
pullers int
|
||||
|
|
@ -128,6 +130,7 @@ func newRWFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Ver
|
|||
checkFreeSpace: cfg.MinDiskFreePct != 0,
|
||||
ignoreDelete: cfg.IgnoreDelete,
|
||||
fsync: cfg.Fsync,
|
||||
useWeakHash: !cfg.DisableWeakHash,
|
||||
|
||||
queue: newJobQueue(),
|
||||
pullTimer: time.NewTimer(time.Second),
|
||||
|
|
@ -1169,7 +1172,7 @@ func (f *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks
|
|||
created: time.Now(),
|
||||
}
|
||||
|
||||
l.Debugf("%v need file %s; copy %d, reused %v", f, file.Name, len(blocks), reused)
|
||||
l.Debugf("%v need file %s; copy %d, reused %v", f, file.Name, len(blocks), len(reused))
|
||||
|
||||
cs := copyBlocksState{
|
||||
sharedPullerState: &s,
|
||||
|
|
@ -1231,6 +1234,21 @@ func (f *rwFolder) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pull
|
|||
}
|
||||
f.model.fmut.RUnlock()
|
||||
|
||||
var weakHashFinder *weakhash.Finder
|
||||
if f.useWeakHash {
|
||||
hashesToFind := make([]uint32, 0, len(state.blocks))
|
||||
for _, block := range state.blocks {
|
||||
if block.WeakHash != 0 {
|
||||
hashesToFind = append(hashesToFind, block.WeakHash)
|
||||
}
|
||||
}
|
||||
|
||||
weakHashFinder, err = weakhash.NewFinder(state.realName, protocol.BlockSize, hashesToFind)
|
||||
if err != nil {
|
||||
l.Debugln("weak hasher", err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, block := range state.blocks {
|
||||
if f.allowSparse && state.reused == 0 && block.IsEmpty() {
|
||||
// The block is a block of all zeroes, and we are not reusing
|
||||
|
|
@ -1245,45 +1263,70 @@ func (f *rwFolder) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pull
|
|||
}
|
||||
|
||||
buf = buf[:int(block.Size)]
|
||||
found := f.model.finder.Iterate(folders, block.Hash, func(folder, file string, index int32) bool {
|
||||
inFile, err := rootedJoinedPath(folderRoots[folder], file)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
fd, err := os.Open(inFile)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
_, err = fd.ReadAt(buf, protocol.BlockSize*int64(index))
|
||||
fd.Close()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
hash, err := scanner.VerifyBuffer(buf, block)
|
||||
if err != nil {
|
||||
if hash != nil {
|
||||
l.Debugf("Finder block mismatch in %s:%s:%d expected %q got %q", folder, file, index, block.Hash, hash)
|
||||
err = f.model.finder.Fix(folder, file, index, block.Hash, hash)
|
||||
if err != nil {
|
||||
l.Warnln("finder fix:", err)
|
||||
}
|
||||
} else {
|
||||
l.Debugln("Finder failed to verify buffer", err)
|
||||
}
|
||||
return false
|
||||
found, err := weakHashFinder.Iterate(block.WeakHash, buf, func(offset int64) bool {
|
||||
if _, err := scanner.VerifyBuffer(buf, block); err != nil {
|
||||
return true
|
||||
}
|
||||
|
||||
_, err = dstFd.WriteAt(buf, block.Offset)
|
||||
if err != nil {
|
||||
state.fail("dst write", err)
|
||||
|
||||
}
|
||||
if file == state.file.Name {
|
||||
if offset == block.Offset {
|
||||
state.copiedFromOrigin()
|
||||
} else {
|
||||
state.copiedFromOriginShifted()
|
||||
}
|
||||
return true
|
||||
|
||||
return false
|
||||
})
|
||||
if err != nil {
|
||||
l.Debugln("weak hasher iter", err)
|
||||
}
|
||||
|
||||
if !found {
|
||||
found = f.model.finder.Iterate(folders, block.Hash, func(folder, file string, index int32) bool {
|
||||
inFile, err := rootedJoinedPath(folderRoots[folder], file)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
fd, err := os.Open(inFile)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
_, err = fd.ReadAt(buf, protocol.BlockSize*int64(index))
|
||||
fd.Close()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
hash, err := scanner.VerifyBuffer(buf, block)
|
||||
if err != nil {
|
||||
if hash != nil {
|
||||
l.Debugf("Finder block mismatch in %s:%s:%d expected %q got %q", folder, file, index, block.Hash, hash)
|
||||
err = f.model.finder.Fix(folder, file, index, block.Hash, hash)
|
||||
if err != nil {
|
||||
l.Warnln("finder fix:", err)
|
||||
}
|
||||
} else {
|
||||
l.Debugln("Finder failed to verify buffer", err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
_, err = dstFd.WriteAt(buf, block.Offset)
|
||||
if err != nil {
|
||||
state.fail("dst write", err)
|
||||
}
|
||||
if file == state.file.Name {
|
||||
state.copiedFromOrigin()
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
if state.failed() != nil {
|
||||
break
|
||||
|
|
@ -1300,6 +1343,7 @@ func (f *rwFolder) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pull
|
|||
state.copyDone(block)
|
||||
}
|
||||
}
|
||||
weakHashFinder.Close()
|
||||
out <- state.sharedPullerState
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,12 +7,15 @@
|
|||
package model
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/db"
|
||||
"github.com/syncthing/syncthing/lib/fs"
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
"github.com/syncthing/syncthing/lib/scanner"
|
||||
"github.com/syncthing/syncthing/lib/sync"
|
||||
|
|
@ -72,6 +75,8 @@ func setUpRwFolder(model *Model) rwFolder {
|
|||
stateTracker: newStateTracker("default"),
|
||||
model: model,
|
||||
},
|
||||
|
||||
mtimeFS: fs.NewMtimeFS(db.NewNamespacedKV(model.db, "mtime")),
|
||||
dir: "testdata",
|
||||
queue: newJobQueue(),
|
||||
errors: make(map[string]string),
|
||||
|
|
@ -199,7 +204,7 @@ func TestCopierFinder(t *testing.T) {
|
|||
|
||||
select {
|
||||
case <-pullChan:
|
||||
t.Fatal("Finisher channel has data to be read")
|
||||
t.Fatal("Pull channel has data to be read")
|
||||
case <-finisherChan:
|
||||
t.Fatal("Finisher channel has data to be read")
|
||||
default:
|
||||
|
|
@ -240,6 +245,132 @@ func TestCopierFinder(t *testing.T) {
|
|||
os.Remove(tempFile)
|
||||
}
|
||||
|
||||
func TestWeakHash(t *testing.T) {
|
||||
tempFile := filepath.Join("testdata", defTempNamer.TempName("weakhash"))
|
||||
var shift int64 = 10
|
||||
var size int64 = 1 << 20
|
||||
expectBlocks := int(size / protocol.BlockSize)
|
||||
expectPulls := int(shift / protocol.BlockSize)
|
||||
if shift > 0 {
|
||||
expectPulls++
|
||||
}
|
||||
|
||||
cleanup := func() {
|
||||
for _, path := range []string{tempFile, "testdata/weakhash"} {
|
||||
os.Remove(path)
|
||||
}
|
||||
}
|
||||
|
||||
cleanup()
|
||||
defer cleanup()
|
||||
|
||||
f, err := os.Create("testdata/weakhash")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
defer f.Close()
|
||||
_, err = io.CopyN(f, rand.Reader, size)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
// Create two files, second file has `shifted` bytes random prefix, yet
|
||||
// both are of the same length, for example:
|
||||
// File 1: abcdefgh
|
||||
// File 2: xyabcdef
|
||||
f.Seek(0, os.SEEK_SET)
|
||||
existing, err := scanner.Blocks(f, protocol.BlockSize, size, nil)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
f.Seek(0, os.SEEK_SET)
|
||||
remainder := io.LimitReader(f, size-shift)
|
||||
prefix := io.LimitReader(rand.Reader, shift)
|
||||
nf := io.MultiReader(prefix, remainder)
|
||||
desired, err := scanner.Blocks(nf, protocol.BlockSize, size, nil)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
existingFile := protocol.FileInfo{
|
||||
Name: "weakhash",
|
||||
Blocks: existing,
|
||||
Size: size,
|
||||
ModifiedS: info.ModTime().Unix(),
|
||||
ModifiedNs: int32(info.ModTime().Nanosecond()),
|
||||
}
|
||||
desiredFile := protocol.FileInfo{
|
||||
Name: "weakhash",
|
||||
Size: size,
|
||||
Blocks: desired,
|
||||
ModifiedS: info.ModTime().Unix() + 1,
|
||||
}
|
||||
|
||||
// Setup the model/pull environment
|
||||
m := setUpModel(existingFile)
|
||||
fo := setUpRwFolder(m)
|
||||
copyChan := make(chan copyBlocksState)
|
||||
pullChan := make(chan pullBlockState, expectBlocks)
|
||||
finisherChan := make(chan *sharedPullerState, 1)
|
||||
|
||||
// Run a single fetcher routine
|
||||
go fo.copierRoutine(copyChan, pullChan, finisherChan)
|
||||
|
||||
// Test 1 - no weak hashing, file gets fully repulled (`expectBlocks` pulls).
|
||||
fo.handleFile(desiredFile, copyChan, finisherChan)
|
||||
|
||||
var pulls []pullBlockState
|
||||
for len(pulls) < expectBlocks {
|
||||
select {
|
||||
case pull := <-pullChan:
|
||||
pulls = append(pulls, pull)
|
||||
case <-time.After(time.Second):
|
||||
t.Error("timed out")
|
||||
}
|
||||
}
|
||||
finish := <-finisherChan
|
||||
|
||||
select {
|
||||
case <-pullChan:
|
||||
t.Fatal("Pull channel has data to be read")
|
||||
case <-finisherChan:
|
||||
t.Fatal("Finisher channel has data to be read")
|
||||
default:
|
||||
}
|
||||
|
||||
finish.fd.Close()
|
||||
if err := os.Remove(tempFile); err != nil && !os.IsNotExist(err) {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
// Test 2 - using weak hash, expectPulls blocks pulled.
|
||||
fo.useWeakHash = true
|
||||
fo.handleFile(desiredFile, copyChan, finisherChan)
|
||||
|
||||
pulls = pulls[:0]
|
||||
for len(pulls) < expectPulls {
|
||||
select {
|
||||
case pull := <-pullChan:
|
||||
pulls = append(pulls, pull)
|
||||
case <-time.After(time.Second):
|
||||
t.Error("timed out")
|
||||
}
|
||||
}
|
||||
|
||||
finish = <-finisherChan
|
||||
finish.fd.Close()
|
||||
|
||||
expectShifted := expectBlocks - expectPulls
|
||||
if finish.copyOriginShifted != expectShifted {
|
||||
t.Errorf("did not copy %d shifted", expectShifted)
|
||||
}
|
||||
}
|
||||
|
||||
// Test that updating a file removes it's old blocks from the blockmap
|
||||
func TestCopierCleanup(t *testing.T) {
|
||||
iterFn := func(folder, file string, index int32) bool {
|
||||
|
|
|
|||
|
|
@ -31,30 +31,32 @@ type sharedPullerState struct {
|
|||
created time.Time
|
||||
|
||||
// Mutable, must be locked for access
|
||||
err error // The first error we hit
|
||||
fd *os.File // The fd of the temp file
|
||||
copyTotal int // Total number of copy actions for the whole job
|
||||
pullTotal int // Total number of pull actions for the whole job
|
||||
copyOrigin int // Number of blocks copied from the original file
|
||||
copyNeeded int // Number of copy actions still pending
|
||||
pullNeeded int // Number of block pulls still pending
|
||||
updated time.Time // Time when any of the counters above were last updated
|
||||
closed bool // True if the file has been finalClosed.
|
||||
available []int32 // Indexes of the blocks that are available in the temporary file
|
||||
availableUpdated time.Time // Time when list of available blocks was last updated
|
||||
mut sync.RWMutex // Protects the above
|
||||
err error // The first error we hit
|
||||
fd *os.File // The fd of the temp file
|
||||
copyTotal int // Total number of copy actions for the whole job
|
||||
pullTotal int // Total number of pull actions for the whole job
|
||||
copyOrigin int // Number of blocks copied from the original file
|
||||
copyOriginShifted int // Number of blocks copied from the original file but shifted
|
||||
copyNeeded int // Number of copy actions still pending
|
||||
pullNeeded int // Number of block pulls still pending
|
||||
updated time.Time // Time when any of the counters above were last updated
|
||||
closed bool // True if the file has been finalClosed.
|
||||
available []int32 // Indexes of the blocks that are available in the temporary file
|
||||
availableUpdated time.Time // Time when list of available blocks was last updated
|
||||
mut sync.RWMutex // Protects the above
|
||||
}
|
||||
|
||||
// A momentary state representing the progress of the puller
|
||||
type pullerProgress struct {
|
||||
Total int `json:"total"`
|
||||
Reused int `json:"reused"`
|
||||
CopiedFromOrigin int `json:"copiedFromOrigin"`
|
||||
CopiedFromElsewhere int `json:"copiedFromElsewhere"`
|
||||
Pulled int `json:"pulled"`
|
||||
Pulling int `json:"pulling"`
|
||||
BytesDone int64 `json:"bytesDone"`
|
||||
BytesTotal int64 `json:"bytesTotal"`
|
||||
Total int `json:"total"`
|
||||
Reused int `json:"reused"`
|
||||
CopiedFromOrigin int `json:"copiedFromOrigin"`
|
||||
CopiedFromOriginShifted int `json:"copiedFromOriginShifted"`
|
||||
CopiedFromElsewhere int `json:"copiedFromElsewhere"`
|
||||
Pulled int `json:"pulled"`
|
||||
Pulling int `json:"pulling"`
|
||||
BytesDone int64 `json:"bytesDone"`
|
||||
BytesTotal int64 `json:"bytesTotal"`
|
||||
}
|
||||
|
||||
// A lockedWriterAt synchronizes WriteAt calls with an external mutex.
|
||||
|
|
@ -241,6 +243,14 @@ func (s *sharedPullerState) copiedFromOrigin() {
|
|||
s.mut.Unlock()
|
||||
}
|
||||
|
||||
func (s *sharedPullerState) copiedFromOriginShifted() {
|
||||
s.mut.Lock()
|
||||
s.copyOrigin++
|
||||
s.copyOriginShifted++
|
||||
s.updated = time.Now()
|
||||
s.mut.Unlock()
|
||||
}
|
||||
|
||||
func (s *sharedPullerState) pullStarted() {
|
||||
s.mut.Lock()
|
||||
s.copyTotal--
|
||||
|
|
|
|||
|
|
@ -303,7 +303,7 @@ type FileInfo struct {
|
|||
NoPermissions bool `protobuf:"varint,8,opt,name=no_permissions,json=noPermissions,proto3" json:"no_permissions,omitempty"`
|
||||
Version Vector `protobuf:"bytes,9,opt,name=version" json:"version"`
|
||||
Sequence int64 `protobuf:"varint,10,opt,name=sequence,proto3" json:"sequence,omitempty"`
|
||||
Blocks []BlockInfo `protobuf:"bytes,16,rep,name=Blocks" json:"Blocks"`
|
||||
Blocks []BlockInfo `protobuf:"bytes,16,rep,name=Blocks,json=blocks" json:"Blocks"`
|
||||
SymlinkTarget string `protobuf:"bytes,17,opt,name=symlink_target,json=symlinkTarget,proto3" json:"symlink_target,omitempty"`
|
||||
}
|
||||
|
||||
|
|
@ -312,9 +312,10 @@ func (*FileInfo) ProtoMessage() {}
|
|||
func (*FileInfo) Descriptor() ([]byte, []int) { return fileDescriptorBep, []int{7} }
|
||||
|
||||
type BlockInfo struct {
|
||||
Offset int64 `protobuf:"varint,1,opt,name=offset,proto3" json:"offset,omitempty"`
|
||||
Size int32 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`
|
||||
Hash []byte `protobuf:"bytes,3,opt,name=hash,proto3" json:"hash,omitempty"`
|
||||
Offset int64 `protobuf:"varint,1,opt,name=offset,proto3" json:"offset,omitempty"`
|
||||
Size int32 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`
|
||||
Hash []byte `protobuf:"bytes,3,opt,name=hash,proto3" json:"hash,omitempty"`
|
||||
WeakHash uint32 `protobuf:"varint,4,opt,name=weak_hash,json=weakHash,proto3" json:"weak_hash,omitempty"`
|
||||
}
|
||||
|
||||
func (m *BlockInfo) Reset() { *m = BlockInfo{} }
|
||||
|
|
@ -913,6 +914,11 @@ func (m *BlockInfo) MarshalTo(data []byte) (int, error) {
|
|||
i = encodeVarintBep(data, i, uint64(len(m.Hash)))
|
||||
i += copy(data[i:], m.Hash)
|
||||
}
|
||||
if m.WeakHash != 0 {
|
||||
data[i] = 0x20
|
||||
i++
|
||||
i = encodeVarintBep(data, i, uint64(m.WeakHash))
|
||||
}
|
||||
return i, nil
|
||||
}
|
||||
|
||||
|
|
@ -1423,6 +1429,9 @@ func (m *BlockInfo) ProtoSize() (n int) {
|
|||
if l > 0 {
|
||||
n += 1 + l + sovBep(uint64(l))
|
||||
}
|
||||
if m.WeakHash != 0 {
|
||||
n += 1 + sovBep(uint64(m.WeakHash))
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
|
|
@ -3011,6 +3020,25 @@ func (m *BlockInfo) Unmarshal(data []byte) error {
|
|||
m.Hash = []byte{}
|
||||
}
|
||||
iNdEx = postIndex
|
||||
case 4:
|
||||
if wireType != 0 {
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field WeakHash", wireType)
|
||||
}
|
||||
m.WeakHash = 0
|
||||
for shift := uint(0); ; shift += 7 {
|
||||
if shift >= 64 {
|
||||
return ErrIntOverflowBep
|
||||
}
|
||||
if iNdEx >= l {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
b := data[iNdEx]
|
||||
iNdEx++
|
||||
m.WeakHash |= (uint32(b) & 0x7F) << shift
|
||||
if b < 0x80 {
|
||||
break
|
||||
}
|
||||
}
|
||||
default:
|
||||
iNdEx = preIndex
|
||||
skippy, err := skipBep(data[iNdEx:])
|
||||
|
|
@ -4029,108 +4057,112 @@ var (
|
|||
)
|
||||
|
||||
var fileDescriptorBep = []byte{
|
||||
// 1645 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x57, 0x4f, 0x73, 0xdb, 0x5a,
|
||||
0x15, 0x8f, 0x6d, 0xf9, 0xdf, 0xb5, 0x93, 0xe7, 0xdc, 0xa6, 0x79, 0x46, 0xcd, 0x4b, 0x82, 0xde,
|
||||
0x2b, 0x04, 0x0f, 0x4d, 0xa1, 0x05, 0x3a, 0xc3, 0x00, 0x33, 0x8e, 0xad, 0xa4, 0x9a, 0x3a, 0xb2,
|
||||
0x2b, 0xdb, 0x29, 0x65, 0x81, 0x46, 0xb6, 0xae, 0x1d, 0x4d, 0x64, 0x5d, 0x23, 0xc9, 0x6d, 0xc3,
|
||||
0x47, 0x80, 0x2f, 0xc0, 0x86, 0x99, 0x6e, 0xd9, 0xf3, 0x21, 0xca, 0xae, 0xd3, 0x25, 0x8b, 0x0e,
|
||||
0x94, 0x0d, 0x4b, 0x36, 0xec, 0x39, 0xf7, 0x5e, 0x49, 0x96, 0xf3, 0x87, 0xe9, 0xe2, 0x2d, 0x32,
|
||||
0xd6, 0x3d, 0xe7, 0x77, 0xcf, 0xb9, 0xe7, 0x77, 0xfe, 0xdc, 0x1b, 0x54, 0x1e, 0x91, 0xf9, 0xe1,
|
||||
0xdc, 0xa7, 0x21, 0xc5, 0x25, 0xfe, 0x33, 0xa6, 0xae, 0xfc, 0x60, 0xea, 0x84, 0xe7, 0x8b, 0xd1,
|
||||
0xe1, 0x98, 0xce, 0x1e, 0x4e, 0xe9, 0x94, 0x3e, 0xe4, 0x9a, 0xd1, 0x62, 0xc2, 0x57, 0x7c, 0xc1,
|
||||
0xbf, 0xc4, 0x46, 0x65, 0x8e, 0xf2, 0x4f, 0x89, 0xeb, 0x52, 0xbc, 0x87, 0x2a, 0x36, 0x79, 0xe5,
|
||||
0x8c, 0x89, 0xe9, 0x59, 0x33, 0x52, 0xcf, 0xec, 0x67, 0x0e, 0xca, 0x06, 0x12, 0x22, 0x1d, 0x24,
|
||||
0x0c, 0x30, 0x76, 0x1d, 0xe2, 0x85, 0x02, 0x90, 0x15, 0x00, 0x21, 0xe2, 0x80, 0xfb, 0x68, 0x23,
|
||||
0x02, 0xbc, 0x22, 0x7e, 0xe0, 0x50, 0xaf, 0x9e, 0xe3, 0x98, 0x75, 0x21, 0x3d, 0x13, 0x42, 0x25,
|
||||
0x40, 0x85, 0xa7, 0xc4, 0xb2, 0x89, 0x8f, 0x7f, 0x80, 0xa4, 0xf0, 0x72, 0x2e, 0x7c, 0x6d, 0x3c,
|
||||
0xba, 0x7b, 0x18, 0xc7, 0x70, 0x78, 0x4a, 0x82, 0xc0, 0x9a, 0x92, 0x01, 0x28, 0x0d, 0x0e, 0xc1,
|
||||
0xbf, 0x02, 0xe7, 0x74, 0x36, 0xf7, 0x41, 0xc1, 0x0c, 0x67, 0xf9, 0x8e, 0x9d, 0x6b, 0x3b, 0x5a,
|
||||
0x4b, 0x8c, 0x91, 0xde, 0xa0, 0x34, 0xd1, 0x7a, 0xcb, 0x5d, 0x04, 0x21, 0xf1, 0x5b, 0xd4, 0x9b,
|
||||
0x38, 0x53, 0xfc, 0x23, 0x54, 0x9c, 0x50, 0x17, 0x4e, 0x11, 0x80, 0xfb, 0xdc, 0x41, 0xe5, 0x51,
|
||||
0x6d, 0x69, 0xec, 0x98, 0x2b, 0x8e, 0xa4, 0x77, 0x1f, 0xf7, 0xd6, 0x8c, 0x18, 0xa6, 0xfc, 0x31,
|
||||
0x8b, 0x0a, 0x42, 0x83, 0xb7, 0x51, 0xd6, 0xb1, 0x05, 0x45, 0x47, 0x85, 0x4f, 0x1f, 0xf7, 0xb2,
|
||||
0x5a, 0xdb, 0x00, 0x09, 0xde, 0x42, 0x79, 0xd7, 0x1a, 0x11, 0x37, 0x22, 0x47, 0x2c, 0xf0, 0x3d,
|
||||
0x54, 0xf6, 0x21, 0x60, 0x93, 0x7a, 0xee, 0x25, 0xa7, 0xa4, 0x64, 0x94, 0x98, 0xa0, 0x0b, 0x6b,
|
||||
0xfc, 0x00, 0x61, 0x67, 0xea, 0x51, 0x9f, 0x98, 0x73, 0xe2, 0xcf, 0x1c, 0x7e, 0xda, 0xa0, 0x2e,
|
||||
0x71, 0xd4, 0xa6, 0xd0, 0xf4, 0x96, 0x0a, 0xfc, 0x35, 0x5a, 0x8f, 0xe0, 0x36, 0x71, 0x49, 0x48,
|
||||
0xea, 0x79, 0x8e, 0xac, 0x0a, 0x61, 0x9b, 0xcb, 0x20, 0xb6, 0x2d, 0xdb, 0x09, 0xac, 0x91, 0x4b,
|
||||
0xcc, 0x90, 0xcc, 0xe6, 0xa6, 0xe3, 0xd9, 0xe4, 0x0d, 0x09, 0xea, 0x05, 0x8e, 0xc5, 0x91, 0x6e,
|
||||
0x00, 0x2a, 0x4d, 0x68, 0x18, 0x1b, 0x22, 0xd3, 0x41, 0xbd, 0x76, 0x95, 0x8d, 0x36, 0x57, 0xc4,
|
||||
0x6c, 0x44, 0x30, 0xe5, 0x3f, 0xc0, 0x86, 0xd0, 0xe0, 0xef, 0x25, 0x6c, 0x54, 0x8f, 0xb6, 0x19,
|
||||
0xea, 0xef, 0x1f, 0xf7, 0x4a, 0x42, 0xa7, 0xb5, 0x53, 0xec, 0x60, 0x24, 0xa5, 0x2a, 0x87, 0x7f,
|
||||
0xe3, 0x1d, 0x54, 0xb6, 0x6c, 0x9b, 0x65, 0x09, 0x5c, 0xe7, 0xc0, 0x75, 0xd9, 0x58, 0x0a, 0xf0,
|
||||
0x93, 0xd5, 0xac, 0x4b, 0x57, 0xeb, 0xe4, 0xb6, 0x74, 0x33, 0xca, 0xc7, 0xc4, 0x8f, 0x2a, 0x35,
|
||||
0xcf, 0xfd, 0x95, 0x98, 0x80, 0xd7, 0xe9, 0x77, 0x51, 0x75, 0x66, 0xbd, 0x31, 0x03, 0xf2, 0xbb,
|
||||
0x05, 0xf1, 0xc6, 0x84, 0xd3, 0x92, 0x33, 0x2a, 0x20, 0xeb, 0x47, 0x22, 0xbc, 0x8b, 0x90, 0xe3,
|
||||
0x85, 0x3e, 0xb5, 0x17, 0xb0, 0xab, 0x5e, 0xe4, 0xbc, 0xa5, 0x24, 0xf8, 0xa7, 0xa8, 0xc4, 0x49,
|
||||
0x35, 0x21, 0xf0, 0x12, 0x68, 0xa5, 0x23, 0x39, 0x0a, 0xbc, 0xc8, 0x29, 0xe5, 0x71, 0xc7, 0x9f,
|
||||
0x46, 0x91, 0x63, 0x35, 0x1b, 0xff, 0x02, 0xc9, 0xc1, 0x85, 0xc3, 0x12, 0x22, 0x2c, 0x85, 0x70,
|
||||
0x56, 0xd3, 0x27, 0x33, 0xfa, 0xca, 0x72, 0x83, 0x7a, 0x99, 0xbb, 0xa9, 0x33, 0x84, 0x96, 0x02,
|
||||
0x18, 0x91, 0x5e, 0xe9, 0xa2, 0x3c, 0xb7, 0x08, 0xe5, 0x57, 0x10, 0x45, 0x19, 0x75, 0x69, 0xb4,
|
||||
0xc2, 0x87, 0x28, 0x3f, 0x71, 0x5c, 0x20, 0x32, 0xcb, 0x73, 0x88, 0x53, 0x15, 0x0d, 0x62, 0xcd,
|
||||
0x9b, 0xd0, 0x28, 0x8b, 0x02, 0xa6, 0x0c, 0x51, 0x85, 0x1b, 0x1c, 0xce, 0x6d, 0x0b, 0xca, 0xe6,
|
||||
0xdb, 0x32, 0xfb, 0xb7, 0x1c, 0x2a, 0xc5, 0x9a, 0x24, 0xe9, 0x99, 0x54, 0xd2, 0x1b, 0x51, 0xdf,
|
||||
0x8b, 0x2e, 0xde, 0xbe, 0x6e, 0x2f, 0xd5, 0xf8, 0xb0, 0x3f, 0x70, 0x7e, 0x4f, 0x78, 0xdf, 0xe4,
|
||||
0x0c, 0xfe, 0x8d, 0xf7, 0x51, 0xe5, 0x6a, 0xb3, 0xac, 0x1b, 0x69, 0x11, 0xfe, 0x0a, 0xa1, 0x19,
|
||||
0xb5, 0x9d, 0x89, 0x43, 0x6c, 0x33, 0xe0, 0x05, 0x90, 0x33, 0xca, 0xb1, 0xa4, 0x8f, 0xeb, 0xac,
|
||||
0xdc, 0x59, 0xab, 0xd8, 0x51, 0x4f, 0xc4, 0x4b, 0xa6, 0x71, 0x3c, 0x60, 0x1b, 0xf2, 0x2a, 0xb2,
|
||||
0x1e, 0x2f, 0xd9, 0x74, 0xf3, 0xe8, 0x4a, 0x93, 0x96, 0x38, 0x60, 0xdd, 0xa3, 0xe9, 0x06, 0x85,
|
||||
0x4e, 0x8a, 0xa7, 0x1f, 0xcb, 0xe7, 0x4a, 0x27, 0x9d, 0x91, 0x71, 0x48, 0x93, 0xb9, 0x12, 0xc1,
|
||||
0xb0, 0x8c, 0x4a, 0x49, 0x29, 0x22, 0x7e, 0xd2, 0x64, 0xcd, 0x66, 0x6e, 0x12, 0x07, 0x78, 0xac,
|
||||
0x80, 0x3a, 0x6f, 0x24, 0xa1, 0xe9, 0x01, 0xfe, 0x31, 0x2a, 0x1c, 0xb9, 0x74, 0x7c, 0x11, 0xf7,
|
||||
0xed, 0x9d, 0xa5, 0x37, 0x2e, 0x4f, 0x65, 0x27, 0x02, 0xb2, 0x40, 0x82, 0xcb, 0x99, 0xeb, 0x78,
|
||||
0x17, 0x66, 0x68, 0xf9, 0x53, 0x12, 0xd6, 0x37, 0xc5, 0x98, 0x8e, 0xa4, 0x03, 0x2e, 0xfc, 0xb9,
|
||||
0xf4, 0xa7, 0xb7, 0x7b, 0x6b, 0xca, 0x73, 0x54, 0x4e, 0xec, 0xb0, 0x02, 0xa1, 0x93, 0x49, 0x00,
|
||||
0x3b, 0x32, 0xfc, 0x9c, 0xd1, 0x2a, 0xc9, 0x51, 0x96, 0x1f, 0x4f, 0xe4, 0x08, 0x64, 0xe7, 0x56,
|
||||
0x70, 0xce, 0xf3, 0x56, 0x35, 0xf8, 0x77, 0x64, 0xf2, 0x97, 0xa8, 0x20, 0x88, 0xc0, 0x8f, 0x51,
|
||||
0x69, 0x4c, 0x17, 0x5e, 0xb8, 0x1c, 0xc2, 0x9b, 0xe9, 0xde, 0xe6, 0x9a, 0xe8, 0xf0, 0x09, 0x50,
|
||||
0x39, 0x46, 0xc5, 0x48, 0x05, 0x91, 0xc4, 0x83, 0x47, 0x3a, 0xba, 0x1b, 0xf7, 0x5f, 0xff, 0x9c,
|
||||
0xfa, 0xe1, 0xca, 0xdc, 0x81, 0xa9, 0x0c, 0x29, 0x5c, 0x88, 0xf3, 0x49, 0x86, 0x58, 0x28, 0x7f,
|
||||
0xcd, 0xa0, 0xa2, 0xc1, 0x78, 0x0e, 0xc2, 0xd4, 0x3c, 0xcf, 0xaf, 0xcc, 0xf3, 0x65, 0x47, 0x64,
|
||||
0x57, 0x3a, 0x22, 0x2e, 0xea, 0x5c, 0xaa, 0xa8, 0x97, 0xe4, 0x48, 0x37, 0x92, 0x93, 0xbf, 0x81,
|
||||
0x9c, 0xc2, 0x92, 0x1c, 0x96, 0x96, 0x89, 0x4f, 0x67, 0x7c, 0x62, 0x53, 0xdf, 0xf2, 0x2f, 0xa3,
|
||||
0x02, 0x5c, 0x67, 0xd2, 0x41, 0x2c, 0x54, 0x4c, 0x54, 0x32, 0x48, 0x30, 0x87, 0x52, 0x23, 0xb7,
|
||||
0x1e, 0x1b, 0xcc, 0x43, 0x43, 0x5b, 0xfc, 0xd0, 0x60, 0x9e, 0x7d, 0xe3, 0xef, 0x23, 0x69, 0x4c,
|
||||
0x6d, 0x71, 0xe4, 0x8d, 0x74, 0x99, 0xa8, 0xbe, 0x4f, 0xe1, 0x52, 0xb4, 0xa1, 0xe1, 0x18, 0x00,
|
||||
0x1e, 0x04, 0xb5, 0x36, 0x7d, 0xed, 0xb9, 0xd4, 0xb2, 0x7b, 0x3e, 0x9d, 0xb2, 0x89, 0x7a, 0xeb,
|
||||
0x64, 0x68, 0xa3, 0xe2, 0x82, 0xcf, 0x8e, 0x78, 0x36, 0x7c, 0xb3, 0xda, 0xcb, 0x57, 0x0d, 0x89,
|
||||
0x41, 0x13, 0x37, 0x40, 0xb4, 0x55, 0xf9, 0x90, 0x41, 0xf2, 0xed, 0x68, 0xac, 0xa1, 0x8a, 0x40,
|
||||
0x9a, 0xa9, 0xc7, 0xc2, 0xc1, 0xe7, 0x38, 0xe2, 0x63, 0x04, 0x2d, 0x92, 0xef, 0x1b, 0x6f, 0xa0,
|
||||
0x54, 0xc3, 0xe6, 0x3e, 0xaf, 0x61, 0xe1, 0x0e, 0x1e, 0xb1, 0x9e, 0x48, 0xee, 0x55, 0x09, 0x62,
|
||||
0xcf, 0x1b, 0xd5, 0x91, 0x68, 0x14, 0x2e, 0x53, 0x0a, 0x48, 0xea, 0x39, 0xde, 0x54, 0xd9, 0x43,
|
||||
0xf9, 0x96, 0x4b, 0x79, 0xb2, 0x0a, 0x70, 0xe9, 0x07, 0xe0, 0x26, 0xe2, 0x50, 0xac, 0x1a, 0x1f,
|
||||
0xb2, 0xa8, 0x92, 0x7a, 0xef, 0xc0, 0x79, 0x36, 0x5a, 0x9d, 0x61, 0x7f, 0xa0, 0x1a, 0x66, 0xab,
|
||||
0xab, 0x1f, 0x6b, 0x27, 0xb5, 0x35, 0x79, 0xe7, 0x0f, 0x7f, 0xde, 0xaf, 0xcf, 0x96, 0xa0, 0xd5,
|
||||
0xa7, 0x0c, 0xb8, 0xd0, 0xf4, 0xb6, 0xfa, 0xeb, 0x5a, 0x46, 0xde, 0x02, 0x60, 0x2d, 0x05, 0x14,
|
||||
0xf7, 0xc5, 0x0f, 0x51, 0x95, 0x03, 0xcc, 0x61, 0xaf, 0xdd, 0x1c, 0xa8, 0xb5, 0xac, 0x2c, 0x03,
|
||||
0x6e, 0xfb, 0x2a, 0x2e, 0xe2, 0xfb, 0x6b, 0xe8, 0x0b, 0xf5, 0xf9, 0x50, 0xed, 0x0f, 0x6a, 0x39,
|
||||
0x79, 0x1b, 0x80, 0x38, 0x05, 0x8c, 0x3b, 0xe6, 0x3e, 0x94, 0xa1, 0xda, 0xef, 0x75, 0xf5, 0xbe,
|
||||
0x5a, 0x93, 0xe4, 0x2f, 0x01, 0x75, 0x67, 0x05, 0x15, 0x55, 0xe8, 0xcf, 0xd0, 0x66, 0xbb, 0xfb,
|
||||
0x42, 0xef, 0x74, 0x9b, 0x6d, 0xb3, 0x67, 0x74, 0x4f, 0x60, 0x4f, 0xbf, 0x96, 0x97, 0xf7, 0x00,
|
||||
0x7f, 0x2f, 0x85, 0xbf, 0x56, 0x70, 0x5f, 0x01, 0x7b, 0x9a, 0x7e, 0x52, 0x2b, 0xc8, 0x77, 0x00,
|
||||
0xfa, 0x45, 0x0a, 0xca, 0x48, 0x65, 0x11, 0xb7, 0x3a, 0x5d, 0x70, 0x5d, 0xbc, 0x16, 0x31, 0x27,
|
||||
0xbb, 0xf1, 0x5b, 0x84, 0xaf, 0xbf, 0x08, 0xf1, 0x37, 0x48, 0xd2, 0xbb, 0xba, 0x0a, 0x84, 0xf2,
|
||||
0xf8, 0xaf, 0x23, 0x74, 0xea, 0x11, 0xac, 0xa0, 0x5c, 0xe7, 0x37, 0x3f, 0x01, 0x32, 0xbf, 0x03,
|
||||
0xa0, 0xbb, 0xd7, 0x41, 0xa0, 0x6c, 0x50, 0x54, 0x49, 0x1b, 0x56, 0x50, 0xe9, 0x54, 0x1d, 0x34,
|
||||
0x81, 0xdc, 0x26, 0x18, 0xe7, 0x47, 0x8a, 0xd5, 0xa7, 0x24, 0xb4, 0x78, 0x03, 0xee, 0xa0, 0xbc,
|
||||
0xae, 0x9e, 0xa9, 0x06, 0x18, 0xde, 0x04, 0xc0, 0x7a, 0x0c, 0xd0, 0x09, 0xd4, 0x15, 0x3c, 0x38,
|
||||
0x0a, 0xcd, 0xce, 0x8b, 0xe6, 0xcb, 0x3e, 0x24, 0x07, 0x83, 0x7a, 0x23, 0x56, 0x37, 0xdd, 0xd7,
|
||||
0xd6, 0x65, 0xd0, 0xf8, 0x6f, 0x06, 0x55, 0xd3, 0xb7, 0x23, 0x6c, 0x90, 0x8e, 0xb5, 0x8e, 0x1a,
|
||||
0xbb, 0x4b, 0xeb, 0xd8, 0x37, 0x3e, 0x40, 0xe5, 0xb6, 0x66, 0xa8, 0xad, 0x41, 0xd7, 0x78, 0x19,
|
||||
0xc7, 0x92, 0x06, 0xb5, 0x1d, 0x9f, 0x17, 0x37, 0x7b, 0x81, 0x56, 0xfb, 0x2f, 0x4f, 0x3b, 0x9a,
|
||||
0xfe, 0xcc, 0xe4, 0x16, 0xb3, 0xf2, 0x3d, 0x00, 0x7f, 0x99, 0x06, 0xf7, 0xc5, 0xcd, 0xc0, 0x0d,
|
||||
0x3f, 0x41, 0x9b, 0x31, 0x7c, 0xe9, 0x20, 0x27, 0xef, 0xc3, 0x9e, 0x9d, 0x1b, 0xf6, 0x2c, 0xfd,
|
||||
0x3c, 0x46, 0x5f, 0xc4, 0x1b, 0x87, 0xfa, 0x33, 0x1d, 0xca, 0x02, 0x2a, 0x67, 0x17, 0xb6, 0xc9,
|
||||
0x37, 0x6c, 0x1b, 0x7a, 0x17, 0x1e, 0x14, 0x45, 0xe3, 0x2f, 0x19, 0x54, 0x4e, 0x26, 0x14, 0xe3,
|
||||
0x59, 0xef, 0x9a, 0xaa, 0x61, 0x74, 0x8d, 0x38, 0xf0, 0x44, 0xa9, 0x53, 0xfe, 0x09, 0xaf, 0xbb,
|
||||
0xe2, 0x89, 0xaa, 0xab, 0x86, 0xd6, 0x8a, 0xfb, 0x21, 0x81, 0x9c, 0x10, 0x8f, 0xf8, 0xce, 0x18,
|
||||
0xfe, 0xef, 0xa8, 0x82, 0x99, 0xfe, 0xb0, 0xf5, 0x34, 0x8e, 0x98, 0x17, 0x70, 0xca, 0x54, 0x7f,
|
||||
0x31, 0x3e, 0xe7, 0xd1, 0x36, 0x58, 0xeb, 0x9c, 0x35, 0x3b, 0x5a, 0x5b, 0x40, 0x73, 0x72, 0x1d,
|
||||
0xa0, 0x5b, 0x09, 0x54, 0x13, 0xaf, 0x03, 0x86, 0x6d, 0xd8, 0x68, 0xf7, 0xff, 0xcf, 0x22, 0x78,
|
||||
0xb8, 0x14, 0x9a, 0xbd, 0x9e, 0xaa, 0xb7, 0xe3, 0xd3, 0x2f, 0x75, 0xcd, 0xf9, 0x9c, 0x78, 0x36,
|
||||
0x43, 0x1c, 0x77, 0x8d, 0x13, 0x75, 0x10, 0x1f, 0x7e, 0x89, 0x38, 0xa6, 0xec, 0x5e, 0x3e, 0xda,
|
||||
0x79, 0xf7, 0xcf, 0xdd, 0xb5, 0xf7, 0xf0, 0xf7, 0xee, 0xd3, 0x6e, 0xe6, 0x3d, 0xfc, 0xfd, 0xe3,
|
||||
0xd3, 0xee, 0xda, 0xbf, 0xe1, 0xf7, 0xed, 0xbf, 0x76, 0x33, 0xa3, 0x02, 0x9f, 0x5d, 0x8f, 0xff,
|
||||
0x17, 0x00, 0x00, 0xff, 0xff, 0x20, 0x74, 0xd7, 0x8f, 0x1b, 0x0e, 0x00, 0x00,
|
||||
// 1700 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x56, 0x4f, 0x73, 0xdb, 0xc6,
|
||||
0x15, 0x17, 0x48, 0xf0, 0xdf, 0x23, 0xa5, 0x40, 0x6b, 0x59, 0x41, 0x61, 0x85, 0x42, 0xe0, 0xb8,
|
||||
0x55, 0x34, 0x8d, 0xe2, 0xc6, 0x69, 0x33, 0xd3, 0x69, 0x3b, 0x43, 0x91, 0x90, 0xcc, 0x09, 0x0d,
|
||||
0xb2, 0x4b, 0xca, 0xae, 0x73, 0x28, 0x06, 0x24, 0x96, 0x14, 0x46, 0x20, 0x96, 0x05, 0x40, 0xd9,
|
||||
0xea, 0x47, 0x60, 0xbf, 0x40, 0x2f, 0x9c, 0xc9, 0xb5, 0xf7, 0x7e, 0x08, 0xf7, 0x96, 0xc9, 0xb1,
|
||||
0x07, 0x4f, 0xa3, 0x5e, 0x7a, 0xec, 0xa5, 0xf7, 0x0e, 0x76, 0x01, 0x10, 0x94, 0xe4, 0x8e, 0x0f,
|
||||
0x3d, 0x71, 0xf7, 0xbd, 0xdf, 0xbe, 0xdd, 0xf7, 0x7b, 0xef, 0xf7, 0x08, 0xa8, 0x0c, 0xc9, 0xec,
|
||||
0x68, 0xe6, 0xd3, 0x90, 0xa2, 0x32, 0xfb, 0x19, 0x51, 0x57, 0xf9, 0x6c, 0xe2, 0x84, 0xe7, 0xf3,
|
||||
0xe1, 0xd1, 0x88, 0x4e, 0x3f, 0x9f, 0xd0, 0x09, 0xfd, 0x9c, 0x79, 0x86, 0xf3, 0x31, 0xdb, 0xb1,
|
||||
0x0d, 0x5b, 0xf1, 0x83, 0xda, 0x0c, 0x0a, 0x4f, 0x89, 0xeb, 0x52, 0xb4, 0x0f, 0x55, 0x9b, 0x5c,
|
||||
0x3a, 0x23, 0x62, 0x7a, 0xd6, 0x94, 0xc8, 0x82, 0x2a, 0x1c, 0x54, 0x30, 0x70, 0x93, 0x61, 0x4d,
|
||||
0x49, 0x04, 0x18, 0xb9, 0x0e, 0xf1, 0x42, 0x0e, 0xc8, 0x71, 0x00, 0x37, 0x31, 0xc0, 0x23, 0xd8,
|
||||
0x8a, 0x01, 0x97, 0xc4, 0x0f, 0x1c, 0xea, 0xc9, 0x79, 0x86, 0xd9, 0xe4, 0xd6, 0xe7, 0xdc, 0xa8,
|
||||
0x05, 0x50, 0x7c, 0x4a, 0x2c, 0x9b, 0xf8, 0xe8, 0x53, 0x10, 0xc3, 0xab, 0x19, 0xbf, 0x6b, 0xeb,
|
||||
0x8b, 0xfb, 0x47, 0x49, 0x0e, 0x47, 0xcf, 0x48, 0x10, 0x58, 0x13, 0x32, 0xb8, 0x9a, 0x11, 0xcc,
|
||||
0x20, 0xe8, 0x37, 0x50, 0x1d, 0xd1, 0xe9, 0xcc, 0x27, 0x01, 0x0b, 0x9c, 0x63, 0x27, 0xf6, 0x6e,
|
||||
0x9d, 0x68, 0xae, 0x30, 0x38, 0x7b, 0x40, 0x6b, 0xc0, 0x66, 0xd3, 0x9d, 0x07, 0x21, 0xf1, 0x9b,
|
||||
0xd4, 0x1b, 0x3b, 0x13, 0xf4, 0x18, 0x4a, 0x63, 0xea, 0xda, 0xc4, 0x0f, 0x64, 0x41, 0xcd, 0x1f,
|
||||
0x54, 0xbf, 0x90, 0x56, 0xc1, 0x4e, 0x98, 0xe3, 0x58, 0x7c, 0xf3, 0x76, 0x7f, 0x03, 0x27, 0x30,
|
||||
0xed, 0x4f, 0x39, 0x28, 0x72, 0x0f, 0xda, 0x85, 0x9c, 0x63, 0x73, 0x8a, 0x8e, 0x8b, 0xd7, 0x6f,
|
||||
0xf7, 0x73, 0xed, 0x16, 0xce, 0x39, 0x36, 0xda, 0x81, 0x82, 0x6b, 0x0d, 0x89, 0x1b, 0x93, 0xc3,
|
||||
0x37, 0xe8, 0x01, 0x54, 0x7c, 0x62, 0xd9, 0x26, 0xf5, 0xdc, 0x2b, 0x46, 0x49, 0x19, 0x97, 0x23,
|
||||
0x43, 0xd7, 0x73, 0xaf, 0xd0, 0x67, 0x80, 0x9c, 0x89, 0x47, 0x7d, 0x62, 0xce, 0x88, 0x3f, 0x75,
|
||||
0xd8, 0x6b, 0x03, 0x59, 0x64, 0xa8, 0x6d, 0xee, 0xe9, 0xad, 0x1c, 0xe8, 0x21, 0x6c, 0xc6, 0x70,
|
||||
0x9b, 0xb8, 0x24, 0x24, 0x72, 0x81, 0x21, 0x6b, 0xdc, 0xd8, 0x62, 0x36, 0xf4, 0x18, 0x76, 0x6c,
|
||||
0x27, 0xb0, 0x86, 0x2e, 0x31, 0x43, 0x32, 0x9d, 0x99, 0x8e, 0x67, 0x93, 0xd7, 0x24, 0x90, 0x8b,
|
||||
0x0c, 0x8b, 0x62, 0xdf, 0x80, 0x4c, 0x67, 0x6d, 0xee, 0x89, 0xd8, 0xe0, 0x95, 0x0e, 0x64, 0xe9,
|
||||
0x26, 0x1b, 0x2d, 0xe6, 0x48, 0xd8, 0x88, 0x61, 0xda, 0xbf, 0x73, 0x50, 0xe4, 0x1e, 0xf4, 0xe3,
|
||||
0x94, 0x8d, 0xda, 0xf1, 0x6e, 0x84, 0xfa, 0xfb, 0xdb, 0xfd, 0x32, 0xf7, 0xb5, 0x5b, 0x19, 0x76,
|
||||
0x10, 0x88, 0x99, 0xce, 0x61, 0x6b, 0xb4, 0x07, 0x15, 0xcb, 0xb6, 0xa3, 0x2a, 0x91, 0x40, 0xce,
|
||||
0xab, 0xf9, 0x83, 0x0a, 0x5e, 0x19, 0xd0, 0x57, 0xeb, 0x55, 0x17, 0x6f, 0xf6, 0xc9, 0xbb, 0xca,
|
||||
0x1d, 0x51, 0x3e, 0x22, 0x7e, 0xdc, 0xa9, 0x05, 0x76, 0x5f, 0x39, 0x32, 0xb0, 0x3e, 0xfd, 0x18,
|
||||
0x6a, 0x53, 0xeb, 0xb5, 0x19, 0x90, 0x3f, 0xcc, 0x89, 0x37, 0x22, 0x8c, 0x96, 0x3c, 0xae, 0x4e,
|
||||
0xad, 0xd7, 0xfd, 0xd8, 0x84, 0xea, 0x00, 0x8e, 0x17, 0xfa, 0xd4, 0x9e, 0x8f, 0x88, 0x2f, 0x97,
|
||||
0x18, 0x6f, 0x19, 0x0b, 0xfa, 0x39, 0x94, 0x19, 0xa9, 0xa6, 0x63, 0xcb, 0x65, 0x55, 0x38, 0x10,
|
||||
0x8f, 0x95, 0x38, 0xf1, 0x12, 0xa3, 0x94, 0xe5, 0x9d, 0x2c, 0x71, 0x89, 0x61, 0xdb, 0x36, 0xfa,
|
||||
0x15, 0x28, 0xc1, 0x85, 0x13, 0x15, 0x84, 0x47, 0x0a, 0x1d, 0xea, 0x99, 0x3e, 0x99, 0xd2, 0x4b,
|
||||
0xcb, 0x0d, 0xe4, 0x0a, 0xbb, 0x46, 0x8e, 0x10, 0xed, 0x0c, 0x00, 0xc7, 0x7e, 0xad, 0x0b, 0x05,
|
||||
0x16, 0x11, 0xed, 0x42, 0x91, 0x37, 0x65, 0xac, 0xd2, 0x78, 0x87, 0x8e, 0xa0, 0x30, 0x76, 0x5c,
|
||||
0x12, 0xc8, 0x39, 0x56, 0x43, 0x94, 0xe9, 0x68, 0xc7, 0x25, 0x6d, 0x6f, 0x4c, 0xe3, 0x2a, 0x72,
|
||||
0x98, 0x76, 0x06, 0x55, 0x16, 0xf0, 0x6c, 0x66, 0x5b, 0x21, 0xf9, 0xbf, 0x85, 0xfd, 0x5b, 0x1e,
|
||||
0xca, 0x89, 0x27, 0x2d, 0xba, 0x90, 0x29, 0xfa, 0x61, 0xac, 0x7b, 0xae, 0xe2, 0xdd, 0xdb, 0xf1,
|
||||
0x32, 0xc2, 0x47, 0x20, 0x06, 0xce, 0x1f, 0x09, 0xd3, 0x4d, 0x1e, 0xb3, 0x35, 0x52, 0xa1, 0x7a,
|
||||
0x53, 0x2c, 0x9b, 0x38, 0x6b, 0x42, 0x1f, 0x01, 0x4c, 0xa9, 0xed, 0x8c, 0x1d, 0x62, 0x9b, 0x01,
|
||||
0x6b, 0x80, 0x3c, 0xae, 0x24, 0x96, 0x3e, 0x92, 0xa3, 0x76, 0x8f, 0xa4, 0x62, 0xc7, 0x9a, 0x48,
|
||||
0xb6, 0x91, 0xc7, 0xf1, 0x2e, 0x2d, 0xd7, 0xb1, 0xe3, 0xaa, 0x27, 0xdb, 0x68, 0xba, 0x79, 0x74,
|
||||
0x4d, 0xa4, 0x65, 0x06, 0xd8, 0xf4, 0x68, 0x56, 0xa0, 0x8f, 0xa1, 0x94, 0x4c, 0xbf, 0xa8, 0x9e,
|
||||
0x6b, 0x4a, 0x7a, 0x4e, 0x46, 0x21, 0x4d, 0xe7, 0x4a, 0x0c, 0x43, 0x0a, 0x94, 0xd3, 0x56, 0x04,
|
||||
0xf6, 0xd2, 0x74, 0x1f, 0xcd, 0xdc, 0x34, 0x0f, 0x2f, 0x90, 0xab, 0xaa, 0x70, 0x50, 0xc0, 0x69,
|
||||
0x6a, 0x46, 0x80, 0x7e, 0x06, 0xc5, 0x63, 0x97, 0x8e, 0x2e, 0x12, 0xdd, 0xde, 0x5b, 0xdd, 0xc6,
|
||||
0xec, 0x99, 0xea, 0x14, 0x87, 0x0c, 0x18, 0x25, 0x12, 0x5c, 0x4d, 0x5d, 0xc7, 0xbb, 0x30, 0x43,
|
||||
0xcb, 0x9f, 0x90, 0x50, 0xde, 0xe6, 0x63, 0x3a, 0xb6, 0x0e, 0x98, 0xf1, 0x97, 0xe2, 0x9f, 0xbf,
|
||||
0xdd, 0xdf, 0xd0, 0x3c, 0xa8, 0xa4, 0x71, 0xa2, 0x06, 0xa1, 0xe3, 0x71, 0x40, 0x42, 0x56, 0xcd,
|
||||
0x3c, 0x8e, 0x77, 0x69, 0x8d, 0x72, 0xec, 0x79, 0xbc, 0x46, 0x08, 0xc4, 0x73, 0x2b, 0x38, 0x67,
|
||||
0x75, 0xab, 0x61, 0xb6, 0x8e, 0x54, 0xf9, 0x8a, 0x58, 0x17, 0x26, 0x73, 0xf0, 0xaa, 0x95, 0x23,
|
||||
0xc3, 0x53, 0x2b, 0x38, 0x8f, 0xef, 0xfb, 0x35, 0x14, 0x39, 0x4b, 0xe8, 0x09, 0x94, 0x47, 0x74,
|
||||
0xee, 0x85, 0xab, 0x09, 0xbd, 0x9d, 0x15, 0x3e, 0xf3, 0xc4, 0x99, 0xa5, 0x40, 0xed, 0x04, 0x4a,
|
||||
0xb1, 0x0b, 0x3d, 0x4a, 0xa7, 0x92, 0x78, 0x7c, 0x3f, 0x11, 0x67, 0xff, 0x9c, 0xfa, 0xe1, 0xda,
|
||||
0x50, 0xda, 0x81, 0xc2, 0xa5, 0xe5, 0xce, 0xf9, 0xe3, 0x45, 0xcc, 0x37, 0xda, 0x5f, 0x05, 0x28,
|
||||
0xe1, 0xa8, 0x08, 0x41, 0x98, 0x19, 0xf6, 0x85, 0xb5, 0x61, 0xbf, 0x92, 0x4b, 0x6e, 0x4d, 0x2e,
|
||||
0x49, 0xc7, 0xe7, 0x33, 0x1d, 0xbf, 0x62, 0x4e, 0xbc, 0x93, 0xb9, 0xc2, 0x1d, 0xcc, 0x15, 0x33,
|
||||
0xcc, 0x3d, 0x82, 0xad, 0xb1, 0x4f, 0xa7, 0x6c, 0x9c, 0x53, 0xdf, 0xf2, 0xaf, 0xe2, 0xee, 0xdc,
|
||||
0x8c, 0xac, 0x83, 0xc4, 0xa8, 0x99, 0x50, 0xc6, 0x24, 0x98, 0x51, 0x2f, 0x20, 0xef, 0x7c, 0x36,
|
||||
0x02, 0xd1, 0xb6, 0x42, 0x8b, 0x3d, 0xba, 0x86, 0xd9, 0x1a, 0xfd, 0x04, 0xc4, 0x11, 0xb5, 0xf9,
|
||||
0x93, 0xb7, 0xb2, 0x3d, 0xa4, 0xfb, 0x3e, 0xf5, 0x9b, 0xd4, 0x26, 0x98, 0x01, 0xb4, 0x19, 0x48,
|
||||
0x2d, 0xfa, 0xca, 0x73, 0xa9, 0x65, 0xf7, 0x7c, 0x3a, 0x89, 0xc6, 0xed, 0x3b, 0xc7, 0x46, 0x0b,
|
||||
0x4a, 0x73, 0x36, 0x58, 0x92, 0xc1, 0xf1, 0xc9, 0xba, 0xd0, 0x6f, 0x06, 0xe2, 0x53, 0x28, 0x51,
|
||||
0x47, 0x7c, 0x54, 0xfb, 0x5e, 0x00, 0xe5, 0xdd, 0x68, 0xd4, 0x86, 0x2a, 0x47, 0x9a, 0x99, 0x2f,
|
||||
0x89, 0x83, 0xf7, 0xb9, 0x88, 0xcd, 0x18, 0x98, 0xa7, 0xeb, 0x3b, 0xff, 0x9e, 0x32, 0x6a, 0xce,
|
||||
0xbf, 0x9f, 0x9a, 0x1f, 0xc2, 0x26, 0xd3, 0x59, 0xfa, 0xa7, 0x2b, 0xaa, 0xf9, 0x83, 0x02, 0xae,
|
||||
0x0d, 0xb9, 0x8a, 0x98, 0x4d, 0x2b, 0x82, 0xd8, 0x73, 0xbc, 0x89, 0xb6, 0x0f, 0x85, 0xa6, 0x4b,
|
||||
0x59, 0xb1, 0x8a, 0x3e, 0xb1, 0x02, 0xea, 0x25, 0x1c, 0xf2, 0xdd, 0xe1, 0xf7, 0x39, 0xa8, 0x66,
|
||||
0x3e, 0x86, 0xd0, 0x63, 0xd8, 0x6a, 0x76, 0xce, 0xfa, 0x03, 0x1d, 0x9b, 0xcd, 0xae, 0x71, 0xd2,
|
||||
0x3e, 0x95, 0x36, 0x94, 0xbd, 0xc5, 0x52, 0x95, 0xa7, 0x2b, 0xd0, 0xfa, 0x77, 0xce, 0x3e, 0x14,
|
||||
0xda, 0x46, 0x4b, 0xff, 0x9d, 0x24, 0x28, 0x3b, 0x8b, 0xa5, 0x2a, 0x65, 0x80, 0xfc, 0xcf, 0xe4,
|
||||
0xa7, 0x50, 0x63, 0x00, 0xf3, 0xac, 0xd7, 0x6a, 0x0c, 0x74, 0x29, 0xa7, 0x28, 0x8b, 0xa5, 0xba,
|
||||
0x7b, 0x13, 0x17, 0xf3, 0xfd, 0x10, 0x4a, 0x58, 0xff, 0xed, 0x99, 0xde, 0x1f, 0x48, 0x79, 0x65,
|
||||
0x77, 0xb1, 0x54, 0x51, 0x06, 0x98, 0x28, 0xe6, 0x11, 0x94, 0xb1, 0xde, 0xef, 0x75, 0x8d, 0xbe,
|
||||
0x2e, 0x89, 0xca, 0x87, 0x8b, 0xa5, 0x7a, 0x6f, 0x0d, 0x15, 0x77, 0xe8, 0x2f, 0x60, 0xbb, 0xd5,
|
||||
0x7d, 0x61, 0x74, 0xba, 0x8d, 0x96, 0xd9, 0xc3, 0xdd, 0x53, 0xac, 0xf7, 0xfb, 0x52, 0x41, 0xd9,
|
||||
0x5f, 0x2c, 0xd5, 0x07, 0x19, 0xfc, 0xad, 0x86, 0xfb, 0x08, 0xc4, 0x5e, 0xdb, 0x38, 0x95, 0x8a,
|
||||
0xca, 0xbd, 0xc5, 0x52, 0xfd, 0x20, 0x03, 0x8d, 0x48, 0x8d, 0x32, 0x6e, 0x76, 0xba, 0x7d, 0x5d,
|
||||
0x2a, 0xdd, 0xca, 0x98, 0x91, 0x7d, 0xf8, 0x7b, 0x40, 0xb7, 0x3f, 0x17, 0xd1, 0x27, 0x20, 0x1a,
|
||||
0x5d, 0x43, 0x97, 0x36, 0x78, 0xfe, 0xb7, 0x11, 0x06, 0xf5, 0x08, 0xd2, 0x20, 0xdf, 0xf9, 0xe6,
|
||||
0x4b, 0x49, 0x50, 0x7e, 0xb4, 0x58, 0xaa, 0xf7, 0x6f, 0x83, 0x3a, 0xdf, 0x7c, 0x79, 0x48, 0xa1,
|
||||
0x9a, 0x0d, 0xac, 0x41, 0xf9, 0x99, 0x3e, 0x68, 0xb4, 0x1a, 0x83, 0x86, 0xb4, 0xc1, 0x9f, 0x94,
|
||||
0xb8, 0x9f, 0x91, 0xd0, 0x62, 0x02, 0xdc, 0x83, 0x82, 0xa1, 0x3f, 0xd7, 0xb1, 0x24, 0x28, 0xdb,
|
||||
0x8b, 0xa5, 0xba, 0x99, 0x00, 0x0c, 0x72, 0x49, 0x7c, 0x54, 0x87, 0x62, 0xa3, 0xf3, 0xa2, 0xf1,
|
||||
0xb2, 0x2f, 0xe5, 0x14, 0xb4, 0x58, 0xaa, 0x5b, 0x89, 0xbb, 0xe1, 0xbe, 0xb2, 0xae, 0x82, 0xc3,
|
||||
0xff, 0x08, 0x50, 0xcb, 0xfe, 0x75, 0xa2, 0x3a, 0x88, 0x27, 0xed, 0x8e, 0x9e, 0x5c, 0x97, 0xf5,
|
||||
0x45, 0x6b, 0x74, 0x00, 0x95, 0x56, 0x1b, 0xeb, 0xcd, 0x41, 0x17, 0xbf, 0x4c, 0x72, 0xc9, 0x82,
|
||||
0x5a, 0x8e, 0xcf, 0x9a, 0x3b, 0xfa, 0x3c, 0xad, 0xf5, 0x5f, 0x3e, 0xeb, 0xb4, 0x8d, 0xaf, 0x4d,
|
||||
0x16, 0x31, 0xa7, 0x3c, 0x58, 0x2c, 0xd5, 0x0f, 0xb3, 0xe0, 0x3e, 0xff, 0xdb, 0x60, 0x81, 0xbf,
|
||||
0x82, 0xed, 0x04, 0xbe, 0xba, 0x20, 0xaf, 0xa8, 0x8b, 0xa5, 0xba, 0x77, 0xc7, 0x99, 0xd5, 0x3d,
|
||||
0x4f, 0xe0, 0x83, 0xe4, 0xe0, 0x99, 0xf1, 0xb5, 0xd1, 0x7d, 0x61, 0x48, 0xa2, 0x52, 0x5f, 0x2c,
|
||||
0x55, 0xe5, 0x8e, 0x63, 0x67, 0xde, 0x85, 0x47, 0x5f, 0x79, 0x87, 0x7f, 0x11, 0xa0, 0x92, 0x4e,
|
||||
0xa8, 0x88, 0x67, 0xa3, 0x6b, 0xea, 0x18, 0x77, 0x71, 0x92, 0x78, 0xea, 0x34, 0x28, 0x5b, 0xa2,
|
||||
0x8f, 0xa1, 0x74, 0xaa, 0x1b, 0x3a, 0x6e, 0x37, 0x13, 0x3d, 0xa4, 0x90, 0x53, 0xe2, 0x11, 0xdf,
|
||||
0x19, 0xa1, 0x4f, 0xa1, 0x66, 0x74, 0xcd, 0xfe, 0x59, 0xf3, 0x69, 0x92, 0x31, 0x6b, 0xe0, 0x4c,
|
||||
0xa8, 0xfe, 0x7c, 0x74, 0xce, 0xb2, 0x3d, 0x8c, 0xa4, 0xf3, 0xbc, 0xd1, 0x69, 0xb7, 0x38, 0x34,
|
||||
0xaf, 0xc8, 0x8b, 0xa5, 0xba, 0x93, 0x42, 0xdb, 0xfc, 0xd3, 0x21, 0xc2, 0x1e, 0xda, 0x50, 0xff,
|
||||
0xdf, 0xb3, 0x08, 0xa9, 0x50, 0x6c, 0xf4, 0x7a, 0xba, 0xd1, 0x4a, 0x5e, 0xbf, 0xf2, 0x35, 0x66,
|
||||
0x33, 0xe2, 0xd9, 0x11, 0xe2, 0xa4, 0x8b, 0x4f, 0xf5, 0x41, 0xf2, 0xf8, 0x15, 0xe2, 0x84, 0x46,
|
||||
0x7f, 0xda, 0xc7, 0x7b, 0x6f, 0x7e, 0xa8, 0x6f, 0x7c, 0xf7, 0x43, 0x7d, 0xe3, 0xcd, 0x75, 0x5d,
|
||||
0xf8, 0xee, 0xba, 0x2e, 0xfc, 0xe3, 0xba, 0xbe, 0xf1, 0xaf, 0xeb, 0xba, 0xf0, 0xed, 0x3f, 0xeb,
|
||||
0xc2, 0xb0, 0xc8, 0x66, 0xd7, 0x93, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xee, 0xe2, 0x0f, 0x00,
|
||||
0x38, 0x0e, 0x00, 0x00,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -120,9 +120,10 @@ enum FileInfoType {
|
|||
|
||||
message BlockInfo {
|
||||
option (gogoproto.goproto_stringer) = false;
|
||||
int64 offset = 1;
|
||||
int32 size = 2;
|
||||
bytes hash = 3;
|
||||
int64 offset = 1;
|
||||
int32 size = 2;
|
||||
bytes hash = 3;
|
||||
uint32 weak_hash = 4;
|
||||
}
|
||||
|
||||
message Vector {
|
||||
|
|
|
|||
|
|
@ -114,7 +114,7 @@ func (f FileInfo) WinsConflict(other FileInfo) bool {
|
|||
}
|
||||
|
||||
func (b BlockInfo) String() string {
|
||||
return fmt.Sprintf("Block{%d/%d/%x}", b.Offset, b.Size, b.Hash)
|
||||
return fmt.Sprintf("Block{%d/%d/%d/%x}", b.Offset, b.Size, b.WeakHash, b.Hash)
|
||||
}
|
||||
|
||||
// IsEmpty returns true if the block is a full block of zeroes.
|
||||
|
|
|
|||
|
|
@ -427,16 +427,16 @@ var (
|
|||
)
|
||||
|
||||
var fileDescriptorDeviceidTest = []byte{
|
||||
// 171 bytes of a gzipped FileDescriptorProto
|
||||
// 176 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0x12, 0x4e, 0x49, 0x2d, 0xcb,
|
||||
0x4c, 0x4e, 0xcd, 0x4c, 0x89, 0x2f, 0x49, 0x2d, 0x2e, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17,
|
||||
0xe2, 0x00, 0x53, 0xc9, 0xf9, 0x39, 0x52, 0xba, 0xe9, 0x99, 0x25, 0x19, 0xa5, 0x49, 0x7a, 0xc9,
|
||||
0xf9, 0xb9, 0xfa, 0xe9, 0xf9, 0xe9, 0xf9, 0xfa, 0x60, 0x99, 0xa4, 0xd2, 0x34, 0x30, 0x0f, 0xcc,
|
||||
0x01, 0xb3, 0x20, 0x1a, 0x95, 0x54, 0xb9, 0xf8, 0x43, 0x80, 0xc6, 0xf8, 0xe7, 0xa4, 0xb8, 0x80,
|
||||
0x8d, 0xf5, 0x74, 0x11, 0x12, 0xe2, 0x62, 0x01, 0x99, 0x2c, 0xc1, 0xa8, 0xc0, 0xa8, 0xc1, 0x13,
|
||||
0x04, 0x66, 0x2b, 0x99, 0x43, 0x94, 0xf9, 0xa5, 0x96, 0xc3, 0x95, 0xa9, 0x20, 0x2b, 0x73, 0x12,
|
||||
0x38, 0x71, 0x4f, 0x9e, 0xe1, 0xd6, 0x3d, 0x79, 0x0e, 0x98, 0x3c, 0x44, 0xa3, 0x93, 0xcc, 0x89,
|
||||
0x87, 0x72, 0x0c, 0x17, 0x80, 0xf8, 0xc4, 0x23, 0x39, 0xc6, 0x0b, 0x40, 0xfc, 0xe0, 0x91, 0x1c,
|
||||
0xc3, 0x0b, 0x20, 0x5e, 0xf0, 0x58, 0x8e, 0x31, 0x89, 0x0d, 0xec, 0x08, 0x63, 0x40, 0x00, 0x00,
|
||||
0x00, 0xff, 0xff, 0x35, 0x9c, 0x00, 0x78, 0xd4, 0x00, 0x00, 0x00,
|
||||
0x01, 0xb3, 0x20, 0x1a, 0x95, 0x54, 0xb9, 0xf8, 0x43, 0x52, 0x8b, 0x4b, 0xfc, 0x73, 0x52, 0x5c,
|
||||
0xc0, 0xc6, 0x7a, 0xba, 0x08, 0x09, 0x71, 0xb1, 0x80, 0x4c, 0x96, 0x60, 0x54, 0x60, 0xd4, 0xe0,
|
||||
0x09, 0x02, 0xb3, 0x95, 0xcc, 0x21, 0xca, 0xfc, 0x52, 0xcb, 0xe1, 0xca, 0x54, 0x90, 0x95, 0x39,
|
||||
0x09, 0x9c, 0xb8, 0x27, 0xcf, 0x70, 0xeb, 0x9e, 0x3c, 0x07, 0x4c, 0x1e, 0xa2, 0xd1, 0x49, 0xe6,
|
||||
0xc4, 0x43, 0x39, 0x86, 0x0b, 0x0f, 0xe5, 0x18, 0x4e, 0x3c, 0x92, 0x63, 0xbc, 0xf0, 0x48, 0x8e,
|
||||
0xf1, 0xc1, 0x23, 0x39, 0x86, 0x17, 0x8f, 0xe4, 0x18, 0x16, 0x3c, 0x96, 0x63, 0x4c, 0x62, 0x03,
|
||||
0x3b, 0xc2, 0x18, 0x10, 0x00, 0x00, 0xff, 0xff, 0x35, 0x9c, 0x00, 0x78, 0xd4, 0x00, 0x00, 0x00,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import (
|
|||
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
"github.com/syncthing/syncthing/lib/sha256"
|
||||
"github.com/syncthing/syncthing/lib/weakhash"
|
||||
)
|
||||
|
||||
var SHA256OfNothing = []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}
|
||||
|
|
@ -25,6 +26,7 @@ type Counter interface {
|
|||
func Blocks(r io.Reader, blocksize int, sizehint int64, counter Counter) ([]protocol.BlockInfo, error) {
|
||||
hf := sha256.New()
|
||||
hashLength := hf.Size()
|
||||
whf := weakhash.NewHash(blocksize)
|
||||
|
||||
var blocks []protocol.BlockInfo
|
||||
var hashes, thisHash []byte
|
||||
|
|
@ -44,7 +46,7 @@ func Blocks(r io.Reader, blocksize int, sizehint int64, counter Counter) ([]prot
|
|||
var offset int64
|
||||
for {
|
||||
lr := io.LimitReader(r, int64(blocksize))
|
||||
n, err := copyBuffer(hf, lr, buf)
|
||||
n, err := io.CopyBuffer(hf, io.TeeReader(lr, whf), buf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -63,15 +65,17 @@ func Blocks(r io.Reader, blocksize int, sizehint int64, counter Counter) ([]prot
|
|||
thisHash, hashes = hashes[:hashLength], hashes[hashLength:]
|
||||
|
||||
b := protocol.BlockInfo{
|
||||
Size: int32(n),
|
||||
Offset: offset,
|
||||
Hash: thisHash,
|
||||
Size: int32(n),
|
||||
Offset: offset,
|
||||
Hash: thisHash,
|
||||
WeakHash: whf.Sum32(),
|
||||
}
|
||||
|
||||
blocks = append(blocks, b)
|
||||
offset += int64(n)
|
||||
|
||||
hf.Reset()
|
||||
whf.Reset()
|
||||
}
|
||||
|
||||
if len(blocks) == 0 {
|
||||
|
|
@ -123,9 +127,12 @@ func BlockDiff(src, tgt []protocol.BlockInfo) (have, need []protocol.BlockInfo)
|
|||
// list and actual reader contents
|
||||
func Verify(r io.Reader, blocksize int, blocks []protocol.BlockInfo) error {
|
||||
hf := sha256.New()
|
||||
// A 32k buffer is used for copying into the hash function.
|
||||
buf := make([]byte, 32<<10)
|
||||
|
||||
for i, block := range blocks {
|
||||
lr := &io.LimitedReader{R: r, N: int64(blocksize)}
|
||||
_, err := io.Copy(hf, lr)
|
||||
_, err := io.CopyBuffer(hf, lr, buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -180,48 +187,3 @@ func BlocksEqual(src, tgt []protocol.BlockInfo) bool {
|
|||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// This is a copy & paste of io.copyBuffer from the Go 1.5 standard library,
|
||||
// as we want this but also want to build with Go 1.3+.
|
||||
|
||||
// copyBuffer is the actual implementation of Copy and CopyBuffer.
|
||||
// if buf is nil, one is allocated.
|
||||
func copyBuffer(dst io.Writer, src io.Reader, buf []byte) (written int64, err error) {
|
||||
// If the reader has a WriteTo method, use it to do the copy.
|
||||
// Avoids an allocation and a copy.
|
||||
if wt, ok := src.(io.WriterTo); ok {
|
||||
return wt.WriteTo(dst)
|
||||
}
|
||||
// Similarly, if the writer has a ReadFrom method, use it to do the copy.
|
||||
if rt, ok := dst.(io.ReaderFrom); ok {
|
||||
return rt.ReadFrom(src)
|
||||
}
|
||||
if buf == nil {
|
||||
buf = make([]byte, 32*1024)
|
||||
}
|
||||
for {
|
||||
nr, er := src.Read(buf)
|
||||
if nr > 0 {
|
||||
nw, ew := dst.Write(buf[0:nr])
|
||||
if nw > 0 {
|
||||
written += int64(nw)
|
||||
}
|
||||
if ew != nil {
|
||||
err = ew
|
||||
break
|
||||
}
|
||||
if nr != nw {
|
||||
err = io.ErrShortWrite
|
||||
break
|
||||
}
|
||||
}
|
||||
if er == io.EOF {
|
||||
break
|
||||
}
|
||||
if er != nil {
|
||||
err = er
|
||||
break
|
||||
}
|
||||
}
|
||||
return written, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,30 @@
|
|||
// Copyright (C) 2016 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
package weakhash
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const testFile = "../model/testdata/~syncthing~file.tmp"
|
||||
|
||||
func BenchmarkFind1MFile(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(1 << 20)
|
||||
for i := 0; i < b.N; i++ {
|
||||
fd, err := os.Open(testFile)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
_, err = Find(fd, []uint32{0, 1, 2}, 128<<10)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
fd.Close()
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,169 @@
|
|||
// Copyright (C) 2016 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
package weakhash
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"hash"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
const (
|
||||
Size = 4
|
||||
)
|
||||
|
||||
func NewHash(size int) hash.Hash32 {
|
||||
return &digest{
|
||||
buf: make([]byte, size),
|
||||
size: size,
|
||||
}
|
||||
}
|
||||
|
||||
// Find finds all the blocks of the given size within io.Reader that matches
|
||||
// the hashes provided, and returns a hash -> slice of offsets within reader
|
||||
// map, that produces the same weak hash.
|
||||
func Find(ir io.Reader, hashesToFind []uint32, size int) (map[uint32][]int64, error) {
|
||||
if ir == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
r := bufio.NewReader(ir)
|
||||
hf := NewHash(size)
|
||||
|
||||
n, err := io.CopyN(hf, r, int64(size))
|
||||
if err == io.EOF {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if n != int64(size) {
|
||||
return nil, io.ErrShortBuffer
|
||||
}
|
||||
|
||||
offsets := make(map[uint32][]int64)
|
||||
for _, hashToFind := range hashesToFind {
|
||||
offsets[hashToFind] = nil
|
||||
}
|
||||
|
||||
var i int64
|
||||
var hash uint32
|
||||
for {
|
||||
hash = hf.Sum32()
|
||||
if existing, ok := offsets[hash]; ok {
|
||||
offsets[hash] = append(existing, i)
|
||||
}
|
||||
i++
|
||||
|
||||
bt, err := r.ReadByte()
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
return offsets, err
|
||||
}
|
||||
hf.Write([]byte{bt})
|
||||
}
|
||||
return offsets, nil
|
||||
}
|
||||
|
||||
// Using this: http://tutorials.jenkov.com/rsync/checksums.html
|
||||
// Example implementations: https://gist.github.com/csabahenk/1096262/revisions
|
||||
// Alternative that could be used is adler32 http://blog.liw.fi/posts/rsync-in-python/#comment-fee8d5e07794fdba3fe2d76aa2706a13
|
||||
type digest struct {
|
||||
buf []byte
|
||||
size int
|
||||
a uint16
|
||||
b uint16
|
||||
j int
|
||||
}
|
||||
|
||||
func (d *digest) Write(data []byte) (int, error) {
|
||||
for _, c := range data {
|
||||
// TODO: Use this in Go 1.6
|
||||
// d.a = d.a - uint16(d.buf[d.j]) + uint16(c)
|
||||
// d.b = d.b - uint16(d.size)*uint16(d.buf[d.j]) + d.a
|
||||
d.a -= uint16(d.buf[d.j])
|
||||
d.a += uint16(c)
|
||||
d.b -= uint16(d.size) * uint16(d.buf[d.j])
|
||||
d.b += d.a
|
||||
|
||||
d.buf[d.j] = c
|
||||
d.j = (d.j + 1) % d.size
|
||||
}
|
||||
return len(data), nil
|
||||
}
|
||||
|
||||
func (d *digest) Reset() {
|
||||
for i := range d.buf {
|
||||
d.buf[i] = 0x0
|
||||
}
|
||||
d.a = 0
|
||||
d.b = 0
|
||||
d.j = 0
|
||||
}
|
||||
|
||||
func (d *digest) Sum(b []byte) []byte {
|
||||
r := d.Sum32()
|
||||
return append(b, byte(r>>24), byte(r>>16), byte(r>>8), byte(r))
|
||||
}
|
||||
|
||||
func (d *digest) Sum32() uint32 { return uint32(d.a) | (uint32(d.b) << 16) }
|
||||
func (digest) Size() int { return Size }
|
||||
func (digest) BlockSize() int { return 1 }
|
||||
|
||||
func NewFinder(path string, size int, hashesToFind []uint32) (*Finder, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
offsets, err := Find(file, hashesToFind, size)
|
||||
if err != nil {
|
||||
file.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Finder{
|
||||
file: file,
|
||||
size: size,
|
||||
offsets: offsets,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type Finder struct {
|
||||
file *os.File
|
||||
size int
|
||||
offsets map[uint32][]int64
|
||||
}
|
||||
|
||||
// Iterate iterates all available blocks that matches the provided hash, reads
|
||||
// them into buf, and calls the iterator function. The iterator function should
|
||||
// return wether it wishes to continue interating.
|
||||
func (h *Finder) Iterate(hash uint32, buf []byte, iterFunc func(int64) bool) (bool, error) {
|
||||
if h == nil || hash == 0 || len(buf) != h.size {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
for _, offset := range h.offsets[hash] {
|
||||
_, err := h.file.ReadAt(buf, offset)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !iterFunc(offset) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Close releases any resource associated with the finder
|
||||
func (h *Finder) Close() {
|
||||
if h != nil {
|
||||
h.file.Close()
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,188 @@
|
|||
// Copyright (C) 2016 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
// The existence of this file means we get 0% test coverage rather than no
|
||||
// test coverage at all. Remove when implementing an actual test.
|
||||
|
||||
package weakhash
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var payload = []byte("abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz")
|
||||
var hashes = []uint32{
|
||||
64225674,
|
||||
64881038,
|
||||
65536402,
|
||||
66191766,
|
||||
66847130,
|
||||
67502494,
|
||||
68157858,
|
||||
68813222,
|
||||
69468586,
|
||||
70123950,
|
||||
70779314,
|
||||
71434678,
|
||||
72090042,
|
||||
72745406,
|
||||
73400770,
|
||||
74056134,
|
||||
74711498,
|
||||
75366862,
|
||||
76022226,
|
||||
76677590,
|
||||
77332954,
|
||||
77988318,
|
||||
78643682,
|
||||
77595084,
|
||||
74842550,
|
||||
70386080,
|
||||
64225674,
|
||||
64881038,
|
||||
65536402,
|
||||
66191766,
|
||||
66847130,
|
||||
67502494,
|
||||
68157858,
|
||||
68813222,
|
||||
69468586,
|
||||
70123950,
|
||||
70779314,
|
||||
71434678,
|
||||
72090042,
|
||||
72745406,
|
||||
73400770,
|
||||
74056134,
|
||||
74711498,
|
||||
75366862,
|
||||
76022226,
|
||||
76677590,
|
||||
77332954,
|
||||
77988318,
|
||||
78643682,
|
||||
77595084,
|
||||
74842550,
|
||||
70386080,
|
||||
64225674,
|
||||
64881038,
|
||||
65536402,
|
||||
66191766,
|
||||
66847130,
|
||||
67502494,
|
||||
68157858,
|
||||
68813222,
|
||||
69468586,
|
||||
70123950,
|
||||
70779314,
|
||||
71434678,
|
||||
72090042,
|
||||
72745406,
|
||||
73400770,
|
||||
74056134,
|
||||
74711498,
|
||||
75366862,
|
||||
76022226,
|
||||
76677590,
|
||||
77332954,
|
||||
77988318,
|
||||
78643682,
|
||||
77595084,
|
||||
74842550,
|
||||
70386080,
|
||||
64225674,
|
||||
64881038,
|
||||
65536402,
|
||||
66191766,
|
||||
66847130,
|
||||
67502494,
|
||||
68157858,
|
||||
68813222,
|
||||
69468586,
|
||||
70123950,
|
||||
70779314,
|
||||
71434678,
|
||||
72090042,
|
||||
72745406,
|
||||
73400770,
|
||||
74056134,
|
||||
74711498,
|
||||
75366862,
|
||||
76022226,
|
||||
76677590,
|
||||
77332954,
|
||||
77988318,
|
||||
78643682,
|
||||
71893365,
|
||||
71893365,
|
||||
}
|
||||
|
||||
// Tested using an alternative C implementation at https://gist.github.com/csabahenk/1096262
|
||||
func TestHashCorrect(t *testing.T) {
|
||||
h := NewHash(Size)
|
||||
pos := 0
|
||||
for pos < Size {
|
||||
h.Write([]byte{payload[pos]})
|
||||
pos++
|
||||
}
|
||||
|
||||
for i := 0; pos < len(payload); i++ {
|
||||
if h.Sum32() != hashes[i] {
|
||||
t.Errorf("mismatch at %d", i)
|
||||
}
|
||||
h.Write([]byte{payload[pos]})
|
||||
pos++
|
||||
}
|
||||
}
|
||||
|
||||
func TestFinder(t *testing.T) {
|
||||
f, err := ioutil.TempFile("", "")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
defer f.Close()
|
||||
|
||||
if _, err := f.Write(payload); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
hashes := []uint32{64881038, 65536402}
|
||||
finder, err := NewFinder(f.Name(), 4, hashes)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
defer finder.Close()
|
||||
|
||||
expected := map[uint32][]int64{
|
||||
64881038: []int64{1, 27, 53, 79},
|
||||
65536402: []int64{2, 28, 54, 80},
|
||||
}
|
||||
actual := make(map[uint32][]int64)
|
||||
|
||||
b := make([]byte, Size)
|
||||
|
||||
for _, hash := range hashes {
|
||||
_, err := finder.Iterate(hash, b[:4], func(offset int64) bool {
|
||||
if !bytes.Equal(b, payload[offset:offset+4]) {
|
||||
t.Errorf("Not equal at %d: %s != %s", offset, string(b), string(payload[offset:offset+4]))
|
||||
}
|
||||
actual[hash] = append(actual[hash], offset)
|
||||
return true
|
||||
})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(actual, expected) {
|
||||
t.Errorf("Not equal: %#v != %#v", actual, expected)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,42 +1,6 @@
|
|||
<configuration version="16">
|
||||
<folder id="default" label="" path="s2/" type="readwrite" rescanIntervalS="15" ignorePerms="false" autoNormalize="true">
|
||||
<folder id="default" label="" path="s2" type="readwrite" rescanIntervalS="15" ignorePerms="false" autoNormalize="true">
|
||||
<device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></device>
|
||||
<device id="MRIW7OK-NETT3M4-N6SBWME-N25O76W-YJKVXPH-FUMQJ3S-P57B74J-GBITBAC"></device>
|
||||
<device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU"></device>
|
||||
<minDiskFreePct>1</minDiskFreePct>
|
||||
<versioning type="trashcan">
|
||||
<param key="cleanoutDays" val="1"></param>
|
||||
</versioning>
|
||||
<copiers>1</copiers>
|
||||
<pullers>16</pullers>
|
||||
<hashers>0</hashers>
|
||||
<order>random</order>
|
||||
<ignoreDelete>false</ignoreDelete>
|
||||
<scanProgressIntervalS>0</scanProgressIntervalS>
|
||||
<pullerSleepS>0</pullerSleepS>
|
||||
<pullerPauseS>0</pullerPauseS>
|
||||
<maxConflicts>-1</maxConflicts>
|
||||
<disableSparseFiles>false</disableSparseFiles>
|
||||
<disableTempIndexes>false</disableTempIndexes>
|
||||
</folder>
|
||||
<folder id="¯\_(ツ)_/¯ Räksmörgås 动作 Адрес" label="" path="s12-2/" type="readwrite" rescanIntervalS="15" ignorePerms="false" autoNormalize="true">
|
||||
<device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></device>
|
||||
<device id="MRIW7OK-NETT3M4-N6SBWME-N25O76W-YJKVXPH-FUMQJ3S-P57B74J-GBITBAC"></device>
|
||||
<minDiskFreePct>1</minDiskFreePct>
|
||||
<versioning></versioning>
|
||||
<copiers>1</copiers>
|
||||
<pullers>16</pullers>
|
||||
<hashers>0</hashers>
|
||||
<order>random</order>
|
||||
<ignoreDelete>false</ignoreDelete>
|
||||
<scanProgressIntervalS>0</scanProgressIntervalS>
|
||||
<pullerSleepS>0</pullerSleepS>
|
||||
<pullerPauseS>0</pullerPauseS>
|
||||
<maxConflicts>-1</maxConflicts>
|
||||
<disableSparseFiles>false</disableSparseFiles>
|
||||
<disableTempIndexes>false</disableTempIndexes>
|
||||
</folder>
|
||||
<folder id="s23" label="" path="s23-2/" type="readwrite" rescanIntervalS="15" ignorePerms="false" autoNormalize="true">
|
||||
<device id="MRIW7OK-NETT3M4-N6SBWME-N25O76W-YJKVXPH-FUMQJ3S-P57B74J-GBITBAC"></device>
|
||||
<device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU"></device>
|
||||
<minDiskFreePct>1</minDiskFreePct>
|
||||
|
|
@ -52,6 +16,43 @@
|
|||
<maxConflicts>-1</maxConflicts>
|
||||
<disableSparseFiles>false</disableSparseFiles>
|
||||
<disableTempIndexes>false</disableTempIndexes>
|
||||
<disableWeakHash>false</disableWeakHash>
|
||||
</folder>
|
||||
<folder id="¯\_(ツ)_/¯ Räksmörgås 动作 Адрес" label="" path="s12-2" type="readwrite" rescanIntervalS="15" ignorePerms="false" autoNormalize="true">
|
||||
<device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></device>
|
||||
<device id="MRIW7OK-NETT3M4-N6SBWME-N25O76W-YJKVXPH-FUMQJ3S-P57B74J-GBITBAC"></device>
|
||||
<minDiskFreePct>1</minDiskFreePct>
|
||||
<versioning></versioning>
|
||||
<copiers>1</copiers>
|
||||
<pullers>16</pullers>
|
||||
<hashers>0</hashers>
|
||||
<order>random</order>
|
||||
<ignoreDelete>false</ignoreDelete>
|
||||
<scanProgressIntervalS>0</scanProgressIntervalS>
|
||||
<pullerSleepS>0</pullerSleepS>
|
||||
<pullerPauseS>0</pullerPauseS>
|
||||
<maxConflicts>-1</maxConflicts>
|
||||
<disableSparseFiles>false</disableSparseFiles>
|
||||
<disableTempIndexes>false</disableTempIndexes>
|
||||
<disableWeakHash>false</disableWeakHash>
|
||||
</folder>
|
||||
<folder id="s23" label="" path="s23-2" type="readwrite" rescanIntervalS="15" ignorePerms="false" autoNormalize="true">
|
||||
<device id="MRIW7OK-NETT3M4-N6SBWME-N25O76W-YJKVXPH-FUMQJ3S-P57B74J-GBITBAC"></device>
|
||||
<device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU"></device>
|
||||
<minDiskFreePct>1</minDiskFreePct>
|
||||
<versioning></versioning>
|
||||
<copiers>1</copiers>
|
||||
<pullers>16</pullers>
|
||||
<hashers>0</hashers>
|
||||
<order>random</order>
|
||||
<ignoreDelete>false</ignoreDelete>
|
||||
<scanProgressIntervalS>0</scanProgressIntervalS>
|
||||
<pullerSleepS>0</pullerSleepS>
|
||||
<pullerPauseS>0</pullerPauseS>
|
||||
<maxConflicts>-1</maxConflicts>
|
||||
<disableSparseFiles>false</disableSparseFiles>
|
||||
<disableTempIndexes>false</disableTempIndexes>
|
||||
<disableWeakHash>false</disableWeakHash>
|
||||
</folder>
|
||||
<device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU" name="s1" compression="metadata" introducer="false">
|
||||
<address>tcp://127.0.0.1:22001</address>
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
<configuration version="15">
|
||||
<folder id="s23" label="" path="s23-3/" type="readwrite" rescanIntervalS="20" ignorePerms="false" autoNormalize="true">
|
||||
<configuration version="16">
|
||||
<folder id="s23" label="" path="s23-3" type="readwrite" rescanIntervalS="20" ignorePerms="false" autoNormalize="true">
|
||||
<device id="MRIW7OK-NETT3M4-N6SBWME-N25O76W-YJKVXPH-FUMQJ3S-P57B74J-GBITBAC"></device>
|
||||
<device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU"></device>
|
||||
<minDiskFreePct>1</minDiskFreePct>
|
||||
|
|
@ -15,8 +15,9 @@
|
|||
<maxConflicts>-1</maxConflicts>
|
||||
<disableSparseFiles>false</disableSparseFiles>
|
||||
<disableTempIndexes>false</disableTempIndexes>
|
||||
<disableWeakHash>false</disableWeakHash>
|
||||
</folder>
|
||||
<folder id="default" label="" path="s3/" type="readwrite" rescanIntervalS="20" ignorePerms="false" autoNormalize="true">
|
||||
<folder id="default" label="" path="s3" type="readwrite" rescanIntervalS="20" ignorePerms="false" autoNormalize="true">
|
||||
<device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></device>
|
||||
<device id="MRIW7OK-NETT3M4-N6SBWME-N25O76W-YJKVXPH-FUMQJ3S-P57B74J-GBITBAC"></device>
|
||||
<device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU"></device>
|
||||
|
|
@ -35,6 +36,7 @@
|
|||
<maxConflicts>-1</maxConflicts>
|
||||
<disableSparseFiles>false</disableSparseFiles>
|
||||
<disableTempIndexes>false</disableTempIndexes>
|
||||
<disableWeakHash>false</disableWeakHash>
|
||||
</folder>
|
||||
<device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU" name="s1" compression="metadata" introducer="false">
|
||||
<address>tcp://127.0.0.1:22001</address>
|
||||
|
|
|
|||
Loading…
Reference in New Issue