lib/model: Verify request content against weak (and possibly strong) hash (#4767)

This commit is contained in:
Audrius Butkevicius 2018-05-05 09:24:44 +01:00 committed by Jakob Borg
parent 678c80ffe4
commit ef0dcea6a4
27 changed files with 518 additions and 454 deletions

View File

@ -46,7 +46,6 @@ import (
"github.com/syncthing/syncthing/lib/sha256"
"github.com/syncthing/syncthing/lib/tlsutil"
"github.com/syncthing/syncthing/lib/upgrade"
"github.com/syncthing/syncthing/lib/weakhash"
"github.com/thejerf/suture"
@ -697,26 +696,8 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
},
}
if opts := cfg.Options(); opts.WeakHashSelectionMethod == config.WeakHashAuto {
perfWithWeakHash := cpuBench(3, 150*time.Millisecond, true)
l.Infof("Hashing performance with rolling hash is %.02f MB/s", perfWithWeakHash)
perfWithoutWeakHash := cpuBench(3, 150*time.Millisecond, false)
l.Infof("Hashing performance without rolling hash is %.02f MB/s", perfWithoutWeakHash)
if perfWithoutWeakHash*0.8 > perfWithWeakHash {
l.Infof("Rolling hash disabled, as it has an unacceptable performance impact.")
weakhash.Enabled = false
} else {
l.Infof("Rolling hash enabled, as it has an acceptable performance impact.")
weakhash.Enabled = true
}
} else if opts.WeakHashSelectionMethod == config.WeakHashNever {
l.Infof("Disabling rolling hash")
weakhash.Enabled = false
} else if opts.WeakHashSelectionMethod == config.WeakHashAlways {
l.Infof("Enabling rolling hash")
weakhash.Enabled = true
}
perf := cpuBench(3, 150*time.Millisecond, true)
l.Infof("Hashing performance is %.02f MB/s", perf)
dbFile := locations[locDatabase]
ldb, err := db.Open(dbFile)

View File

@ -26,7 +26,6 @@ import (
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/scanner"
"github.com/syncthing/syncthing/lib/upgrade"
"github.com/syncthing/syncthing/lib/weakhash"
)
// Current version number of the usage report, for acceptance purposes. If
@ -190,8 +189,6 @@ func reportData(cfg configIntf, m modelIntf, connectionsService connectionsIntf,
res["overwriteRemoteDeviceNames"] = opts.OverwriteRemoteDevNames
res["progressEmitterEnabled"] = opts.ProgressUpdateIntervalS > -1
res["customDefaultFolderPath"] = opts.DefaultFolderPath != "~"
res["weakHashSelection"] = opts.WeakHashSelectionMethod.String()
res["weakHashEnabled"] = weakhash.Enabled
res["customTrafficClass"] = opts.TrafficClass != 0
res["customTempIndexMinBlocks"] = opts.TempIndexMinBlocks != 10
res["temporariesDisabled"] = opts.KeepTemporariesH == 0

View File

@ -67,7 +67,6 @@ func TestDefaultValues(t *testing.T) {
OverwriteRemoteDevNames: false,
TempIndexMinBlocks: 10,
UnackedNotificationIDs: []string{},
WeakHashSelectionMethod: WeakHashAuto,
DefaultFolderPath: "~",
SetLowPriority: true,
}
@ -209,9 +208,8 @@ func TestOverriddenValues(t *testing.T) {
"channelNotification", // added in 17->18 migration
"fsWatcherNotification", // added in 27->28 migration
},
WeakHashSelectionMethod: WeakHashNever,
DefaultFolderPath: "/media/syncthing",
SetLowPriority: false,
DefaultFolderPath: "/media/syncthing",
SetLowPriority: false,
}
os.Unsetenv("STNOUPGRADE")

View File

@ -7,135 +7,50 @@
package config
import (
"encoding/json"
"encoding/xml"
"fmt"
"github.com/syncthing/syncthing/lib/util"
)
type WeakHashSelectionMethod int
const (
WeakHashAuto WeakHashSelectionMethod = iota
WeakHashAlways
WeakHashNever
)
func (m WeakHashSelectionMethod) MarshalString() (string, error) {
switch m {
case WeakHashAuto:
return "auto", nil
case WeakHashAlways:
return "always", nil
case WeakHashNever:
return "never", nil
default:
return "", fmt.Errorf("unrecognized hash selection method")
}
}
func (m WeakHashSelectionMethod) String() string {
s, err := m.MarshalString()
if err != nil {
panic(err)
}
return s
}
func (m *WeakHashSelectionMethod) UnmarshalString(value string) error {
switch value {
case "auto":
*m = WeakHashAuto
return nil
case "always":
*m = WeakHashAlways
return nil
case "never":
*m = WeakHashNever
return nil
}
return fmt.Errorf("unrecognized hash selection method")
}
func (m WeakHashSelectionMethod) MarshalJSON() ([]byte, error) {
val, err := m.MarshalString()
if err != nil {
return nil, err
}
return json.Marshal(val)
}
func (m *WeakHashSelectionMethod) UnmarshalJSON(data []byte) error {
var value string
if err := json.Unmarshal(data, &value); err != nil {
return err
}
return m.UnmarshalString(value)
}
func (m WeakHashSelectionMethod) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
val, err := m.MarshalString()
if err != nil {
return err
}
return e.EncodeElement(val, start)
}
func (m *WeakHashSelectionMethod) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
var value string
if err := d.DecodeElement(&value, &start); err != nil {
return err
}
return m.UnmarshalString(value)
}
func (WeakHashSelectionMethod) ParseDefault(value string) (interface{}, error) {
var m WeakHashSelectionMethod
err := m.UnmarshalString(value)
return m, err
}
type OptionsConfiguration struct {
ListenAddresses []string `xml:"listenAddress" json:"listenAddresses" default:"default"`
GlobalAnnServers []string `xml:"globalAnnounceServer" json:"globalAnnounceServers" json:"globalAnnounceServer" default:"default" restart:"true"`
GlobalAnnEnabled bool `xml:"globalAnnounceEnabled" json:"globalAnnounceEnabled" default:"true" restart:"true"`
LocalAnnEnabled bool `xml:"localAnnounceEnabled" json:"localAnnounceEnabled" default:"true" restart:"true"`
LocalAnnPort int `xml:"localAnnouncePort" json:"localAnnouncePort" default:"21027" restart:"true"`
LocalAnnMCAddr string `xml:"localAnnounceMCAddr" json:"localAnnounceMCAddr" default:"[ff12::8384]:21027" restart:"true"`
MaxSendKbps int `xml:"maxSendKbps" json:"maxSendKbps"`
MaxRecvKbps int `xml:"maxRecvKbps" json:"maxRecvKbps"`
ReconnectIntervalS int `xml:"reconnectionIntervalS" json:"reconnectionIntervalS" default:"60"`
RelaysEnabled bool `xml:"relaysEnabled" json:"relaysEnabled" default:"true"`
RelayReconnectIntervalM int `xml:"relayReconnectIntervalM" json:"relayReconnectIntervalM" default:"10"`
StartBrowser bool `xml:"startBrowser" json:"startBrowser" default:"true"`
NATEnabled bool `xml:"natEnabled" json:"natEnabled" default:"true"`
NATLeaseM int `xml:"natLeaseMinutes" json:"natLeaseMinutes" default:"60"`
NATRenewalM int `xml:"natRenewalMinutes" json:"natRenewalMinutes" default:"30"`
NATTimeoutS int `xml:"natTimeoutSeconds" json:"natTimeoutSeconds" default:"10"`
URAccepted int `xml:"urAccepted" json:"urAccepted"` // Accepted usage reporting version; 0 for off (undecided), -1 for off (permanently)
URSeen int `xml:"urSeen" json:"urSeen"` // Report which the user has been prompted for.
URUniqueID string `xml:"urUniqueID" json:"urUniqueId"` // Unique ID for reporting purposes, regenerated when UR is turned on.
URURL string `xml:"urURL" json:"urURL" default:"https://data.syncthing.net/newdata"`
URPostInsecurely bool `xml:"urPostInsecurely" json:"urPostInsecurely" default:"false"` // For testing
URInitialDelayS int `xml:"urInitialDelayS" json:"urInitialDelayS" default:"1800"`
RestartOnWakeup bool `xml:"restartOnWakeup" json:"restartOnWakeup" default:"true" restart:"true"`
AutoUpgradeIntervalH int `xml:"autoUpgradeIntervalH" json:"autoUpgradeIntervalH" default:"12" restart:"true"` // 0 for off
UpgradeToPreReleases bool `xml:"upgradeToPreReleases" json:"upgradeToPreReleases" restart:"true"` // when auto upgrades are enabled
KeepTemporariesH int `xml:"keepTemporariesH" json:"keepTemporariesH" default:"24"` // 0 for off
CacheIgnoredFiles bool `xml:"cacheIgnoredFiles" json:"cacheIgnoredFiles" default:"false" restart:"true"`
ProgressUpdateIntervalS int `xml:"progressUpdateIntervalS" json:"progressUpdateIntervalS" default:"5"`
LimitBandwidthInLan bool `xml:"limitBandwidthInLan" json:"limitBandwidthInLan" default:"false"`
MinHomeDiskFree Size `xml:"minHomeDiskFree" json:"minHomeDiskFree" default:"1 %"`
ReleasesURL string `xml:"releasesURL" json:"releasesURL" default:"https://upgrades.syncthing.net/meta.json" restart:"true"`
AlwaysLocalNets []string `xml:"alwaysLocalNet" json:"alwaysLocalNets"`
OverwriteRemoteDevNames bool `xml:"overwriteRemoteDeviceNamesOnConnect" json:"overwriteRemoteDeviceNamesOnConnect" default:"false"`
TempIndexMinBlocks int `xml:"tempIndexMinBlocks" json:"tempIndexMinBlocks" default:"10"`
UnackedNotificationIDs []string `xml:"unackedNotificationID" json:"unackedNotificationIDs"`
TrafficClass int `xml:"trafficClass" json:"trafficClass"`
WeakHashSelectionMethod WeakHashSelectionMethod `xml:"weakHashSelectionMethod" json:"weakHashSelectionMethod" restart:"true"`
DefaultFolderPath string `xml:"defaultFolderPath" json:"defaultFolderPath" default:"~"`
SetLowPriority bool `xml:"setLowPriority" json:"setLowPriority" default:"true"`
ListenAddresses []string `xml:"listenAddress" json:"listenAddresses" default:"default"`
GlobalAnnServers []string `xml:"globalAnnounceServer" json:"globalAnnounceServers" json:"globalAnnounceServer" default:"default" restart:"true"`
GlobalAnnEnabled bool `xml:"globalAnnounceEnabled" json:"globalAnnounceEnabled" default:"true" restart:"true"`
LocalAnnEnabled bool `xml:"localAnnounceEnabled" json:"localAnnounceEnabled" default:"true" restart:"true"`
LocalAnnPort int `xml:"localAnnouncePort" json:"localAnnouncePort" default:"21027" restart:"true"`
LocalAnnMCAddr string `xml:"localAnnounceMCAddr" json:"localAnnounceMCAddr" default:"[ff12::8384]:21027" restart:"true"`
MaxSendKbps int `xml:"maxSendKbps" json:"maxSendKbps"`
MaxRecvKbps int `xml:"maxRecvKbps" json:"maxRecvKbps"`
ReconnectIntervalS int `xml:"reconnectionIntervalS" json:"reconnectionIntervalS" default:"60"`
RelaysEnabled bool `xml:"relaysEnabled" json:"relaysEnabled" default:"true"`
RelayReconnectIntervalM int `xml:"relayReconnectIntervalM" json:"relayReconnectIntervalM" default:"10"`
StartBrowser bool `xml:"startBrowser" json:"startBrowser" default:"true"`
NATEnabled bool `xml:"natEnabled" json:"natEnabled" default:"true"`
NATLeaseM int `xml:"natLeaseMinutes" json:"natLeaseMinutes" default:"60"`
NATRenewalM int `xml:"natRenewalMinutes" json:"natRenewalMinutes" default:"30"`
NATTimeoutS int `xml:"natTimeoutSeconds" json:"natTimeoutSeconds" default:"10"`
URAccepted int `xml:"urAccepted" json:"urAccepted"` // Accepted usage reporting version; 0 for off (undecided), -1 for off (permanently)
URSeen int `xml:"urSeen" json:"urSeen"` // Report which the user has been prompted for.
URUniqueID string `xml:"urUniqueID" json:"urUniqueId"` // Unique ID for reporting purposes, regenerated when UR is turned on.
URURL string `xml:"urURL" json:"urURL" default:"https://data.syncthing.net/newdata"`
URPostInsecurely bool `xml:"urPostInsecurely" json:"urPostInsecurely" default:"false"` // For testing
URInitialDelayS int `xml:"urInitialDelayS" json:"urInitialDelayS" default:"1800"`
RestartOnWakeup bool `xml:"restartOnWakeup" json:"restartOnWakeup" default:"true" restart:"true"`
AutoUpgradeIntervalH int `xml:"autoUpgradeIntervalH" json:"autoUpgradeIntervalH" default:"12" restart:"true"` // 0 for off
UpgradeToPreReleases bool `xml:"upgradeToPreReleases" json:"upgradeToPreReleases" restart:"true"` // when auto upgrades are enabled
KeepTemporariesH int `xml:"keepTemporariesH" json:"keepTemporariesH" default:"24"` // 0 for off
CacheIgnoredFiles bool `xml:"cacheIgnoredFiles" json:"cacheIgnoredFiles" default:"false" restart:"true"`
ProgressUpdateIntervalS int `xml:"progressUpdateIntervalS" json:"progressUpdateIntervalS" default:"5"`
LimitBandwidthInLan bool `xml:"limitBandwidthInLan" json:"limitBandwidthInLan" default:"false"`
MinHomeDiskFree Size `xml:"minHomeDiskFree" json:"minHomeDiskFree" default:"1 %"`
ReleasesURL string `xml:"releasesURL" json:"releasesURL" default:"https://upgrades.syncthing.net/meta.json" restart:"true"`
AlwaysLocalNets []string `xml:"alwaysLocalNet" json:"alwaysLocalNets"`
OverwriteRemoteDevNames bool `xml:"overwriteRemoteDeviceNamesOnConnect" json:"overwriteRemoteDeviceNamesOnConnect" default:"false"`
TempIndexMinBlocks int `xml:"tempIndexMinBlocks" json:"tempIndexMinBlocks" default:"10"`
UnackedNotificationIDs []string `xml:"unackedNotificationID" json:"unackedNotificationIDs"`
TrafficClass int `xml:"trafficClass" json:"trafficClass"`
DefaultFolderPath string `xml:"defaultFolderPath" json:"defaultFolderPath" default:"~"`
SetLowPriority bool `xml:"setLowPriority" json:"setLowPriority" default:"true"`
DeprecatedUPnPEnabled bool `xml:"upnpEnabled,omitempty" json:"-"`
DeprecatedUPnPLeaseM int `xml:"upnpLeaseMinutes,omitempty" json:"-"`

View File

@ -34,7 +34,6 @@
<releasesURL>https://localhost/releases</releasesURL>
<overwriteRemoteDeviceNamesOnConnect>true</overwriteRemoteDeviceNamesOnConnect>
<tempIndexMinBlocks>100</tempIndexMinBlocks>
<weakHashSelectionMethod>never</weakHashSelectionMethod>
<defaultFolderPath>/media/syncthing</defaultFolderPath>
<setLowPriority>false</setLowPriority>
</options>

View File

@ -182,19 +182,6 @@ func (f *BlockFinder) Iterate(folders []string, hash []byte, iterFn func(string,
return false
}
// Fix repairs incorrect blockmap entries, removing the old entry and
// replacing it with a new entry for the given block
func (f *BlockFinder) Fix(folder, file string, index int32, oldHash, newHash []byte) error {
buf := make([]byte, 4)
binary.BigEndian.PutUint32(buf, uint32(index))
folderID := f.db.folderIdx.ID([]byte(folder))
batch := new(leveldb.Batch)
batch.Delete(blockKeyInto(nil, oldHash, folderID, file))
batch.Put(blockKeyInto(nil, newHash, folderID, file), buf)
return f.db.Write(batch, nil)
}
// m.blockKey returns a byte slice encoding the following information:
// keyTypeBlock (1 byte)
// folder (4 bytes)

View File

@ -219,34 +219,3 @@ func TestBlockFinderLookup(t *testing.T) {
f1.Deleted = false
}
func TestBlockFinderFix(t *testing.T) {
db, f := setup()
iterFn := func(folder, file string, index int32) bool {
return true
}
m := NewBlockMap(db, db.folderIdx.ID([]byte("folder1")))
err := m.Add([]protocol.FileInfo{f1})
if err != nil {
t.Fatal(err)
}
if !f.Iterate(folders, f1.Blocks[0].Hash, iterFn) {
t.Fatal("Block not found")
}
err = f.Fix("folder1", f1.Name, 0, f1.Blocks[0].Hash, f2.Blocks[0].Hash)
if err != nil {
t.Fatal(err)
}
if f.Iterate(folders, f1.Blocks[0].Hash, iterFn) {
t.Fatal("Unexpected block")
}
if !f.Iterate(folders, f2.Blocks[0].Hash, iterFn) {
t.Fatal("Block not found")
}
}

View File

@ -7,6 +7,7 @@
package model
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
@ -34,7 +35,6 @@ import (
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/upgrade"
"github.com/syncthing/syncthing/lib/versioner"
"github.com/syncthing/syncthing/lib/weakhash"
"github.com/thejerf/suture"
)
@ -1304,7 +1304,7 @@ func (m *Model) closeLocked(device protocol.DeviceID) {
// Request returns the specified data segment by reading it from local disk.
// Implements the protocol.Model interface.
func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte, weakHash uint32, fromTemporary bool, buf []byte) error {
if offset < 0 {
return protocol.ErrInvalid
}
@ -1362,8 +1362,8 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
// other than a regular file.
return protocol.ErrNoSuchFile
}
if err := readOffsetIntoBuf(folderFs, tempFn, offset, buf); err == nil {
err := readOffsetIntoBuf(folderFs, tempFn, offset, buf)
if err == nil && scanner.Validate(buf, hash, weakHash) {
return nil
}
// Fall through to reading from a non-temp file, just incase the temp
@ -1382,9 +1382,55 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
return protocol.ErrGeneric
}
if !scanner.Validate(buf, hash, weakHash) {
m.recheckFile(deviceID, folderFs, folder, name, int(offset)/len(buf), hash)
return protocol.ErrNoSuchFile
}
return nil
}
func (m *Model) recheckFile(deviceID protocol.DeviceID, folderFs fs.Filesystem, folder, name string, blockIndex int, hash []byte) {
cf, ok := m.CurrentFolderFile(folder, name)
if !ok {
l.Debugf("%v recheckFile: %s: %q / %q: no current file", m, deviceID, folder, name)
return
}
if cf.IsDeleted() || cf.IsInvalid() || cf.IsSymlink() || cf.IsDirectory() {
l.Debugf("%v recheckFile: %s: %q / %q: not a regular file", m, deviceID, folder, name)
return
}
if blockIndex > len(cf.Blocks) {
l.Debugf("%v recheckFile: %s: %q / %q i=%d: block index too far", m, deviceID, folder, name, blockIndex)
return
}
block := cf.Blocks[blockIndex]
// Seems to want a different version of the file, whatever.
if !bytes.Equal(block.Hash, hash) {
l.Debugf("%v recheckFile: %s: %q / %q i=%d: hash mismatch %x != %x", m, deviceID, folder, name, blockIndex, block.Hash, hash)
return
}
// The hashes provided part of the request match what we expect to find according
// to what we have in the database, yet the content we've read off the filesystem doesn't
// Something is fishy, invalidate the file and rescan it.
cf.Invalidate(m.shortID)
// Update the index and tell others
// The file will temporarily become invalid, which is ok as the content is messed up.
m.updateLocalsFromScanning(folder, []protocol.FileInfo{cf})
if err := m.ScanFolderSubdirs(folder, []string{name}); err != nil {
l.Debugf("%v recheckFile: %s: %q / %q rescan: %s", m, deviceID, folder, name, err)
} else {
l.Debugf("%v recheckFile: %s: %q / %q", m, deviceID, folder, name)
}
}
func (m *Model) CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool) {
m.fmut.RLock()
fs, ok := m.folderFiles[folder]
@ -1836,7 +1882,7 @@ func (m *Model) diskChangeDetected(folderCfg config.FolderConfiguration, files [
}
}
func (m *Model) requestGlobal(deviceID protocol.DeviceID, folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
func (m *Model) requestGlobal(deviceID protocol.DeviceID, folder, name string, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) {
m.pmut.RLock()
nc, ok := m.conn[deviceID]
m.pmut.RUnlock()
@ -1845,9 +1891,9 @@ func (m *Model) requestGlobal(deviceID protocol.DeviceID, folder, name string, o
return nil, fmt.Errorf("requestGlobal: no such device: %s", deviceID)
}
l.Debugf("%v REQ(out): %s: %q / %q o=%d s=%d h=%x ft=%t", m, deviceID, folder, name, offset, size, hash, fromTemporary)
l.Debugf("%v REQ(out): %s: %q / %q o=%d s=%d h=%x wh=%x ft=%t", m, deviceID, folder, name, offset, size, hash, weakHash, fromTemporary)
return nc.Request(folder, name, offset, size, hash, fromTemporary)
return nc.Request(folder, name, offset, size, hash, weakHash, fromTemporary)
}
func (m *Model) ScanFolders() map[string]error {
@ -1973,7 +2019,6 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
Hashers: m.numHashers(folder),
ShortID: m.shortID,
ProgressTickIntervalS: folderCfg.ScanProgressIntervalS,
UseWeakHashes: weakhash.Enabled,
UseLargeBlocks: folderCfg.UseLargeBlocks,
})

View File

@ -181,7 +181,7 @@ func TestRequest(t *testing.T) {
// Existing, shared file
bs = bs[:6]
err := m.Request(device1, "default", "foo", 0, nil, false, bs)
err := m.Request(device1, "default", "foo", 0, nil, 0, false, bs)
if err != nil {
t.Error(err)
}
@ -190,32 +190,32 @@ func TestRequest(t *testing.T) {
}
// Existing, nonshared file
err = m.Request(device2, "default", "foo", 0, nil, false, bs)
err = m.Request(device2, "default", "foo", 0, nil, 0, false, bs)
if err == nil {
t.Error("Unexpected nil error on insecure file read")
}
// Nonexistent file
err = m.Request(device1, "default", "nonexistent", 0, nil, false, bs)
err = m.Request(device1, "default", "nonexistent", 0, nil, 0, false, bs)
if err == nil {
t.Error("Unexpected nil error on insecure file read")
}
// Shared folder, but disallowed file name
err = m.Request(device1, "default", "../walk.go", 0, nil, false, bs)
err = m.Request(device1, "default", "../walk.go", 0, nil, 0, false, bs)
if err == nil {
t.Error("Unexpected nil error on insecure file read")
}
// Negative offset
err = m.Request(device1, "default", "foo", -4, nil, false, bs[:0])
err = m.Request(device1, "default", "foo", -4, nil, 0, false, bs[:0])
if err == nil {
t.Error("Unexpected nil error on insecure file read")
}
// Larger block than available
bs = bs[:42]
err = m.Request(device1, "default", "foo", 0, nil, false, bs)
err = m.Request(device1, "default", "foo", 0, nil, 0, false, bs)
if err == nil {
t.Error("Unexpected nil error on insecure file read")
}
@ -357,7 +357,7 @@ func (f *fakeConnection) IndexUpdate(folder string, fs []protocol.FileInfo) erro
return nil
}
func (f *fakeConnection) Request(folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
func (f *fakeConnection) Request(folder, name string, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) {
f.mut.Lock()
defer f.mut.Unlock()
if f.requestFn != nil {
@ -485,7 +485,7 @@ func BenchmarkRequestOut(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
data, err := m.requestGlobal(device1, "default", files[i%n].Name, 0, 32, nil, false)
data, err := m.requestGlobal(device1, "default", files[i%n].Name, 0, 32, nil, 0, false)
if err != nil {
b.Error(err)
}
@ -513,7 +513,7 @@ func BenchmarkRequestInSingleFile(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
if err := m.Request(device1, "default", "request/for/a/file/in/a/couple/of/dirs/128k", 0, nil, false, buf); err != nil {
if err := m.Request(device1, "default", "request/for/a/file/in/a/couple/of/dirs/128k", 0, nil, 0, false, buf); err != nil {
b.Error(err)
}
}

View File

@ -93,7 +93,7 @@ func TestSymlinkTraversalRead(t *testing.T) {
// Request a file by traversing the symlink
buf := make([]byte, 10)
err := m.Request(device1, "default", "symlink/requests_test.go", 0, nil, false, buf)
err := m.Request(device1, "default", "symlink/requests_test.go", 0, nil, 0, false, buf)
if err == nil || !bytes.Equal(buf, make([]byte, 10)) {
t.Error("Managed to traverse symlink")
}
@ -464,6 +464,73 @@ func TestIssue4841(t *testing.T) {
}
}
func TestRescanIfHaveInvalidContent(t *testing.T) {
m, fc, tmpDir := setupModelWithConnection()
defer m.Stop()
defer os.RemoveAll(tmpDir)
payload := []byte("hello")
if err := ioutil.WriteFile(filepath.Join(tmpDir, "foo"), payload, 0777); err != nil {
t.Fatal(err)
}
received := make(chan protocol.FileInfo)
fc.mut.Lock()
fc.indexFn = func(folder string, fs []protocol.FileInfo) {
if len(fs) != 1 {
t.Fatalf("Sent index with %d files, should be 1", len(fs))
}
if fs[0].Name != "foo" {
t.Fatalf(`Sent index with file %v, should be "foo"`, fs[0].Name)
}
received <- fs[0]
return
}
fc.mut.Unlock()
// Scan without ignore patterns with "foo" not existing locally
if err := m.ScanFolder("default"); err != nil {
t.Fatal("Failed scanning:", err)
}
f := <-received
if f.Blocks[0].WeakHash != 103547413 {
t.Fatalf("unexpected weak hash: %d != 103547413", f.Blocks[0].WeakHash)
}
buf := make([]byte, len(payload))
err := m.Request(device2, "default", "foo", 0, f.Blocks[0].Hash, f.Blocks[0].WeakHash, false, buf)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(buf, payload) {
t.Errorf("%s != %s", buf, payload)
}
payload = []byte("bye")
buf = make([]byte, len(payload))
if err := ioutil.WriteFile(filepath.Join(tmpDir, "foo"), payload, 0777); err != nil {
t.Fatal(err)
}
err = m.Request(device2, "default", "foo", 0, f.Blocks[0].Hash, f.Blocks[0].WeakHash, false, buf)
if err == nil {
t.Fatalf("expected failure")
}
select {
case f := <-received:
if f.Blocks[0].WeakHash != 41943361 {
t.Fatalf("unexpected weak hash: %d != 41943361", f.Blocks[0].WeakHash)
}
case <-time.After(time.Second):
t.Fatalf("timed out")
}
}
func setupModelWithConnection() (*Model, *fakeConnection, string) {
tmpDir := createTmpDir()
cfg := defaultCfgWrapper.RawCopy()

View File

@ -1186,36 +1186,32 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
var file fs.File
var weakHashFinder *weakhash.Finder
if weakhash.Enabled {
blocksPercentChanged := 0
if tot := len(state.file.Blocks); tot > 0 {
blocksPercentChanged = (tot - state.have) * 100 / tot
blocksPercentChanged := 0
if tot := len(state.file.Blocks); tot > 0 {
blocksPercentChanged = (tot - state.have) * 100 / tot
}
if blocksPercentChanged >= f.WeakHashThresholdPct {
hashesToFind := make([]uint32, 0, len(state.blocks))
for _, block := range state.blocks {
if block.WeakHash != 0 {
hashesToFind = append(hashesToFind, block.WeakHash)
}
}
if blocksPercentChanged >= f.WeakHashThresholdPct {
hashesToFind := make([]uint32, 0, len(state.blocks))
for _, block := range state.blocks {
if block.WeakHash != 0 {
hashesToFind = append(hashesToFind, block.WeakHash)
if len(hashesToFind) > 0 {
file, err = f.fs.Open(state.file.Name)
if err == nil {
weakHashFinder, err = weakhash.NewFinder(file, int(state.file.BlockSize()), hashesToFind)
if err != nil {
l.Debugln("weak hasher", err)
}
}
if len(hashesToFind) > 0 {
file, err = f.fs.Open(state.file.Name)
if err == nil {
weakHashFinder, err = weakhash.NewFinder(file, int(state.file.BlockSize()), hashesToFind)
if err != nil {
l.Debugln("weak hasher", err)
}
}
} else {
l.Debugf("not weak hashing %s. file did not contain any weak hashes", state.file.Name)
}
} else {
l.Debugf("not weak hashing %s. not enough changed %.02f < %d", state.file.Name, blocksPercentChanged, f.WeakHashThresholdPct)
l.Debugf("not weak hashing %s. file did not contain any weak hashes", state.file.Name)
}
} else {
l.Debugf("not weak hashing %s. weak hashing disabled", state.file.Name)
l.Debugf("not weak hashing %s. not enough changed %.02f < %d", state.file.Name, blocksPercentChanged, f.WeakHashThresholdPct)
}
for _, block := range state.blocks {
@ -1239,7 +1235,7 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
}
found, err := weakHashFinder.Iterate(block.WeakHash, buf, func(offset int64) bool {
if _, err := verifyBuffer(buf, block); err != nil {
if verifyBuffer(buf, block) != nil {
return true
}
@ -1274,17 +1270,8 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
return false
}
hash, err := verifyBuffer(buf, block)
if err != nil {
if hash != nil {
l.Debugf("Finder block mismatch in %s:%s:%d expected %q got %q", folder, path, index, block.Hash, hash)
err = f.model.finder.Fix(folder, path, index, block.Hash, hash)
if err != nil {
l.Warnln("finder fix:", err)
}
} else {
l.Debugln("Finder failed to verify buffer", err)
}
if err := verifyBuffer(buf, block); err != nil {
l.Debugln("Finder failed to verify buffer", err)
return false
}
@ -1324,22 +1311,22 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
}
}
func verifyBuffer(buf []byte, block protocol.BlockInfo) ([]byte, error) {
func verifyBuffer(buf []byte, block protocol.BlockInfo) error {
if len(buf) != int(block.Size) {
return nil, fmt.Errorf("length mismatch %d != %d", len(buf), block.Size)
return fmt.Errorf("length mismatch %d != %d", len(buf), block.Size)
}
hf := sha256.New()
_, err := hf.Write(buf)
if err != nil {
return nil, err
return err
}
hash := hf.Sum(nil)
if !bytes.Equal(hash, block.Hash) {
return hash, fmt.Errorf("hash mismatch %x != %x", hash, block.Hash)
return fmt.Errorf("hash mismatch %x != %x", hash, block.Hash)
}
return hash, nil
return nil
}
func (f *sendReceiveFolder) pullerRoutine(in <-chan pullBlockState, out chan<- *sharedPullerState) {
@ -1411,7 +1398,7 @@ func (f *sendReceiveFolder) pullBlock(state pullBlockState, out chan<- *sharedPu
// Fetch the block, while marking the selected device as in use so that
// leastBusy can select another device when someone else asks.
activity.using(selected)
buf, lastError := f.model.requestGlobal(selected.ID, f.folderID, state.file.Name, state.block.Offset, int(state.block.Size), state.block.Hash, selected.FromTemporary)
buf, lastError := f.model.requestGlobal(selected.ID, f.folderID, state.file.Name, state.block.Offset, int(state.block.Size), state.block.Hash, state.block.WeakHash, selected.FromTemporary)
activity.done(selected)
if lastError != nil {
l.Debugln("request:", f.folderID, state.file.Name, state.block.Offset, state.block.Size, "returned error:", lastError)
@ -1420,7 +1407,7 @@ func (f *sendReceiveFolder) pullBlock(state pullBlockState, out chan<- *sharedPu
// Verify that the received block matches the desired hash, if not
// try pulling it from another device.
_, lastError = verifyBuffer(buf, state.block)
lastError = verifyBuffer(buf, state.block)
if lastError != nil {
l.Debugln("request:", f.folderID, state.file.Name, state.block.Offset, state.block.Size, "hash mismatch")
continue

View File

@ -434,52 +434,6 @@ func TestCopierCleanup(t *testing.T) {
}
}
// Make sure that the copier routine hashes the content when asked, and pulls
// if it fails to find the block.
func TestLastResortPulling(t *testing.T) {
// Add a file to index (with the incorrect block representation, as content
// doesn't actually match the block list)
file := setUpFile("empty", []int{0})
m := setUpModel(file)
// Pretend that we are handling a new file of the same content but
// with a different name (causing to copy that particular block)
file.Name = "newfile"
iterFn := func(folder, file string, index int32) bool {
return true
}
f := setUpSendReceiveFolder(m)
copyChan := make(chan copyBlocksState)
pullChan := make(chan pullBlockState, 1)
finisherChan := make(chan *sharedPullerState, 1)
dbUpdateChan := make(chan dbUpdateJob, 1)
// Run a single copier routine
go f.copierRoutine(copyChan, pullChan, finisherChan)
f.handleFile(file, copyChan, finisherChan, dbUpdateChan)
// Copier should hash empty file, realise that the region it has read
// doesn't match the hash which was advertised by the block map, fix it
// and ask to pull the block.
<-pullChan
// Verify that it did fix the incorrect hash.
if m.finder.Iterate(folders, blocks[0].Hash, iterFn) {
t.Error("Found unexpected block")
}
if !m.finder.Iterate(folders, scanner.SHA256OfNothing, iterFn) {
t.Error("Expected block not found")
}
(<-finisherChan).fd.Close()
os.Remove(filepath.Join("testdata", fs.TempName("newfile")))
}
func TestDeregisterOnFailInCopy(t *testing.T) {
file := setUpFile("filex", []int{0, 2, 0, 0, 5, 0, 0, 8})
defer os.Remove("testdata/" + fs.TempName("filex"))

View File

@ -80,9 +80,9 @@ func benchmarkRequestsConnPair(b *testing.B, conn0, conn1 net.Conn) {
// Use c0 and c1 for each alternating request, so we get as much
// data flowing in both directions.
if i%2 == 0 {
buf, err = c0.Request("folder", "file", int64(i), 128<<10, nil, false)
buf, err = c0.Request("folder", "file", int64(i), 128<<10, nil, 0, false)
} else {
buf, err = c1.Request("folder", "file", int64(i), 128<<10, nil, false)
buf, err = c1.Request("folder", "file", int64(i), 128<<10, nil, 0, false)
}
if err != nil {
@ -171,7 +171,7 @@ func (m *fakeModel) Index(deviceID DeviceID, folder string, files []FileInfo) {
func (m *fakeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) {
}
func (m *fakeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
func (m *fakeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, weakHAsh uint32, fromTemporary bool, buf []byte) error {
// We write the offset to the end of the buffer, so the receiver
// can verify that it did in fact get some data back over the
// connection.

View File

@ -353,6 +353,7 @@ type Request struct {
Size int32 `protobuf:"varint,5,opt,name=size,proto3" json:"size,omitempty"`
Hash []byte `protobuf:"bytes,6,opt,name=hash,proto3" json:"hash,omitempty"`
FromTemporary bool `protobuf:"varint,7,opt,name=from_temporary,json=fromTemporary,proto3" json:"from_temporary,omitempty"`
WeakHash uint32 `protobuf:"varint,8,opt,name=weak_hash,json=weakHash,proto3" json:"weak_hash,omitempty"`
}
func (m *Request) Reset() { *m = Request{} }
@ -1062,6 +1063,11 @@ func (m *Request) MarshalTo(dAtA []byte) (int, error) {
}
i++
}
if m.WeakHash != 0 {
dAtA[i] = 0x40
i++
i = encodeVarintBep(dAtA, i, uint64(m.WeakHash))
}
return i, nil
}
@ -1519,6 +1525,9 @@ func (m *Request) ProtoSize() (n int) {
if m.FromTemporary {
n += 2
}
if m.WeakHash != 0 {
n += 1 + sovBep(uint64(m.WeakHash))
}
return n
}
@ -3515,6 +3524,25 @@ func (m *Request) Unmarshal(dAtA []byte) error {
}
}
m.FromTemporary = bool(v != 0)
case 8:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field WeakHash", wireType)
}
m.WeakHash = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowBep
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.WeakHash |= (uint32(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipBep(dAtA[iNdEx:])
@ -4192,115 +4220,116 @@ var (
func init() { proto.RegisterFile("bep.proto", fileDescriptorBep) }
var fileDescriptorBep = []byte{
// 1757 bytes of a gzipped FileDescriptorProto
// 1762 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4f, 0x8f, 0xdb, 0xc6,
0x15, 0x17, 0x25, 0xea, 0xdf, 0x93, 0xb4, 0xe1, 0x8e, 0xed, 0x2d, 0xcb, 0x6c, 0x24, 0x5a, 0xb1,
0xe3, 0xcd, 0x22, 0x59, 0xbb, 0x49, 0xda, 0xa2, 0x45, 0x5b, 0x40, 0x7f, 0xb8, 0x6b, 0xa1, 0x32,
0xa5, 0x8e, 0xb4, 0x4e, 0x9d, 0x43, 0x09, 0x4a, 0x1c, 0x69, 0x09, 0x53, 0x1c, 0x95, 0xa4, 0xd6,
0x56, 0x3e, 0x82, 0x3e, 0x41, 0x2f, 0x02, 0x02, 0xf4, 0x54, 0xa0, 0xc7, 0x7e, 0x08, 0x1f, 0x83,
0x1e, 0x7a, 0xe8, 0xc1, 0x68, 0xb6, 0x97, 0x1e, 0xfb, 0x09, 0x8a, 0x82, 0x33, 0xa4, 0x44, 0xed,
0xda, 0x81, 0x0f, 0x39, 0x71, 0xe6, 0xbd, 0xdf, 0xbc, 0x99, 0xf9, 0xcd, 0xef, 0xbd, 0x47, 0x28,
0x8e, 0xc8, 0xfc, 0x64, 0xee, 0xd1, 0x80, 0xa2, 0x02, 0xfb, 0x8c, 0xa9, 0xa3, 0x7c, 0x3a, 0xb5,
0x83, 0x8b, 0xc5, 0xe8, 0x64, 0x4c, 0x67, 0x0f, 0xa7, 0x74, 0x4a, 0x1f, 0x32, 0xcf, 0x68, 0x31,
0x61, 0x33, 0x36, 0x61, 0x23, 0xbe, 0xb0, 0x3e, 0x87, 0xec, 0x63, 0xe2, 0x38, 0x14, 0xd5, 0xa0,
0x64, 0x91, 0x4b, 0x7b, 0x4c, 0x0c, 0xd7, 0x9c, 0x11, 0x59, 0x50, 0x85, 0xa3, 0x22, 0x06, 0x6e,
0xd2, 0xcd, 0x19, 0x09, 0x01, 0x63, 0xc7, 0x26, 0x6e, 0xc0, 0x01, 0x69, 0x0e, 0xe0, 0x26, 0x06,
0xb8, 0x0f, 0x7b, 0x11, 0xe0, 0x92, 0x78, 0xbe, 0x4d, 0x5d, 0x39, 0xc3, 0x30, 0x15, 0x6e, 0x7d,
0xca, 0x8d, 0x75, 0x1f, 0x72, 0x8f, 0x89, 0x69, 0x11, 0x0f, 0x7d, 0x0c, 0x62, 0xb0, 0x9c, 0xf3,
0xbd, 0xf6, 0x3e, 0xbb, 0x73, 0x12, 0xdf, 0xe1, 0xe4, 0x09, 0xf1, 0x7d, 0x73, 0x4a, 0x86, 0xcb,
0x39, 0xc1, 0x0c, 0x82, 0x7e, 0x03, 0xa5, 0x31, 0x9d, 0xcd, 0x3d, 0xe2, 0xb3, 0xc0, 0x69, 0xb6,
0xe2, 0xf0, 0xc6, 0x8a, 0xd6, 0x16, 0x83, 0x93, 0x0b, 0xea, 0x0d, 0xa8, 0xb4, 0x9c, 0x85, 0x1f,
0x10, 0xaf, 0x45, 0xdd, 0x89, 0x3d, 0x45, 0x8f, 0x20, 0x3f, 0xa1, 0x8e, 0x45, 0x3c, 0x5f, 0x16,
0xd4, 0xcc, 0x51, 0xe9, 0x33, 0x69, 0x1b, 0xec, 0x94, 0x39, 0x9a, 0xe2, 0xab, 0xd7, 0xb5, 0x14,
0x8e, 0x61, 0xf5, 0x3f, 0xa7, 0x21, 0xc7, 0x3d, 0xe8, 0x00, 0xd2, 0xb6, 0xc5, 0x29, 0x6a, 0xe6,
0xae, 0x5e, 0xd7, 0xd2, 0x9d, 0x36, 0x4e, 0xdb, 0x16, 0xba, 0x0d, 0x59, 0xc7, 0x1c, 0x11, 0x27,
0x22, 0x87, 0x4f, 0xd0, 0xfb, 0x50, 0xf4, 0x88, 0x69, 0x19, 0xd4, 0x75, 0x96, 0x8c, 0x92, 0x02,
0x2e, 0x84, 0x86, 0x9e, 0xeb, 0x2c, 0xd1, 0xa7, 0x80, 0xec, 0xa9, 0x4b, 0x3d, 0x62, 0xcc, 0x89,
0x37, 0xb3, 0xd9, 0x69, 0x7d, 0x59, 0x64, 0xa8, 0x7d, 0xee, 0xe9, 0x6f, 0x1d, 0xe8, 0x43, 0xa8,
0x44, 0x70, 0x8b, 0x38, 0x24, 0x20, 0x72, 0x96, 0x21, 0xcb, 0xdc, 0xd8, 0x66, 0x36, 0xf4, 0x08,
0x6e, 0x5b, 0xb6, 0x6f, 0x8e, 0x1c, 0x62, 0x04, 0x64, 0x36, 0x37, 0x6c, 0xd7, 0x22, 0x2f, 0x89,
0x2f, 0xe7, 0x18, 0x16, 0x45, 0xbe, 0x21, 0x99, 0xcd, 0x3b, 0xdc, 0x83, 0x0e, 0x20, 0x37, 0x37,
0x17, 0x3e, 0xb1, 0xe4, 0x3c, 0xc3, 0x44, 0xb3, 0x90, 0x25, 0xae, 0x00, 0x5f, 0x96, 0xae, 0xb3,
0xd4, 0x66, 0x8e, 0x98, 0xa5, 0x08, 0x56, 0xff, 0x6f, 0x1a, 0x72, 0xdc, 0x83, 0x3e, 0xda, 0xb0,
0x54, 0x6e, 0x1e, 0x84, 0xa8, 0x7f, 0xbe, 0xae, 0x15, 0xb8, 0xaf, 0xd3, 0x4e, 0xb0, 0x86, 0x40,
0x4c, 0x28, 0x8a, 0x8d, 0xd1, 0x21, 0x14, 0x4d, 0xcb, 0x0a, 0x5f, 0x8f, 0xf8, 0x72, 0x46, 0xcd,
0x1c, 0x15, 0xf1, 0xd6, 0x80, 0x7e, 0xbe, 0xab, 0x06, 0xf1, 0xba, 0x7e, 0xde, 0x26, 0x83, 0xf0,
0x29, 0xc6, 0xc4, 0x8b, 0x14, 0x9c, 0x65, 0xfb, 0x15, 0x42, 0x03, 0xd3, 0xef, 0x5d, 0x28, 0xcf,
0xcc, 0x97, 0x86, 0x4f, 0xfe, 0xb8, 0x20, 0xee, 0x98, 0x30, 0xba, 0x32, 0xb8, 0x34, 0x33, 0x5f,
0x0e, 0x22, 0x13, 0xaa, 0x02, 0xd8, 0x6e, 0xe0, 0x51, 0x6b, 0x31, 0x26, 0x5e, 0xc4, 0x55, 0xc2,
0x82, 0x7e, 0x0a, 0x05, 0x46, 0xb6, 0x61, 0x5b, 0x72, 0x41, 0x15, 0x8e, 0xc4, 0xa6, 0x12, 0x5d,
0x3c, 0xcf, 0xa8, 0x66, 0xf7, 0x8e, 0x87, 0x38, 0xcf, 0xb0, 0x1d, 0x0b, 0xfd, 0x0a, 0x14, 0xff,
0xb9, 0x1d, 0x3e, 0x14, 0x8f, 0x14, 0xd8, 0xd4, 0x35, 0x3c, 0x32, 0xa3, 0x97, 0xa6, 0xe3, 0xcb,
0x45, 0xb6, 0x8d, 0x1c, 0x22, 0x3a, 0x09, 0x00, 0x8e, 0xfc, 0xf5, 0x1e, 0x64, 0x59, 0xc4, 0xf0,
0x15, 0xb9, 0x58, 0xa3, 0xec, 0x8d, 0x66, 0xe8, 0x04, 0xb2, 0x13, 0xdb, 0x21, 0xbe, 0x9c, 0x66,
0x6f, 0x88, 0x12, 0x4a, 0xb7, 0x1d, 0xd2, 0x71, 0x27, 0x34, 0x7a, 0x45, 0x0e, 0xab, 0x9f, 0x43,
0x89, 0x05, 0x3c, 0x9f, 0x5b, 0x66, 0x40, 0x7e, 0xb0, 0xb0, 0x7f, 0x15, 0xa1, 0x10, 0x7b, 0x36,
0x8f, 0x2e, 0x24, 0x1e, 0xfd, 0x38, 0xaa, 0x07, 0x3c, 0xbb, 0x0f, 0x6e, 0xc6, 0x4b, 0x14, 0x04,
0x04, 0xa2, 0x6f, 0x7f, 0x4d, 0x58, 0x3e, 0x65, 0x30, 0x1b, 0x23, 0x15, 0x4a, 0xd7, 0x93, 0xa8,
0x82, 0x93, 0x26, 0xf4, 0x01, 0xc0, 0x8c, 0x5a, 0xf6, 0xc4, 0x26, 0x96, 0xe1, 0x33, 0x01, 0x64,
0x70, 0x31, 0xb6, 0x0c, 0x90, 0x1c, 0xca, 0x3d, 0x4c, 0x21, 0x2b, 0xca, 0x95, 0x78, 0x1a, 0x7a,
0x6c, 0xf7, 0xd2, 0x74, 0xec, 0x38, 0x43, 0xe2, 0x69, 0x58, 0xf5, 0x5c, 0xba, 0x93, 0xbc, 0x05,
0x06, 0xa8, 0xb8, 0x34, 0x99, 0xb8, 0x8f, 0x20, 0x1f, 0x57, 0xc5, 0xf0, 0x3d, 0x77, 0x32, 0xe9,
0x29, 0x19, 0x07, 0x74, 0x53, 0x6f, 0x22, 0x18, 0x52, 0xa0, 0xb0, 0x91, 0x22, 0xb0, 0x93, 0x6e,
0xe6, 0x61, 0x2d, 0xde, 0xdc, 0xc3, 0xf5, 0xe5, 0x92, 0x2a, 0x1c, 0x65, 0xf1, 0xe6, 0x6a, 0x7a,
0xb8, 0xdd, 0x16, 0x30, 0x5a, 0xca, 0x65, 0xa6, 0xc5, 0xf7, 0x62, 0x2d, 0x0e, 0x2e, 0xa8, 0x17,
0x74, 0xda, 0xdb, 0x15, 0xcd, 0x25, 0x7a, 0x08, 0x30, 0x72, 0xe8, 0xf8, 0xb9, 0xc1, 0x68, 0xad,
0x84, 0x11, 0x9b, 0xd2, 0xd5, 0xeb, 0x5a, 0x19, 0x9b, 0x2f, 0x9a, 0xa1, 0x63, 0x60, 0x7f, 0x4d,
0x70, 0x71, 0x14, 0x0f, 0xd1, 0x4f, 0x20, 0xc7, 0xec, 0x71, 0x69, 0xb8, 0xb5, 0xbd, 0x10, 0xb3,
0x27, 0x04, 0x10, 0x01, 0x43, 0xae, 0xfc, 0xe5, 0xcc, 0xb1, 0xdd, 0xe7, 0x46, 0x60, 0x7a, 0x53,
0x12, 0xc8, 0xfb, 0xbc, 0x43, 0x44, 0xd6, 0x21, 0x33, 0xfe, 0x52, 0xfc, 0xd3, 0x37, 0xb5, 0x54,
0xdd, 0x85, 0xe2, 0x26, 0x4e, 0xa8, 0x41, 0x3a, 0x99, 0xf8, 0x24, 0x60, 0x82, 0xc9, 0xe0, 0x68,
0xb6, 0x91, 0x41, 0x9a, 0x31, 0xc0, 0x65, 0x80, 0x40, 0xbc, 0x30, 0xfd, 0x0b, 0x26, 0x8d, 0x32,
0x66, 0xe3, 0x30, 0xf1, 0x5f, 0x10, 0xf3, 0xb9, 0xc1, 0x1c, 0x5c, 0x18, 0x85, 0xd0, 0xf0, 0xd8,
0xf4, 0x2f, 0xa2, 0xfd, 0x7e, 0x0d, 0x39, 0xfe, 0x10, 0xe8, 0x73, 0x28, 0x8c, 0xe9, 0xc2, 0x0d,
0xb6, 0xcd, 0x61, 0x3f, 0x59, 0x5b, 0x98, 0x27, 0xba, 0xd9, 0x06, 0x58, 0x3f, 0x85, 0x7c, 0xe4,
0x42, 0xf7, 0x37, 0x85, 0x4f, 0x6c, 0xde, 0xb9, 0xc6, 0xf9, 0x6e, 0xb7, 0xb8, 0x34, 0x9d, 0x05,
0x3f, 0xbc, 0x88, 0xf9, 0xa4, 0xfe, 0x37, 0x01, 0xf2, 0x38, 0x7c, 0x67, 0x3f, 0x48, 0xf4, 0x99,
0xec, 0x4e, 0x9f, 0xd9, 0x66, 0x64, 0x7a, 0x27, 0x23, 0xe3, 0xa4, 0xca, 0x24, 0x92, 0x6a, 0xcb,
0x9c, 0xf8, 0x46, 0xe6, 0xb2, 0x6f, 0x60, 0x2e, 0x97, 0x60, 0xee, 0x3e, 0xec, 0x4d, 0x3c, 0x3a,
0x63, 0x9d, 0x84, 0x7a, 0xa6, 0xb7, 0x8c, 0x12, 0xa0, 0x12, 0x5a, 0x87, 0xb1, 0xb1, 0x6e, 0x40,
0x01, 0x13, 0x7f, 0x4e, 0x5d, 0x9f, 0xbc, 0xf5, 0xd8, 0x08, 0x44, 0xcb, 0x0c, 0x4c, 0x76, 0xe8,
0x32, 0x66, 0x63, 0xf4, 0x00, 0xc4, 0x31, 0xb5, 0xf8, 0x91, 0xf7, 0x92, 0x1a, 0xd2, 0x3c, 0x8f,
0x7a, 0x2d, 0x6a, 0x11, 0xcc, 0x00, 0xf5, 0x39, 0x48, 0x6d, 0xfa, 0xc2, 0x75, 0xa8, 0x69, 0xf5,
0x3d, 0x3a, 0x0d, 0x2b, 0xfa, 0x5b, 0x2b, 0x53, 0x1b, 0xf2, 0x0b, 0x56, 0xbb, 0xe2, 0xda, 0x74,
0x6f, 0xb7, 0x96, 0x5c, 0x0f, 0xc4, 0x0b, 0x5d, 0x9c, 0x80, 0xd1, 0xd2, 0xfa, 0x3f, 0x04, 0x50,
0xde, 0x8e, 0x46, 0x1d, 0x28, 0x71, 0xa4, 0x91, 0xf8, 0x89, 0x39, 0x7a, 0x97, 0x8d, 0x58, 0x19,
0x83, 0xc5, 0x66, 0xfc, 0xc6, 0x0e, 0x98, 0x28, 0x18, 0x99, 0x77, 0x2b, 0x18, 0x0f, 0xa0, 0xc2,
0x33, 0x38, 0xee, 0xf7, 0xa2, 0x9a, 0x39, 0xca, 0x36, 0xd3, 0x52, 0x0a, 0x97, 0x47, 0x3c, 0x93,
0x98, 0xbd, 0x9e, 0x03, 0xb1, 0x6f, 0xbb, 0xd3, 0x7a, 0x0d, 0xb2, 0x2d, 0x87, 0xb2, 0x07, 0xcb,
0x79, 0xc4, 0xf4, 0xa9, 0x1b, 0xf3, 0xc8, 0x67, 0xc7, 0x7f, 0x4f, 0x43, 0x29, 0xf1, 0x2f, 0x86,
0x1e, 0xc1, 0x5e, 0xab, 0x7b, 0x3e, 0x18, 0x6a, 0xd8, 0x68, 0xf5, 0xf4, 0xd3, 0xce, 0x99, 0x94,
0x52, 0x0e, 0x57, 0x6b, 0x55, 0x9e, 0x6d, 0x41, 0xbb, 0xbf, 0x59, 0x35, 0xc8, 0x76, 0xf4, 0xb6,
0xf6, 0x7b, 0x49, 0x50, 0x6e, 0xaf, 0xd6, 0xaa, 0x94, 0x00, 0xf2, 0x9e, 0xf5, 0x09, 0x94, 0x19,
0xc0, 0x38, 0xef, 0xb7, 0x1b, 0x43, 0x4d, 0x4a, 0x2b, 0xca, 0x6a, 0xad, 0x1e, 0x5c, 0xc7, 0x45,
0x9c, 0x7f, 0x08, 0x79, 0xac, 0xfd, 0xee, 0x5c, 0x1b, 0x0c, 0xa5, 0x8c, 0x72, 0xb0, 0x5a, 0xab,
0x28, 0x01, 0x8c, 0xb3, 0xe6, 0x3e, 0x14, 0xb0, 0x36, 0xe8, 0xf7, 0xf4, 0x81, 0x26, 0x89, 0xca,
0x8f, 0x56, 0x6b, 0xf5, 0xd6, 0x0e, 0x2a, 0x52, 0xe9, 0xcf, 0x60, 0xbf, 0xdd, 0xfb, 0x52, 0xef,
0xf6, 0x1a, 0x6d, 0xa3, 0x8f, 0x7b, 0x67, 0x58, 0x1b, 0x0c, 0xa4, 0xac, 0x52, 0x5b, 0xad, 0xd5,
0xf7, 0x13, 0xf8, 0x1b, 0xa2, 0xfb, 0x00, 0xc4, 0x7e, 0x47, 0x3f, 0x93, 0x72, 0xca, 0xad, 0xd5,
0x5a, 0x7d, 0x2f, 0x01, 0x0d, 0x49, 0x0d, 0x6f, 0xdc, 0xea, 0xf6, 0x06, 0x9a, 0x94, 0xbf, 0x71,
0x63, 0x46, 0xf6, 0xf1, 0x1f, 0x00, 0xdd, 0xfc, 0x5b, 0x45, 0xf7, 0x40, 0xd4, 0x7b, 0xba, 0x26,
0xa5, 0xf8, 0xfd, 0x6f, 0x22, 0x74, 0xea, 0x12, 0x54, 0x87, 0x4c, 0xf7, 0xab, 0x2f, 0x24, 0x41,
0xf9, 0xf1, 0x6a, 0xad, 0xde, 0xb9, 0x09, 0xea, 0x7e, 0xf5, 0xc5, 0x31, 0x85, 0x52, 0x32, 0x70,
0x1d, 0x0a, 0x4f, 0xb4, 0x61, 0xa3, 0xdd, 0x18, 0x36, 0xa4, 0x14, 0x3f, 0x52, 0xec, 0x7e, 0x42,
0x02, 0x93, 0x25, 0xe1, 0x21, 0x64, 0x75, 0xed, 0xa9, 0x86, 0x25, 0x41, 0xd9, 0x5f, 0xad, 0xd5,
0x4a, 0x0c, 0xd0, 0xc9, 0x25, 0xf1, 0x50, 0x15, 0x72, 0x8d, 0xee, 0x97, 0x8d, 0x67, 0x03, 0x29,
0xad, 0xa0, 0xd5, 0x5a, 0xdd, 0x8b, 0xdd, 0x0d, 0xe7, 0x85, 0xb9, 0xf4, 0x8f, 0xff, 0x27, 0x40,
0x39, 0xd9, 0xa1, 0x51, 0x15, 0xc4, 0xd3, 0x4e, 0x57, 0x8b, 0xb7, 0x4b, 0xfa, 0xc2, 0x31, 0x3a,
0x82, 0x62, 0xbb, 0x83, 0xb5, 0xd6, 0xb0, 0x87, 0x9f, 0xc5, 0x77, 0x49, 0x82, 0xda, 0xb6, 0xc7,
0x04, 0xbe, 0x44, 0xbf, 0x80, 0xf2, 0xe0, 0xd9, 0x93, 0x6e, 0x47, 0xff, 0xad, 0xc1, 0x22, 0xa6,
0x95, 0x07, 0xab, 0xb5, 0x7a, 0x77, 0x07, 0x4c, 0xe6, 0x1e, 0x19, 0x9b, 0x01, 0xb1, 0x06, 0xbc,
0x89, 0x84, 0xce, 0x82, 0x80, 0x5a, 0xb0, 0x1f, 0x2f, 0xdd, 0x6e, 0x96, 0x51, 0x3e, 0x59, 0xad,
0xd5, 0x8f, 0xbe, 0x77, 0xfd, 0x66, 0xf7, 0x82, 0x80, 0xee, 0x41, 0x3e, 0x0a, 0x12, 0x2b, 0x29,
0xb9, 0x34, 0x5a, 0x70, 0xfc, 0x17, 0x01, 0x8a, 0x9b, 0x72, 0x15, 0x12, 0xae, 0xf7, 0x0c, 0x0d,
0xe3, 0x1e, 0x8e, 0x19, 0xd8, 0x38, 0x75, 0xca, 0x86, 0xe8, 0x2e, 0xe4, 0xcf, 0x34, 0x5d, 0xc3,
0x9d, 0x56, 0x9c, 0x18, 0x1b, 0xc8, 0x19, 0x71, 0x89, 0x67, 0x8f, 0xd1, 0xc7, 0x50, 0xd6, 0x7b,
0xc6, 0xe0, 0xbc, 0xf5, 0x38, 0xbe, 0x3a, 0xdb, 0x3f, 0x11, 0x6a, 0xb0, 0x18, 0x5f, 0x30, 0x3e,
0x8f, 0xc3, 0x1c, 0x7a, 0xda, 0xe8, 0x76, 0xda, 0x1c, 0x9a, 0x51, 0xe4, 0xd5, 0x5a, 0xbd, 0xbd,
0x81, 0x76, 0xf8, 0xaf, 0x4a, 0x88, 0x3d, 0xb6, 0xa0, 0xfa, 0xfd, 0x85, 0x09, 0xa9, 0x90, 0x6b,
0xf4, 0xfb, 0x9a, 0xde, 0x8e, 0x4f, 0xbf, 0xf5, 0x35, 0xe6, 0x73, 0xe2, 0x5a, 0x21, 0xe2, 0xb4,
0x87, 0xcf, 0xb4, 0x61, 0x7c, 0xf8, 0x2d, 0xe2, 0x94, 0x86, 0x1d, 0xbc, 0x79, 0xf8, 0xea, 0xbb,
0x6a, 0xea, 0xdb, 0xef, 0xaa, 0xa9, 0x57, 0x57, 0x55, 0xe1, 0xdb, 0xab, 0xaa, 0xf0, 0xaf, 0xab,
0x6a, 0xea, 0x3f, 0x57, 0x55, 0xe1, 0x9b, 0x7f, 0x57, 0x85, 0x51, 0x8e, 0x15, 0xb2, 0xcf, 0xff,
0x1f, 0x00, 0x00, 0xff, 0xff, 0x0d, 0x36, 0x7c, 0x9b, 0xc0, 0x0e, 0x00, 0x00,
0x15, 0x5f, 0x4a, 0x94, 0x44, 0x3d, 0x69, 0x37, 0xdc, 0xb1, 0xbd, 0x65, 0x99, 0x8d, 0x44, 0x2b,
0x76, 0xbc, 0x59, 0x24, 0x6b, 0x37, 0x49, 0x5b, 0xb4, 0x68, 0x0b, 0xe8, 0x0f, 0x77, 0x2d, 0x54,
0xa6, 0xd4, 0x91, 0xd6, 0xa9, 0x73, 0x28, 0x41, 0x89, 0x23, 0x2d, 0x61, 0x8a, 0xa3, 0x92, 0xd4,
0xda, 0xca, 0x47, 0xd0, 0x27, 0xe8, 0x45, 0x40, 0x80, 0x9e, 0x0a, 0xf4, 0x83, 0xf8, 0x98, 0xf6,
0xd0, 0x43, 0x0f, 0x46, 0xb3, 0xbd, 0xf4, 0xd8, 0x4f, 0x50, 0x14, 0x9c, 0x21, 0x25, 0x6a, 0xd7,
0x0e, 0x7c, 0xc8, 0x89, 0x33, 0xef, 0xfd, 0xe6, 0x0d, 0xdf, 0x6f, 0x7e, 0xef, 0xcd, 0x40, 0x71,
0x48, 0x66, 0x27, 0x33, 0x9f, 0x86, 0x14, 0x49, 0xec, 0x33, 0xa2, 0xae, 0xfa, 0xe9, 0xc4, 0x09,
0x2f, 0xe6, 0xc3, 0x93, 0x11, 0x9d, 0x3e, 0x9c, 0xd0, 0x09, 0x7d, 0xc8, 0x3c, 0xc3, 0xf9, 0x98,
0xcd, 0xd8, 0x84, 0x8d, 0xf8, 0xc2, 0xda, 0x0c, 0x72, 0x8f, 0x89, 0xeb, 0x52, 0x54, 0x85, 0x92,
0x4d, 0x2e, 0x9d, 0x11, 0x31, 0x3d, 0x6b, 0x4a, 0x14, 0x41, 0x13, 0x8e, 0x8a, 0x18, 0xb8, 0xc9,
0xb0, 0xa6, 0x24, 0x02, 0x8c, 0x5c, 0x87, 0x78, 0x21, 0x07, 0x64, 0x38, 0x80, 0x9b, 0x18, 0xe0,
0x3e, 0xec, 0xc5, 0x80, 0x4b, 0xe2, 0x07, 0x0e, 0xf5, 0x94, 0x2c, 0xc3, 0xec, 0x72, 0xeb, 0x53,
0x6e, 0xac, 0x05, 0x90, 0x7f, 0x4c, 0x2c, 0x9b, 0xf8, 0xe8, 0x63, 0x10, 0xc3, 0xc5, 0x8c, 0xef,
0xb5, 0xf7, 0xd9, 0x9d, 0x93, 0x24, 0x87, 0x93, 0x27, 0x24, 0x08, 0xac, 0x09, 0x19, 0x2c, 0x66,
0x04, 0x33, 0x08, 0xfa, 0x0d, 0x94, 0x46, 0x74, 0x3a, 0xf3, 0x49, 0xc0, 0x02, 0x67, 0xd8, 0x8a,
0xc3, 0x1b, 0x2b, 0x9a, 0x1b, 0x0c, 0x4e, 0x2f, 0xa8, 0xd5, 0x61, 0xb7, 0xe9, 0xce, 0x83, 0x90,
0xf8, 0x4d, 0xea, 0x8d, 0x9d, 0x09, 0x7a, 0x04, 0x85, 0x31, 0x75, 0x6d, 0xe2, 0x07, 0x8a, 0xa0,
0x65, 0x8f, 0x4a, 0x9f, 0xc9, 0x9b, 0x60, 0xa7, 0xcc, 0xd1, 0x10, 0x5f, 0xbd, 0xae, 0xee, 0xe0,
0x04, 0x56, 0xfb, 0x73, 0x06, 0xf2, 0xdc, 0x83, 0x0e, 0x20, 0xe3, 0xd8, 0x9c, 0xa2, 0x46, 0xfe,
0xea, 0x75, 0x35, 0xd3, 0x6e, 0xe1, 0x8c, 0x63, 0xa3, 0xdb, 0x90, 0x73, 0xad, 0x21, 0x71, 0x63,
0x72, 0xf8, 0x04, 0xbd, 0x0f, 0x45, 0x9f, 0x58, 0xb6, 0x49, 0x3d, 0x77, 0xc1, 0x28, 0x91, 0xb0,
0x14, 0x19, 0xba, 0x9e, 0xbb, 0x40, 0x9f, 0x02, 0x72, 0x26, 0x1e, 0xf5, 0x89, 0x39, 0x23, 0xfe,
0xd4, 0x61, 0x7f, 0x1b, 0x28, 0x22, 0x43, 0xed, 0x73, 0x4f, 0x6f, 0xe3, 0x40, 0x1f, 0xc2, 0x6e,
0x0c, 0xb7, 0x89, 0x4b, 0x42, 0xa2, 0xe4, 0x18, 0xb2, 0xcc, 0x8d, 0x2d, 0x66, 0x43, 0x8f, 0xe0,
0xb6, 0xed, 0x04, 0xd6, 0xd0, 0x25, 0x66, 0x48, 0xa6, 0x33, 0xd3, 0xf1, 0x6c, 0xf2, 0x92, 0x04,
0x4a, 0x9e, 0x61, 0x51, 0xec, 0x1b, 0x90, 0xe9, 0xac, 0xcd, 0x3d, 0xe8, 0x00, 0xf2, 0x33, 0x6b,
0x1e, 0x10, 0x5b, 0x29, 0x30, 0x4c, 0x3c, 0x8b, 0x58, 0xe2, 0x0a, 0x08, 0x14, 0xf9, 0x3a, 0x4b,
0x2d, 0xe6, 0x48, 0x58, 0x8a, 0x61, 0xb5, 0xff, 0x66, 0x20, 0xcf, 0x3d, 0xe8, 0xa3, 0x35, 0x4b,
0xe5, 0xc6, 0x41, 0x84, 0xfa, 0xe7, 0xeb, 0xaa, 0xc4, 0x7d, 0xed, 0x56, 0x8a, 0x35, 0x04, 0x62,
0x4a, 0x51, 0x6c, 0x8c, 0x0e, 0xa1, 0x68, 0xd9, 0x76, 0x74, 0x7a, 0x24, 0x50, 0xb2, 0x5a, 0xf6,
0xa8, 0x88, 0x37, 0x06, 0xf4, 0xf3, 0x6d, 0x35, 0x88, 0xd7, 0xf5, 0xf3, 0x36, 0x19, 0x44, 0x47,
0x31, 0x22, 0x7e, 0xac, 0xe0, 0x1c, 0xdb, 0x4f, 0x8a, 0x0c, 0x4c, 0xbf, 0x77, 0xa1, 0x3c, 0xb5,
0x5e, 0x9a, 0x01, 0xf9, 0xe3, 0x9c, 0x78, 0x23, 0xc2, 0xe8, 0xca, 0xe2, 0xd2, 0xd4, 0x7a, 0xd9,
0x8f, 0x4d, 0xa8, 0x02, 0xe0, 0x78, 0xa1, 0x4f, 0xed, 0xf9, 0x88, 0xf8, 0x31, 0x57, 0x29, 0x0b,
0xfa, 0x29, 0x48, 0x8c, 0x6c, 0xd3, 0xb1, 0x15, 0x49, 0x13, 0x8e, 0xc4, 0x86, 0x1a, 0x27, 0x5e,
0x60, 0x54, 0xb3, 0xbc, 0x93, 0x21, 0x2e, 0x30, 0x6c, 0xdb, 0x46, 0xbf, 0x02, 0x35, 0x78, 0xee,
0x44, 0x07, 0xc5, 0x23, 0x85, 0x0e, 0xf5, 0x4c, 0x9f, 0x4c, 0xe9, 0xa5, 0xe5, 0x06, 0x4a, 0x91,
0x6d, 0xa3, 0x44, 0x88, 0x76, 0x0a, 0x80, 0x63, 0x7f, 0xad, 0x0b, 0x39, 0x16, 0x31, 0x3a, 0x45,
0x2e, 0xd6, 0xb8, 0x7a, 0xe3, 0x19, 0x3a, 0x81, 0xdc, 0xd8, 0x71, 0x49, 0xa0, 0x64, 0xd8, 0x19,
0xa2, 0x94, 0xd2, 0x1d, 0x97, 0xb4, 0xbd, 0x31, 0x8d, 0x4f, 0x91, 0xc3, 0x6a, 0xe7, 0x50, 0x62,
0x01, 0xcf, 0x67, 0xb6, 0x15, 0x92, 0x1f, 0x2c, 0xec, 0x5f, 0x45, 0x90, 0x12, 0xcf, 0xfa, 0xd0,
0x85, 0xd4, 0xa1, 0x1f, 0xc7, 0xfd, 0x80, 0x57, 0xf7, 0xc1, 0xcd, 0x78, 0xa9, 0x86, 0x80, 0x40,
0x0c, 0x9c, 0xaf, 0x09, 0xab, 0xa7, 0x2c, 0x66, 0x63, 0xa4, 0x41, 0xe9, 0x7a, 0x11, 0xed, 0xe2,
0xb4, 0x09, 0x7d, 0x00, 0x30, 0xa5, 0xb6, 0x33, 0x76, 0x88, 0x6d, 0x06, 0x4c, 0x00, 0x59, 0x5c,
0x4c, 0x2c, 0x7d, 0xa4, 0x44, 0x72, 0x8f, 0x4a, 0xc8, 0x8e, 0x6b, 0x25, 0x99, 0x46, 0x1e, 0xc7,
0xbb, 0xb4, 0x5c, 0x27, 0xa9, 0x90, 0x64, 0x1a, 0x75, 0x3d, 0x8f, 0x6e, 0x15, 0xaf, 0xc4, 0x00,
0xbb, 0x1e, 0x4d, 0x17, 0xee, 0x23, 0x28, 0x24, 0x5d, 0x31, 0x3a, 0xcf, 0xad, 0x4a, 0x7a, 0x4a,
0x46, 0x21, 0x5d, 0xf7, 0x9b, 0x18, 0x86, 0x54, 0x90, 0xd6, 0x52, 0x04, 0xf6, 0xa7, 0xeb, 0x79,
0xd4, 0x8b, 0xd7, 0x79, 0x78, 0x81, 0x52, 0xd2, 0x84, 0xa3, 0x1c, 0x5e, 0xa7, 0x66, 0x44, 0xdb,
0x6d, 0x00, 0xc3, 0x85, 0x52, 0x66, 0x5a, 0x7c, 0x2f, 0xd1, 0x62, 0xff, 0x82, 0xfa, 0x61, 0xbb,
0xb5, 0x59, 0xd1, 0x58, 0xa0, 0x87, 0x00, 0x43, 0x97, 0x8e, 0x9e, 0x9b, 0x8c, 0xd6, 0xdd, 0x28,
0x62, 0x43, 0xbe, 0x7a, 0x5d, 0x2d, 0x63, 0xeb, 0x45, 0x23, 0x72, 0xf4, 0x9d, 0xaf, 0x09, 0x2e,
0x0e, 0x93, 0x21, 0xfa, 0x09, 0xe4, 0x99, 0x3d, 0x69, 0x0d, 0xb7, 0x36, 0x09, 0x31, 0x7b, 0x4a,
0x00, 0x31, 0x30, 0xe2, 0x2a, 0x58, 0x4c, 0x5d, 0xc7, 0x7b, 0x6e, 0x86, 0x96, 0x3f, 0x21, 0xa1,
0xb2, 0xcf, 0x6f, 0x88, 0xd8, 0x3a, 0x60, 0xc6, 0x5f, 0x8a, 0x7f, 0xfa, 0xa6, 0xba, 0x53, 0xf3,
0xa0, 0xb8, 0x8e, 0x13, 0x69, 0x90, 0x8e, 0xc7, 0x01, 0x09, 0x99, 0x60, 0xb2, 0x38, 0x9e, 0xad,
0x65, 0x90, 0x61, 0x0c, 0x70, 0x19, 0x20, 0x10, 0x2f, 0xac, 0xe0, 0x82, 0x49, 0xa3, 0x8c, 0xd9,
0x38, 0x2a, 0xfc, 0x17, 0xc4, 0x7a, 0x6e, 0x32, 0x07, 0x17, 0x86, 0x14, 0x19, 0x1e, 0x5b, 0xc1,
0x45, 0xbc, 0xdf, 0xaf, 0x21, 0xcf, 0x0f, 0x02, 0x7d, 0x0e, 0xd2, 0x88, 0xce, 0xbd, 0x70, 0x73,
0x39, 0xec, 0xa7, 0x7b, 0x0b, 0xf3, 0xc4, 0x99, 0xad, 0x81, 0xb5, 0x53, 0x28, 0xc4, 0x2e, 0x74,
0x7f, 0xdd, 0xf8, 0xc4, 0xc6, 0x9d, 0x6b, 0x9c, 0x6f, 0xdf, 0x16, 0x97, 0x96, 0x3b, 0xe7, 0x3f,
0x2f, 0x62, 0x3e, 0xa9, 0xfd, 0x4d, 0x80, 0x02, 0x8e, 0xce, 0x39, 0x08, 0x53, 0xf7, 0x4c, 0x6e,
0xeb, 0x9e, 0xd9, 0x54, 0x64, 0x66, 0xab, 0x22, 0x93, 0xa2, 0xca, 0xa6, 0x8a, 0x6a, 0xc3, 0x9c,
0xf8, 0x46, 0xe6, 0x72, 0x6f, 0x60, 0x2e, 0x9f, 0x62, 0xee, 0x3e, 0xec, 0x8d, 0x7d, 0x3a, 0x65,
0x37, 0x09, 0xf5, 0x2d, 0x7f, 0x11, 0x17, 0xc0, 0x6e, 0x64, 0x1d, 0x24, 0xc6, 0x6d, 0x82, 0xa5,
0x6d, 0x82, 0x6b, 0x26, 0x48, 0x98, 0x04, 0x33, 0xea, 0x05, 0xe4, 0xad, 0x39, 0x21, 0x10, 0x6d,
0x2b, 0xb4, 0x58, 0x46, 0x65, 0xcc, 0xc6, 0xe8, 0x01, 0x88, 0x23, 0x6a, 0xf3, 0x7c, 0xf6, 0xd2,
0x02, 0xd3, 0x7d, 0x9f, 0xfa, 0x4d, 0x6a, 0x13, 0xcc, 0x00, 0xb5, 0x19, 0xc8, 0x2d, 0xfa, 0xc2,
0x73, 0xa9, 0x65, 0xf7, 0x7c, 0x3a, 0x89, 0xda, 0xfd, 0x5b, 0xdb, 0x56, 0x0b, 0x0a, 0x73, 0xd6,
0xd8, 0x92, 0xc6, 0x75, 0x6f, 0xbb, 0xd1, 0x5c, 0x0f, 0xc4, 0xbb, 0x60, 0x52, 0x9d, 0xf1, 0xd2,
0xda, 0x3f, 0x04, 0x50, 0xdf, 0x8e, 0x46, 0x6d, 0x28, 0x71, 0xa4, 0x99, 0x7a, 0xe1, 0x1c, 0xbd,
0xcb, 0x46, 0xac, 0xc7, 0xc1, 0x7c, 0x3d, 0x7e, 0xe3, 0xf5, 0x98, 0xea, 0x26, 0xd9, 0x77, 0xeb,
0x26, 0x0f, 0x60, 0x97, 0x97, 0x77, 0xf2, 0x18, 0x10, 0xb5, 0xec, 0x51, 0xae, 0x91, 0x91, 0x77,
0x70, 0x79, 0xc8, 0xcb, 0x8c, 0xd9, 0x6b, 0x79, 0x10, 0x7b, 0x8e, 0x37, 0xa9, 0x55, 0x21, 0xd7,
0x74, 0x29, 0x3b, 0xb0, 0xbc, 0x4f, 0xac, 0x80, 0x7a, 0x09, 0x8f, 0x7c, 0x76, 0xfc, 0xf7, 0x0c,
0x94, 0x52, 0x0f, 0x35, 0xf4, 0x08, 0xf6, 0x9a, 0x9d, 0xf3, 0xfe, 0x40, 0xc7, 0x66, 0xb3, 0x6b,
0x9c, 0xb6, 0xcf, 0xe4, 0x1d, 0xf5, 0x70, 0xb9, 0xd2, 0x94, 0xe9, 0x06, 0xb4, 0xfd, 0x06, 0xab,
0x42, 0xae, 0x6d, 0xb4, 0xf4, 0xdf, 0xcb, 0x82, 0x7a, 0x7b, 0xb9, 0xd2, 0xe4, 0x14, 0x90, 0x5f,
0x68, 0x9f, 0x40, 0x99, 0x01, 0xcc, 0xf3, 0x5e, 0xab, 0x3e, 0xd0, 0xe5, 0x8c, 0xaa, 0x2e, 0x57,
0xda, 0xc1, 0x75, 0x5c, 0xcc, 0xf9, 0x87, 0x50, 0xc0, 0xfa, 0xef, 0xce, 0xf5, 0xfe, 0x40, 0xce,
0xaa, 0x07, 0xcb, 0x95, 0x86, 0x52, 0xc0, 0xa4, 0xa4, 0xee, 0x83, 0x84, 0xf5, 0x7e, 0xaf, 0x6b,
0xf4, 0x75, 0x59, 0x54, 0x7f, 0xb4, 0x5c, 0x69, 0xb7, 0xb6, 0x50, 0xb1, 0x4a, 0x7f, 0x06, 0xfb,
0xad, 0xee, 0x97, 0x46, 0xa7, 0x5b, 0x6f, 0x99, 0x3d, 0xdc, 0x3d, 0xc3, 0x7a, 0xbf, 0x2f, 0xe7,
0xd4, 0xea, 0x72, 0xa5, 0xbd, 0x9f, 0xc2, 0xdf, 0x10, 0xdd, 0x07, 0x20, 0xf6, 0xda, 0xc6, 0x99,
0x9c, 0x57, 0x6f, 0x2d, 0x57, 0xda, 0x7b, 0x29, 0x68, 0x44, 0x6a, 0x94, 0x71, 0xb3, 0xd3, 0xed,
0xeb, 0x72, 0xe1, 0x46, 0xc6, 0x8c, 0xec, 0xe3, 0x3f, 0x00, 0xba, 0xf9, 0x94, 0x45, 0xf7, 0x40,
0x34, 0xba, 0x86, 0x2e, 0xef, 0xf0, 0xfc, 0x6f, 0x22, 0x0c, 0xea, 0x11, 0x54, 0x83, 0x6c, 0xe7,
0xab, 0x2f, 0x64, 0x41, 0xfd, 0xf1, 0x72, 0xa5, 0xdd, 0xb9, 0x09, 0xea, 0x7c, 0xf5, 0xc5, 0x31,
0x85, 0x52, 0x3a, 0x70, 0x0d, 0xa4, 0x27, 0xfa, 0xa0, 0xde, 0xaa, 0x0f, 0xea, 0xf2, 0x0e, 0xff,
0xa5, 0xc4, 0xfd, 0x84, 0x84, 0x16, 0x2b, 0xc2, 0x43, 0xc8, 0x19, 0xfa, 0x53, 0x1d, 0xcb, 0x82,
0xba, 0xbf, 0x5c, 0x69, 0xbb, 0x09, 0xc0, 0x20, 0x97, 0xc4, 0x47, 0x15, 0xc8, 0xd7, 0x3b, 0x5f,
0xd6, 0x9f, 0xf5, 0xe5, 0x8c, 0x8a, 0x96, 0x2b, 0x6d, 0x2f, 0x71, 0xd7, 0xdd, 0x17, 0xd6, 0x22,
0x38, 0xfe, 0x9f, 0x00, 0xe5, 0xf4, 0xf5, 0x8d, 0x2a, 0x20, 0x9e, 0xb6, 0x3b, 0x7a, 0xb2, 0x5d,
0xda, 0x17, 0x8d, 0xd1, 0x11, 0x14, 0x5b, 0x6d, 0xac, 0x37, 0x07, 0x5d, 0xfc, 0x2c, 0xc9, 0x25,
0x0d, 0x6a, 0x39, 0x3e, 0x13, 0xf8, 0x02, 0xfd, 0x02, 0xca, 0xfd, 0x67, 0x4f, 0x3a, 0x6d, 0xe3,
0xb7, 0x26, 0x8b, 0x98, 0x51, 0x1f, 0x2c, 0x57, 0xda, 0xdd, 0x2d, 0x30, 0x99, 0xf9, 0x64, 0x64,
0x85, 0xc4, 0xee, 0xf3, 0x1b, 0x26, 0x72, 0x4a, 0x02, 0x6a, 0xc2, 0x7e, 0xb2, 0x74, 0xb3, 0x59,
0x56, 0xfd, 0x64, 0xb9, 0xd2, 0x3e, 0xfa, 0xde, 0xf5, 0xeb, 0xdd, 0x25, 0x01, 0xdd, 0x83, 0x42,
0x1c, 0x24, 0x51, 0x52, 0x7a, 0x69, 0xbc, 0xe0, 0xf8, 0x2f, 0x02, 0x14, 0xd7, 0xed, 0x2a, 0x22,
0xdc, 0xe8, 0x9a, 0x3a, 0xc6, 0x5d, 0x9c, 0x30, 0xb0, 0x76, 0x1a, 0x94, 0x0d, 0xd1, 0x5d, 0x28,
0x9c, 0xe9, 0x86, 0x8e, 0xdb, 0xcd, 0xa4, 0x30, 0xd6, 0x90, 0x33, 0xe2, 0x11, 0xdf, 0x19, 0xa1,
0x8f, 0xa1, 0x6c, 0x74, 0xcd, 0xfe, 0x79, 0xf3, 0x71, 0x92, 0x3a, 0xdb, 0x3f, 0x15, 0xaa, 0x3f,
0x1f, 0x5d, 0x30, 0x3e, 0x8f, 0xa3, 0x1a, 0x7a, 0x5a, 0xef, 0xb4, 0x5b, 0x1c, 0x9a, 0x55, 0x95,
0xe5, 0x4a, 0xbb, 0xbd, 0x86, 0xb6, 0xf9, 0x3b, 0x26, 0xc2, 0x1e, 0xdb, 0x50, 0xf9, 0xfe, 0xc6,
0x84, 0x34, 0xc8, 0xd7, 0x7b, 0x3d, 0xdd, 0x68, 0x25, 0x7f, 0xbf, 0xf1, 0xd5, 0x67, 0x33, 0xe2,
0xd9, 0x11, 0xe2, 0xb4, 0x8b, 0xcf, 0xf4, 0x41, 0xf2, 0xf3, 0x1b, 0xc4, 0x29, 0x8d, 0xae, 0xf7,
0xc6, 0xe1, 0xab, 0xef, 0x2a, 0x3b, 0xdf, 0x7e, 0x57, 0xd9, 0x79, 0x75, 0x55, 0x11, 0xbe, 0xbd,
0xaa, 0x08, 0xff, 0xba, 0xaa, 0xec, 0xfc, 0xe7, 0xaa, 0x22, 0x7c, 0xf3, 0xef, 0x8a, 0x30, 0xcc,
0xb3, 0x46, 0xf6, 0xf9, 0xff, 0x03, 0x00, 0x00, 0xff, 0xff, 0xe5, 0x17, 0x2b, 0x62, 0xdd, 0x0e,
0x00, 0x00,
}

View File

@ -146,6 +146,7 @@ message Request {
int32 size = 5;
bytes hash = 6;
bool from_temporary = 7;
uint32 weak_hash = 8;
}
// Response

View File

@ -11,6 +11,7 @@ type TestModel struct {
offset int64
size int
hash []byte
weakHash uint32
fromTemporary bool
closedCh chan struct{}
closedErr error
@ -28,12 +29,13 @@ func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo) {
func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) {
}
func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, hash []byte, weakHash uint32, fromTemporary bool, buf []byte) error {
t.folder = folder
t.name = name
t.offset = offset
t.size = len(buf)
t.hash = hash
t.weakHash = weakHash
t.fromTemporary = fromTemporary
copy(buf, t.data)
return nil

View File

@ -26,7 +26,7 @@ func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileI
m.Model.IndexUpdate(deviceID, folder, files)
}
func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, weakHash uint32, fromTemporary bool, buf []byte) error {
name = norm.NFD.String(name)
return m.Model.Request(deviceID, folder, name, offset, hash, fromTemporary, buf)
return m.Model.Request(deviceID, folder, name, offset, hash, weakHash, fromTemporary, buf)
}

View File

@ -25,14 +25,14 @@ func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileI
m.Model.IndexUpdate(deviceID, folder, files)
}
func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, weakHash uint32, fromTemporary bool, buf []byte) error {
if strings.Contains(name, `\`) {
l.Warnf("Dropping request for %s, contains invalid path separator", name)
return ErrNoSuchFile
}
name = filepath.FromSlash(name)
return m.Model.Request(deviceID, folder, name, offset, hash, fromTemporary, buf)
return m.Model.Request(deviceID, folder, name, offset, hash, weakHash, fromTemporary, buf)
}
func fixupFiles(files []FileInfo) []FileInfo {

View File

@ -107,7 +107,7 @@ type Model interface {
// An index update was received from the peer device
IndexUpdate(deviceID DeviceID, folder string, files []FileInfo)
// A request was made by the peer device
Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error
Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, weakHash uint32, fromTemporary bool, buf []byte) error
// A cluster configuration message was received
ClusterConfig(deviceID DeviceID, config ClusterConfig)
// The peer device closed the connection
@ -122,7 +122,7 @@ type Connection interface {
Name() string
Index(folder string, files []FileInfo) error
IndexUpdate(folder string, files []FileInfo) error
Request(folder string, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error)
Request(folder string, name string, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error)
ClusterConfig(config ClusterConfig)
DownloadProgress(folder string, updates []FileDownloadProgressUpdate)
Statistics() Statistics
@ -254,7 +254,7 @@ func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo) error {
}
// Request returns the bytes for the specified block after fetching them from the connected peer.
func (c *rawConnection) Request(folder string, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
func (c *rawConnection) Request(folder string, name string, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) {
c.nextIDMut.Lock()
id := c.nextID
c.nextID++
@ -275,6 +275,7 @@ func (c *rawConnection) Request(folder string, name string, offset int64, size i
Offset: offset,
Size: int32(size),
Hash: hash,
WeakHash: weakHash,
FromTemporary: fromTemporary,
}, nil)
if !ok {
@ -584,7 +585,7 @@ func (c *rawConnection) handleRequest(req Request) {
buf = make([]byte, size)
}
err := c.receiver.Request(c.id, req.Folder, req.Name, req.Offset, req.Hash, req.FromTemporary, buf)
err := c.receiver.Request(c.id, req.Folder, req.Name, req.Offset, req.Hash, req.WeakHash, req.FromTemporary, buf)
if err != nil {
c.send(&Response{
ID: req.ID,

View File

@ -72,7 +72,7 @@ func TestClose(t *testing.T) {
c0.Index("default", nil)
c0.Index("default", nil)
if _, err := c0.Request("default", "foo", 0, 0, nil, false); err == nil {
if _, err := c0.Request("default", "foo", 0, 0, nil, 0, false); err == nil {
t.Error("Request should return an error")
}
}

View File

@ -34,7 +34,7 @@ func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo) error {
return c.Connection.IndexUpdate(folder, myFs)
}
func (c wireFormatConnection) Request(folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
func (c wireFormatConnection) Request(folder, name string, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) {
name = norm.NFC.String(filepath.ToSlash(name))
return c.Connection.Request(folder, name, offset, size, hash, fromTemporary)
return c.Connection.Request(folder, name, offset, size, hash, weakHash, fromTemporary)
}

View File

@ -62,26 +62,24 @@ func HashFile(ctx context.Context, fs fs.Filesystem, path string, blockSize int,
// workers are used in parallel. The outbox will become closed when the inbox
// is closed and all items handled.
type parallelHasher struct {
fs fs.Filesystem
workers int
outbox chan<- protocol.FileInfo
inbox <-chan protocol.FileInfo
counter Counter
done chan<- struct{}
useWeakHashes bool
wg sync.WaitGroup
fs fs.Filesystem
workers int
outbox chan<- protocol.FileInfo
inbox <-chan protocol.FileInfo
counter Counter
done chan<- struct{}
wg sync.WaitGroup
}
func newParallelHasher(ctx context.Context, fs fs.Filesystem, workers int, outbox chan<- protocol.FileInfo, inbox <-chan protocol.FileInfo, counter Counter, done chan<- struct{}, useWeakHashes bool) {
func newParallelHasher(ctx context.Context, fs fs.Filesystem, workers int, outbox chan<- protocol.FileInfo, inbox <-chan protocol.FileInfo, counter Counter, done chan<- struct{}) {
ph := &parallelHasher{
fs: fs,
workers: workers,
outbox: outbox,
inbox: inbox,
counter: counter,
done: done,
useWeakHashes: useWeakHashes,
wg: sync.NewWaitGroup(),
fs: fs,
workers: workers,
outbox: outbox,
inbox: inbox,
counter: counter,
done: done,
wg: sync.NewWaitGroup(),
}
for i := 0; i < workers; i++ {
@ -106,7 +104,7 @@ func (ph *parallelHasher) hashFiles(ctx context.Context) {
panic("Bug. Asked to hash a directory or a deleted file.")
}
blocks, err := HashFile(ctx, ph.fs, f.Name, f.BlockSize(), ph.counter, ph.useWeakHashes)
blocks, err := HashFile(ctx, ph.fs, f.Name, f.BlockSize(), ph.counter, true)
if err != nil {
l.Debugln("hash error:", f.Name, err)
continue

View File

@ -7,6 +7,7 @@
package scanner
import (
"bytes"
"context"
"hash"
"io"
@ -107,6 +108,29 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou
return blocks, nil
}
func Validate(buf, hash []byte, weakHash uint32) bool {
rd := bytes.NewReader(buf)
if weakHash != 0 {
whf := adler32.New()
if _, err := io.Copy(whf, rd); err == nil && whf.Sum32() == weakHash {
return true
}
// Copy error or mismatch, go to next algo.
rd.Seek(0, io.SeekStart)
}
if len(hash) > 0 {
hf := sha256.New()
if _, err := io.Copy(hf, rd); err == nil {
// Sum allocates, so let's hope we don't hit this often.
return bytes.Equal(hf.Sum(nil), hash)
}
}
// Both algos failed or no hashes were specified. Assume it's all good.
return true
}
type noopHash struct{}
func (noopHash) Sum32() uint32 { return 0 }

View File

@ -64,8 +64,6 @@ type Config struct {
// Optional progress tick interval which defines how often FolderScanProgress
// events are emitted. Negative number means disabled.
ProgressTickIntervalS int
// Whether or not we should also compute weak hashes
UseWeakHashes bool
// Whether to use large blocks for large files or the old standard of 128KiB for everything.
UseLargeBlocks bool
}
@ -120,7 +118,7 @@ func (w *walker) walk(ctx context.Context) chan protocol.FileInfo {
// We're not required to emit scan progress events, just kick off hashers,
// and feed inputs directly from the walker.
if w.ProgressTickIntervalS < 0 {
newParallelHasher(ctx, w.Filesystem, w.Hashers, finishedChan, toHashChan, nil, nil, w.UseWeakHashes)
newParallelHasher(ctx, w.Filesystem, w.Hashers, finishedChan, toHashChan, nil, nil)
return finishedChan
}
@ -151,7 +149,7 @@ func (w *walker) walk(ctx context.Context) chan protocol.FileInfo {
done := make(chan struct{})
progress := newByteCounter()
newParallelHasher(ctx, w.Filesystem, w.Hashers, finishedChan, realToHashChan, progress, done, w.UseWeakHashes)
newParallelHasher(ctx, w.Filesystem, w.Hashers, finishedChan, realToHashChan, progress, done)
// A routine which actually emits the FolderScanProgress events
// every w.ProgressTicker ticks, until the hasher routines terminate.

View File

@ -11,6 +11,10 @@ import (
"testing"
"github.com/chmduquesne/rollinghash/adler32"
"github.com/chmduquesne/rollinghash/bozo32"
"github.com/chmduquesne/rollinghash/buzhash32"
"github.com/chmduquesne/rollinghash/buzhash64"
"github.com/chmduquesne/rollinghash/rabinkarp64"
)
const testFile = "../model/testdata/~syncthing~file.tmp"
@ -59,3 +63,115 @@ func BenchmarkWeakHashAdler32Roll(b *testing.B) {
b.SetBytes(size)
}
func BenchmarkWeakHashRabinKarp64(b *testing.B) {
data := make([]byte, size)
hf := rabinkarp64.New()
for i := 0; i < b.N; i++ {
hf.Write(data)
}
_ = hf.Sum64()
b.SetBytes(size)
}
func BenchmarkWeakHashRabinKarp64Roll(b *testing.B) {
data := make([]byte, size)
hf := rabinkarp64.New()
hf.Write(data)
b.ResetTimer()
for i := 0; i < b.N; i++ {
for i := 0; i <= size; i++ {
hf.Roll('a')
}
}
b.SetBytes(size)
}
func BenchmarkWeakHashBozo32(b *testing.B) {
data := make([]byte, size)
hf := bozo32.New()
for i := 0; i < b.N; i++ {
hf.Write(data)
}
_ = hf.Sum32()
b.SetBytes(size)
}
func BenchmarkWeakHashBozo32Roll(b *testing.B) {
data := make([]byte, size)
hf := bozo32.New()
hf.Write(data)
b.ResetTimer()
for i := 0; i < b.N; i++ {
for i := 0; i <= size; i++ {
hf.Roll('a')
}
}
b.SetBytes(size)
}
func BenchmarkWeakHashBuzhash32(b *testing.B) {
data := make([]byte, size)
hf := buzhash32.New()
for i := 0; i < b.N; i++ {
hf.Write(data)
}
_ = hf.Sum32()
b.SetBytes(size)
}
func BenchmarkWeakHashBuzhash32Roll(b *testing.B) {
data := make([]byte, size)
hf := buzhash32.New()
hf.Write(data)
b.ResetTimer()
for i := 0; i < b.N; i++ {
for i := 0; i <= size; i++ {
hf.Roll('a')
}
}
b.SetBytes(size)
}
func BenchmarkWeakHashBuzhash64(b *testing.B) {
data := make([]byte, size)
hf := buzhash64.New()
for i := 0; i < b.N; i++ {
hf.Write(data)
}
_ = hf.Sum64()
b.SetBytes(size)
}
func BenchmarkWeakHashBuzhash64Roll(b *testing.B) {
data := make([]byte, size)
hf := buzhash64.New()
hf.Write(data)
b.ResetTimer()
for i := 0; i < b.N; i++ {
for i := 0; i <= size; i++ {
hf.Roll('a')
}
}
b.SetBytes(size)
}

View File

@ -20,10 +20,6 @@ const (
maxWeakhashFinderHits = 10
)
var (
Enabled = true
)
// Find finds all the blocks of the given size within io.Reader that matches
// the hashes provided, and returns a hash -> slice of offsets within reader
// map, that produces the same weak hash.

2
vendor/manifest vendored
View File

@ -94,7 +94,7 @@
"importpath": "github.com/chmduquesne/rollinghash",
"repository": "https://github.com/chmduquesne/rollinghash",
"vcs": "git",
"revision": "3dc7875a1f890f9bcf0619adb5571fc6f7d516bb",
"revision": "abb8cbaf9915e48ee20cae94bcd94221b61707a2",
"branch": "master",
"notests": true
},