lib/model: Verify request content against weak (and possibly strong) hash (#4767)
This commit is contained in:
parent
678c80ffe4
commit
ef0dcea6a4
|
@ -46,7 +46,6 @@ import (
|
||||||
"github.com/syncthing/syncthing/lib/sha256"
|
"github.com/syncthing/syncthing/lib/sha256"
|
||||||
"github.com/syncthing/syncthing/lib/tlsutil"
|
"github.com/syncthing/syncthing/lib/tlsutil"
|
||||||
"github.com/syncthing/syncthing/lib/upgrade"
|
"github.com/syncthing/syncthing/lib/upgrade"
|
||||||
"github.com/syncthing/syncthing/lib/weakhash"
|
|
||||||
|
|
||||||
"github.com/thejerf/suture"
|
"github.com/thejerf/suture"
|
||||||
|
|
||||||
|
@ -697,26 +696,8 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
if opts := cfg.Options(); opts.WeakHashSelectionMethod == config.WeakHashAuto {
|
perf := cpuBench(3, 150*time.Millisecond, true)
|
||||||
perfWithWeakHash := cpuBench(3, 150*time.Millisecond, true)
|
l.Infof("Hashing performance is %.02f MB/s", perf)
|
||||||
l.Infof("Hashing performance with rolling hash is %.02f MB/s", perfWithWeakHash)
|
|
||||||
perfWithoutWeakHash := cpuBench(3, 150*time.Millisecond, false)
|
|
||||||
l.Infof("Hashing performance without rolling hash is %.02f MB/s", perfWithoutWeakHash)
|
|
||||||
|
|
||||||
if perfWithoutWeakHash*0.8 > perfWithWeakHash {
|
|
||||||
l.Infof("Rolling hash disabled, as it has an unacceptable performance impact.")
|
|
||||||
weakhash.Enabled = false
|
|
||||||
} else {
|
|
||||||
l.Infof("Rolling hash enabled, as it has an acceptable performance impact.")
|
|
||||||
weakhash.Enabled = true
|
|
||||||
}
|
|
||||||
} else if opts.WeakHashSelectionMethod == config.WeakHashNever {
|
|
||||||
l.Infof("Disabling rolling hash")
|
|
||||||
weakhash.Enabled = false
|
|
||||||
} else if opts.WeakHashSelectionMethod == config.WeakHashAlways {
|
|
||||||
l.Infof("Enabling rolling hash")
|
|
||||||
weakhash.Enabled = true
|
|
||||||
}
|
|
||||||
|
|
||||||
dbFile := locations[locDatabase]
|
dbFile := locations[locDatabase]
|
||||||
ldb, err := db.Open(dbFile)
|
ldb, err := db.Open(dbFile)
|
||||||
|
|
|
@ -26,7 +26,6 @@ import (
|
||||||
"github.com/syncthing/syncthing/lib/protocol"
|
"github.com/syncthing/syncthing/lib/protocol"
|
||||||
"github.com/syncthing/syncthing/lib/scanner"
|
"github.com/syncthing/syncthing/lib/scanner"
|
||||||
"github.com/syncthing/syncthing/lib/upgrade"
|
"github.com/syncthing/syncthing/lib/upgrade"
|
||||||
"github.com/syncthing/syncthing/lib/weakhash"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Current version number of the usage report, for acceptance purposes. If
|
// Current version number of the usage report, for acceptance purposes. If
|
||||||
|
@ -190,8 +189,6 @@ func reportData(cfg configIntf, m modelIntf, connectionsService connectionsIntf,
|
||||||
res["overwriteRemoteDeviceNames"] = opts.OverwriteRemoteDevNames
|
res["overwriteRemoteDeviceNames"] = opts.OverwriteRemoteDevNames
|
||||||
res["progressEmitterEnabled"] = opts.ProgressUpdateIntervalS > -1
|
res["progressEmitterEnabled"] = opts.ProgressUpdateIntervalS > -1
|
||||||
res["customDefaultFolderPath"] = opts.DefaultFolderPath != "~"
|
res["customDefaultFolderPath"] = opts.DefaultFolderPath != "~"
|
||||||
res["weakHashSelection"] = opts.WeakHashSelectionMethod.String()
|
|
||||||
res["weakHashEnabled"] = weakhash.Enabled
|
|
||||||
res["customTrafficClass"] = opts.TrafficClass != 0
|
res["customTrafficClass"] = opts.TrafficClass != 0
|
||||||
res["customTempIndexMinBlocks"] = opts.TempIndexMinBlocks != 10
|
res["customTempIndexMinBlocks"] = opts.TempIndexMinBlocks != 10
|
||||||
res["temporariesDisabled"] = opts.KeepTemporariesH == 0
|
res["temporariesDisabled"] = opts.KeepTemporariesH == 0
|
||||||
|
|
|
@ -67,7 +67,6 @@ func TestDefaultValues(t *testing.T) {
|
||||||
OverwriteRemoteDevNames: false,
|
OverwriteRemoteDevNames: false,
|
||||||
TempIndexMinBlocks: 10,
|
TempIndexMinBlocks: 10,
|
||||||
UnackedNotificationIDs: []string{},
|
UnackedNotificationIDs: []string{},
|
||||||
WeakHashSelectionMethod: WeakHashAuto,
|
|
||||||
DefaultFolderPath: "~",
|
DefaultFolderPath: "~",
|
||||||
SetLowPriority: true,
|
SetLowPriority: true,
|
||||||
}
|
}
|
||||||
|
@ -209,9 +208,8 @@ func TestOverriddenValues(t *testing.T) {
|
||||||
"channelNotification", // added in 17->18 migration
|
"channelNotification", // added in 17->18 migration
|
||||||
"fsWatcherNotification", // added in 27->28 migration
|
"fsWatcherNotification", // added in 27->28 migration
|
||||||
},
|
},
|
||||||
WeakHashSelectionMethod: WeakHashNever,
|
DefaultFolderPath: "/media/syncthing",
|
||||||
DefaultFolderPath: "/media/syncthing",
|
SetLowPriority: false,
|
||||||
SetLowPriority: false,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
os.Unsetenv("STNOUPGRADE")
|
os.Unsetenv("STNOUPGRADE")
|
||||||
|
|
|
@ -7,135 +7,50 @@
|
||||||
package config
|
package config
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
|
||||||
"encoding/xml"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"github.com/syncthing/syncthing/lib/util"
|
"github.com/syncthing/syncthing/lib/util"
|
||||||
)
|
)
|
||||||
|
|
||||||
type WeakHashSelectionMethod int
|
|
||||||
|
|
||||||
const (
|
|
||||||
WeakHashAuto WeakHashSelectionMethod = iota
|
|
||||||
WeakHashAlways
|
|
||||||
WeakHashNever
|
|
||||||
)
|
|
||||||
|
|
||||||
func (m WeakHashSelectionMethod) MarshalString() (string, error) {
|
|
||||||
switch m {
|
|
||||||
case WeakHashAuto:
|
|
||||||
return "auto", nil
|
|
||||||
case WeakHashAlways:
|
|
||||||
return "always", nil
|
|
||||||
case WeakHashNever:
|
|
||||||
return "never", nil
|
|
||||||
default:
|
|
||||||
return "", fmt.Errorf("unrecognized hash selection method")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m WeakHashSelectionMethod) String() string {
|
|
||||||
s, err := m.MarshalString()
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *WeakHashSelectionMethod) UnmarshalString(value string) error {
|
|
||||||
switch value {
|
|
||||||
case "auto":
|
|
||||||
*m = WeakHashAuto
|
|
||||||
return nil
|
|
||||||
case "always":
|
|
||||||
*m = WeakHashAlways
|
|
||||||
return nil
|
|
||||||
case "never":
|
|
||||||
*m = WeakHashNever
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return fmt.Errorf("unrecognized hash selection method")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m WeakHashSelectionMethod) MarshalJSON() ([]byte, error) {
|
|
||||||
val, err := m.MarshalString()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return json.Marshal(val)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *WeakHashSelectionMethod) UnmarshalJSON(data []byte) error {
|
|
||||||
var value string
|
|
||||||
if err := json.Unmarshal(data, &value); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return m.UnmarshalString(value)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m WeakHashSelectionMethod) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
|
|
||||||
val, err := m.MarshalString()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return e.EncodeElement(val, start)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *WeakHashSelectionMethod) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
|
||||||
var value string
|
|
||||||
if err := d.DecodeElement(&value, &start); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return m.UnmarshalString(value)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (WeakHashSelectionMethod) ParseDefault(value string) (interface{}, error) {
|
|
||||||
var m WeakHashSelectionMethod
|
|
||||||
err := m.UnmarshalString(value)
|
|
||||||
return m, err
|
|
||||||
}
|
|
||||||
|
|
||||||
type OptionsConfiguration struct {
|
type OptionsConfiguration struct {
|
||||||
ListenAddresses []string `xml:"listenAddress" json:"listenAddresses" default:"default"`
|
ListenAddresses []string `xml:"listenAddress" json:"listenAddresses" default:"default"`
|
||||||
GlobalAnnServers []string `xml:"globalAnnounceServer" json:"globalAnnounceServers" json:"globalAnnounceServer" default:"default" restart:"true"`
|
GlobalAnnServers []string `xml:"globalAnnounceServer" json:"globalAnnounceServers" json:"globalAnnounceServer" default:"default" restart:"true"`
|
||||||
GlobalAnnEnabled bool `xml:"globalAnnounceEnabled" json:"globalAnnounceEnabled" default:"true" restart:"true"`
|
GlobalAnnEnabled bool `xml:"globalAnnounceEnabled" json:"globalAnnounceEnabled" default:"true" restart:"true"`
|
||||||
LocalAnnEnabled bool `xml:"localAnnounceEnabled" json:"localAnnounceEnabled" default:"true" restart:"true"`
|
LocalAnnEnabled bool `xml:"localAnnounceEnabled" json:"localAnnounceEnabled" default:"true" restart:"true"`
|
||||||
LocalAnnPort int `xml:"localAnnouncePort" json:"localAnnouncePort" default:"21027" restart:"true"`
|
LocalAnnPort int `xml:"localAnnouncePort" json:"localAnnouncePort" default:"21027" restart:"true"`
|
||||||
LocalAnnMCAddr string `xml:"localAnnounceMCAddr" json:"localAnnounceMCAddr" default:"[ff12::8384]:21027" restart:"true"`
|
LocalAnnMCAddr string `xml:"localAnnounceMCAddr" json:"localAnnounceMCAddr" default:"[ff12::8384]:21027" restart:"true"`
|
||||||
MaxSendKbps int `xml:"maxSendKbps" json:"maxSendKbps"`
|
MaxSendKbps int `xml:"maxSendKbps" json:"maxSendKbps"`
|
||||||
MaxRecvKbps int `xml:"maxRecvKbps" json:"maxRecvKbps"`
|
MaxRecvKbps int `xml:"maxRecvKbps" json:"maxRecvKbps"`
|
||||||
ReconnectIntervalS int `xml:"reconnectionIntervalS" json:"reconnectionIntervalS" default:"60"`
|
ReconnectIntervalS int `xml:"reconnectionIntervalS" json:"reconnectionIntervalS" default:"60"`
|
||||||
RelaysEnabled bool `xml:"relaysEnabled" json:"relaysEnabled" default:"true"`
|
RelaysEnabled bool `xml:"relaysEnabled" json:"relaysEnabled" default:"true"`
|
||||||
RelayReconnectIntervalM int `xml:"relayReconnectIntervalM" json:"relayReconnectIntervalM" default:"10"`
|
RelayReconnectIntervalM int `xml:"relayReconnectIntervalM" json:"relayReconnectIntervalM" default:"10"`
|
||||||
StartBrowser bool `xml:"startBrowser" json:"startBrowser" default:"true"`
|
StartBrowser bool `xml:"startBrowser" json:"startBrowser" default:"true"`
|
||||||
NATEnabled bool `xml:"natEnabled" json:"natEnabled" default:"true"`
|
NATEnabled bool `xml:"natEnabled" json:"natEnabled" default:"true"`
|
||||||
NATLeaseM int `xml:"natLeaseMinutes" json:"natLeaseMinutes" default:"60"`
|
NATLeaseM int `xml:"natLeaseMinutes" json:"natLeaseMinutes" default:"60"`
|
||||||
NATRenewalM int `xml:"natRenewalMinutes" json:"natRenewalMinutes" default:"30"`
|
NATRenewalM int `xml:"natRenewalMinutes" json:"natRenewalMinutes" default:"30"`
|
||||||
NATTimeoutS int `xml:"natTimeoutSeconds" json:"natTimeoutSeconds" default:"10"`
|
NATTimeoutS int `xml:"natTimeoutSeconds" json:"natTimeoutSeconds" default:"10"`
|
||||||
URAccepted int `xml:"urAccepted" json:"urAccepted"` // Accepted usage reporting version; 0 for off (undecided), -1 for off (permanently)
|
URAccepted int `xml:"urAccepted" json:"urAccepted"` // Accepted usage reporting version; 0 for off (undecided), -1 for off (permanently)
|
||||||
URSeen int `xml:"urSeen" json:"urSeen"` // Report which the user has been prompted for.
|
URSeen int `xml:"urSeen" json:"urSeen"` // Report which the user has been prompted for.
|
||||||
URUniqueID string `xml:"urUniqueID" json:"urUniqueId"` // Unique ID for reporting purposes, regenerated when UR is turned on.
|
URUniqueID string `xml:"urUniqueID" json:"urUniqueId"` // Unique ID for reporting purposes, regenerated when UR is turned on.
|
||||||
URURL string `xml:"urURL" json:"urURL" default:"https://data.syncthing.net/newdata"`
|
URURL string `xml:"urURL" json:"urURL" default:"https://data.syncthing.net/newdata"`
|
||||||
URPostInsecurely bool `xml:"urPostInsecurely" json:"urPostInsecurely" default:"false"` // For testing
|
URPostInsecurely bool `xml:"urPostInsecurely" json:"urPostInsecurely" default:"false"` // For testing
|
||||||
URInitialDelayS int `xml:"urInitialDelayS" json:"urInitialDelayS" default:"1800"`
|
URInitialDelayS int `xml:"urInitialDelayS" json:"urInitialDelayS" default:"1800"`
|
||||||
RestartOnWakeup bool `xml:"restartOnWakeup" json:"restartOnWakeup" default:"true" restart:"true"`
|
RestartOnWakeup bool `xml:"restartOnWakeup" json:"restartOnWakeup" default:"true" restart:"true"`
|
||||||
AutoUpgradeIntervalH int `xml:"autoUpgradeIntervalH" json:"autoUpgradeIntervalH" default:"12" restart:"true"` // 0 for off
|
AutoUpgradeIntervalH int `xml:"autoUpgradeIntervalH" json:"autoUpgradeIntervalH" default:"12" restart:"true"` // 0 for off
|
||||||
UpgradeToPreReleases bool `xml:"upgradeToPreReleases" json:"upgradeToPreReleases" restart:"true"` // when auto upgrades are enabled
|
UpgradeToPreReleases bool `xml:"upgradeToPreReleases" json:"upgradeToPreReleases" restart:"true"` // when auto upgrades are enabled
|
||||||
KeepTemporariesH int `xml:"keepTemporariesH" json:"keepTemporariesH" default:"24"` // 0 for off
|
KeepTemporariesH int `xml:"keepTemporariesH" json:"keepTemporariesH" default:"24"` // 0 for off
|
||||||
CacheIgnoredFiles bool `xml:"cacheIgnoredFiles" json:"cacheIgnoredFiles" default:"false" restart:"true"`
|
CacheIgnoredFiles bool `xml:"cacheIgnoredFiles" json:"cacheIgnoredFiles" default:"false" restart:"true"`
|
||||||
ProgressUpdateIntervalS int `xml:"progressUpdateIntervalS" json:"progressUpdateIntervalS" default:"5"`
|
ProgressUpdateIntervalS int `xml:"progressUpdateIntervalS" json:"progressUpdateIntervalS" default:"5"`
|
||||||
LimitBandwidthInLan bool `xml:"limitBandwidthInLan" json:"limitBandwidthInLan" default:"false"`
|
LimitBandwidthInLan bool `xml:"limitBandwidthInLan" json:"limitBandwidthInLan" default:"false"`
|
||||||
MinHomeDiskFree Size `xml:"minHomeDiskFree" json:"minHomeDiskFree" default:"1 %"`
|
MinHomeDiskFree Size `xml:"minHomeDiskFree" json:"minHomeDiskFree" default:"1 %"`
|
||||||
ReleasesURL string `xml:"releasesURL" json:"releasesURL" default:"https://upgrades.syncthing.net/meta.json" restart:"true"`
|
ReleasesURL string `xml:"releasesURL" json:"releasesURL" default:"https://upgrades.syncthing.net/meta.json" restart:"true"`
|
||||||
AlwaysLocalNets []string `xml:"alwaysLocalNet" json:"alwaysLocalNets"`
|
AlwaysLocalNets []string `xml:"alwaysLocalNet" json:"alwaysLocalNets"`
|
||||||
OverwriteRemoteDevNames bool `xml:"overwriteRemoteDeviceNamesOnConnect" json:"overwriteRemoteDeviceNamesOnConnect" default:"false"`
|
OverwriteRemoteDevNames bool `xml:"overwriteRemoteDeviceNamesOnConnect" json:"overwriteRemoteDeviceNamesOnConnect" default:"false"`
|
||||||
TempIndexMinBlocks int `xml:"tempIndexMinBlocks" json:"tempIndexMinBlocks" default:"10"`
|
TempIndexMinBlocks int `xml:"tempIndexMinBlocks" json:"tempIndexMinBlocks" default:"10"`
|
||||||
UnackedNotificationIDs []string `xml:"unackedNotificationID" json:"unackedNotificationIDs"`
|
UnackedNotificationIDs []string `xml:"unackedNotificationID" json:"unackedNotificationIDs"`
|
||||||
TrafficClass int `xml:"trafficClass" json:"trafficClass"`
|
TrafficClass int `xml:"trafficClass" json:"trafficClass"`
|
||||||
WeakHashSelectionMethod WeakHashSelectionMethod `xml:"weakHashSelectionMethod" json:"weakHashSelectionMethod" restart:"true"`
|
DefaultFolderPath string `xml:"defaultFolderPath" json:"defaultFolderPath" default:"~"`
|
||||||
DefaultFolderPath string `xml:"defaultFolderPath" json:"defaultFolderPath" default:"~"`
|
SetLowPriority bool `xml:"setLowPriority" json:"setLowPriority" default:"true"`
|
||||||
SetLowPriority bool `xml:"setLowPriority" json:"setLowPriority" default:"true"`
|
|
||||||
|
|
||||||
DeprecatedUPnPEnabled bool `xml:"upnpEnabled,omitempty" json:"-"`
|
DeprecatedUPnPEnabled bool `xml:"upnpEnabled,omitempty" json:"-"`
|
||||||
DeprecatedUPnPLeaseM int `xml:"upnpLeaseMinutes,omitempty" json:"-"`
|
DeprecatedUPnPLeaseM int `xml:"upnpLeaseMinutes,omitempty" json:"-"`
|
||||||
|
|
|
@ -34,7 +34,6 @@
|
||||||
<releasesURL>https://localhost/releases</releasesURL>
|
<releasesURL>https://localhost/releases</releasesURL>
|
||||||
<overwriteRemoteDeviceNamesOnConnect>true</overwriteRemoteDeviceNamesOnConnect>
|
<overwriteRemoteDeviceNamesOnConnect>true</overwriteRemoteDeviceNamesOnConnect>
|
||||||
<tempIndexMinBlocks>100</tempIndexMinBlocks>
|
<tempIndexMinBlocks>100</tempIndexMinBlocks>
|
||||||
<weakHashSelectionMethod>never</weakHashSelectionMethod>
|
|
||||||
<defaultFolderPath>/media/syncthing</defaultFolderPath>
|
<defaultFolderPath>/media/syncthing</defaultFolderPath>
|
||||||
<setLowPriority>false</setLowPriority>
|
<setLowPriority>false</setLowPriority>
|
||||||
</options>
|
</options>
|
||||||
|
|
|
@ -182,19 +182,6 @@ func (f *BlockFinder) Iterate(folders []string, hash []byte, iterFn func(string,
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fix repairs incorrect blockmap entries, removing the old entry and
|
|
||||||
// replacing it with a new entry for the given block
|
|
||||||
func (f *BlockFinder) Fix(folder, file string, index int32, oldHash, newHash []byte) error {
|
|
||||||
buf := make([]byte, 4)
|
|
||||||
binary.BigEndian.PutUint32(buf, uint32(index))
|
|
||||||
|
|
||||||
folderID := f.db.folderIdx.ID([]byte(folder))
|
|
||||||
batch := new(leveldb.Batch)
|
|
||||||
batch.Delete(blockKeyInto(nil, oldHash, folderID, file))
|
|
||||||
batch.Put(blockKeyInto(nil, newHash, folderID, file), buf)
|
|
||||||
return f.db.Write(batch, nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
// m.blockKey returns a byte slice encoding the following information:
|
// m.blockKey returns a byte slice encoding the following information:
|
||||||
// keyTypeBlock (1 byte)
|
// keyTypeBlock (1 byte)
|
||||||
// folder (4 bytes)
|
// folder (4 bytes)
|
||||||
|
|
|
@ -219,34 +219,3 @@ func TestBlockFinderLookup(t *testing.T) {
|
||||||
|
|
||||||
f1.Deleted = false
|
f1.Deleted = false
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBlockFinderFix(t *testing.T) {
|
|
||||||
db, f := setup()
|
|
||||||
|
|
||||||
iterFn := func(folder, file string, index int32) bool {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
m := NewBlockMap(db, db.folderIdx.ID([]byte("folder1")))
|
|
||||||
err := m.Add([]protocol.FileInfo{f1})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !f.Iterate(folders, f1.Blocks[0].Hash, iterFn) {
|
|
||||||
t.Fatal("Block not found")
|
|
||||||
}
|
|
||||||
|
|
||||||
err = f.Fix("folder1", f1.Name, 0, f1.Blocks[0].Hash, f2.Blocks[0].Hash)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if f.Iterate(folders, f1.Blocks[0].Hash, iterFn) {
|
|
||||||
t.Fatal("Unexpected block")
|
|
||||||
}
|
|
||||||
|
|
||||||
if !f.Iterate(folders, f2.Blocks[0].Hash, iterFn) {
|
|
||||||
t.Fatal("Block not found")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
@ -7,6 +7,7 @@
|
||||||
package model
|
package model
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/tls"
|
"crypto/tls"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
@ -34,7 +35,6 @@ import (
|
||||||
"github.com/syncthing/syncthing/lib/sync"
|
"github.com/syncthing/syncthing/lib/sync"
|
||||||
"github.com/syncthing/syncthing/lib/upgrade"
|
"github.com/syncthing/syncthing/lib/upgrade"
|
||||||
"github.com/syncthing/syncthing/lib/versioner"
|
"github.com/syncthing/syncthing/lib/versioner"
|
||||||
"github.com/syncthing/syncthing/lib/weakhash"
|
|
||||||
"github.com/thejerf/suture"
|
"github.com/thejerf/suture"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -1304,7 +1304,7 @@ func (m *Model) closeLocked(device protocol.DeviceID) {
|
||||||
|
|
||||||
// Request returns the specified data segment by reading it from local disk.
|
// Request returns the specified data segment by reading it from local disk.
|
||||||
// Implements the protocol.Model interface.
|
// Implements the protocol.Model interface.
|
||||||
func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
|
func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte, weakHash uint32, fromTemporary bool, buf []byte) error {
|
||||||
if offset < 0 {
|
if offset < 0 {
|
||||||
return protocol.ErrInvalid
|
return protocol.ErrInvalid
|
||||||
}
|
}
|
||||||
|
@ -1362,8 +1362,8 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
|
||||||
// other than a regular file.
|
// other than a regular file.
|
||||||
return protocol.ErrNoSuchFile
|
return protocol.ErrNoSuchFile
|
||||||
}
|
}
|
||||||
|
err := readOffsetIntoBuf(folderFs, tempFn, offset, buf)
|
||||||
if err := readOffsetIntoBuf(folderFs, tempFn, offset, buf); err == nil {
|
if err == nil && scanner.Validate(buf, hash, weakHash) {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
// Fall through to reading from a non-temp file, just incase the temp
|
// Fall through to reading from a non-temp file, just incase the temp
|
||||||
|
@ -1382,9 +1382,55 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
|
||||||
return protocol.ErrGeneric
|
return protocol.ErrGeneric
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if !scanner.Validate(buf, hash, weakHash) {
|
||||||
|
m.recheckFile(deviceID, folderFs, folder, name, int(offset)/len(buf), hash)
|
||||||
|
return protocol.ErrNoSuchFile
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (m *Model) recheckFile(deviceID protocol.DeviceID, folderFs fs.Filesystem, folder, name string, blockIndex int, hash []byte) {
|
||||||
|
cf, ok := m.CurrentFolderFile(folder, name)
|
||||||
|
if !ok {
|
||||||
|
l.Debugf("%v recheckFile: %s: %q / %q: no current file", m, deviceID, folder, name)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if cf.IsDeleted() || cf.IsInvalid() || cf.IsSymlink() || cf.IsDirectory() {
|
||||||
|
l.Debugf("%v recheckFile: %s: %q / %q: not a regular file", m, deviceID, folder, name)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if blockIndex > len(cf.Blocks) {
|
||||||
|
l.Debugf("%v recheckFile: %s: %q / %q i=%d: block index too far", m, deviceID, folder, name, blockIndex)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
block := cf.Blocks[blockIndex]
|
||||||
|
|
||||||
|
// Seems to want a different version of the file, whatever.
|
||||||
|
if !bytes.Equal(block.Hash, hash) {
|
||||||
|
l.Debugf("%v recheckFile: %s: %q / %q i=%d: hash mismatch %x != %x", m, deviceID, folder, name, blockIndex, block.Hash, hash)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// The hashes provided part of the request match what we expect to find according
|
||||||
|
// to what we have in the database, yet the content we've read off the filesystem doesn't
|
||||||
|
// Something is fishy, invalidate the file and rescan it.
|
||||||
|
cf.Invalidate(m.shortID)
|
||||||
|
|
||||||
|
// Update the index and tell others
|
||||||
|
// The file will temporarily become invalid, which is ok as the content is messed up.
|
||||||
|
m.updateLocalsFromScanning(folder, []protocol.FileInfo{cf})
|
||||||
|
|
||||||
|
if err := m.ScanFolderSubdirs(folder, []string{name}); err != nil {
|
||||||
|
l.Debugf("%v recheckFile: %s: %q / %q rescan: %s", m, deviceID, folder, name, err)
|
||||||
|
} else {
|
||||||
|
l.Debugf("%v recheckFile: %s: %q / %q", m, deviceID, folder, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (m *Model) CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool) {
|
func (m *Model) CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool) {
|
||||||
m.fmut.RLock()
|
m.fmut.RLock()
|
||||||
fs, ok := m.folderFiles[folder]
|
fs, ok := m.folderFiles[folder]
|
||||||
|
@ -1836,7 +1882,7 @@ func (m *Model) diskChangeDetected(folderCfg config.FolderConfiguration, files [
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *Model) requestGlobal(deviceID protocol.DeviceID, folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
|
func (m *Model) requestGlobal(deviceID protocol.DeviceID, folder, name string, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) {
|
||||||
m.pmut.RLock()
|
m.pmut.RLock()
|
||||||
nc, ok := m.conn[deviceID]
|
nc, ok := m.conn[deviceID]
|
||||||
m.pmut.RUnlock()
|
m.pmut.RUnlock()
|
||||||
|
@ -1845,9 +1891,9 @@ func (m *Model) requestGlobal(deviceID protocol.DeviceID, folder, name string, o
|
||||||
return nil, fmt.Errorf("requestGlobal: no such device: %s", deviceID)
|
return nil, fmt.Errorf("requestGlobal: no such device: %s", deviceID)
|
||||||
}
|
}
|
||||||
|
|
||||||
l.Debugf("%v REQ(out): %s: %q / %q o=%d s=%d h=%x ft=%t", m, deviceID, folder, name, offset, size, hash, fromTemporary)
|
l.Debugf("%v REQ(out): %s: %q / %q o=%d s=%d h=%x wh=%x ft=%t", m, deviceID, folder, name, offset, size, hash, weakHash, fromTemporary)
|
||||||
|
|
||||||
return nc.Request(folder, name, offset, size, hash, fromTemporary)
|
return nc.Request(folder, name, offset, size, hash, weakHash, fromTemporary)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *Model) ScanFolders() map[string]error {
|
func (m *Model) ScanFolders() map[string]error {
|
||||||
|
@ -1973,7 +2019,6 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
|
||||||
Hashers: m.numHashers(folder),
|
Hashers: m.numHashers(folder),
|
||||||
ShortID: m.shortID,
|
ShortID: m.shortID,
|
||||||
ProgressTickIntervalS: folderCfg.ScanProgressIntervalS,
|
ProgressTickIntervalS: folderCfg.ScanProgressIntervalS,
|
||||||
UseWeakHashes: weakhash.Enabled,
|
|
||||||
UseLargeBlocks: folderCfg.UseLargeBlocks,
|
UseLargeBlocks: folderCfg.UseLargeBlocks,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
|
@ -181,7 +181,7 @@ func TestRequest(t *testing.T) {
|
||||||
|
|
||||||
// Existing, shared file
|
// Existing, shared file
|
||||||
bs = bs[:6]
|
bs = bs[:6]
|
||||||
err := m.Request(device1, "default", "foo", 0, nil, false, bs)
|
err := m.Request(device1, "default", "foo", 0, nil, 0, false, bs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Error(err)
|
t.Error(err)
|
||||||
}
|
}
|
||||||
|
@ -190,32 +190,32 @@ func TestRequest(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Existing, nonshared file
|
// Existing, nonshared file
|
||||||
err = m.Request(device2, "default", "foo", 0, nil, false, bs)
|
err = m.Request(device2, "default", "foo", 0, nil, 0, false, bs)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Error("Unexpected nil error on insecure file read")
|
t.Error("Unexpected nil error on insecure file read")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Nonexistent file
|
// Nonexistent file
|
||||||
err = m.Request(device1, "default", "nonexistent", 0, nil, false, bs)
|
err = m.Request(device1, "default", "nonexistent", 0, nil, 0, false, bs)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Error("Unexpected nil error on insecure file read")
|
t.Error("Unexpected nil error on insecure file read")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Shared folder, but disallowed file name
|
// Shared folder, but disallowed file name
|
||||||
err = m.Request(device1, "default", "../walk.go", 0, nil, false, bs)
|
err = m.Request(device1, "default", "../walk.go", 0, nil, 0, false, bs)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Error("Unexpected nil error on insecure file read")
|
t.Error("Unexpected nil error on insecure file read")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Negative offset
|
// Negative offset
|
||||||
err = m.Request(device1, "default", "foo", -4, nil, false, bs[:0])
|
err = m.Request(device1, "default", "foo", -4, nil, 0, false, bs[:0])
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Error("Unexpected nil error on insecure file read")
|
t.Error("Unexpected nil error on insecure file read")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Larger block than available
|
// Larger block than available
|
||||||
bs = bs[:42]
|
bs = bs[:42]
|
||||||
err = m.Request(device1, "default", "foo", 0, nil, false, bs)
|
err = m.Request(device1, "default", "foo", 0, nil, 0, false, bs)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Error("Unexpected nil error on insecure file read")
|
t.Error("Unexpected nil error on insecure file read")
|
||||||
}
|
}
|
||||||
|
@ -357,7 +357,7 @@ func (f *fakeConnection) IndexUpdate(folder string, fs []protocol.FileInfo) erro
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeConnection) Request(folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
|
func (f *fakeConnection) Request(folder, name string, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) {
|
||||||
f.mut.Lock()
|
f.mut.Lock()
|
||||||
defer f.mut.Unlock()
|
defer f.mut.Unlock()
|
||||||
if f.requestFn != nil {
|
if f.requestFn != nil {
|
||||||
|
@ -485,7 +485,7 @@ func BenchmarkRequestOut(b *testing.B) {
|
||||||
|
|
||||||
b.ResetTimer()
|
b.ResetTimer()
|
||||||
for i := 0; i < b.N; i++ {
|
for i := 0; i < b.N; i++ {
|
||||||
data, err := m.requestGlobal(device1, "default", files[i%n].Name, 0, 32, nil, false)
|
data, err := m.requestGlobal(device1, "default", files[i%n].Name, 0, 32, nil, 0, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
b.Error(err)
|
b.Error(err)
|
||||||
}
|
}
|
||||||
|
@ -513,7 +513,7 @@ func BenchmarkRequestInSingleFile(b *testing.B) {
|
||||||
b.ResetTimer()
|
b.ResetTimer()
|
||||||
|
|
||||||
for i := 0; i < b.N; i++ {
|
for i := 0; i < b.N; i++ {
|
||||||
if err := m.Request(device1, "default", "request/for/a/file/in/a/couple/of/dirs/128k", 0, nil, false, buf); err != nil {
|
if err := m.Request(device1, "default", "request/for/a/file/in/a/couple/of/dirs/128k", 0, nil, 0, false, buf); err != nil {
|
||||||
b.Error(err)
|
b.Error(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -93,7 +93,7 @@ func TestSymlinkTraversalRead(t *testing.T) {
|
||||||
|
|
||||||
// Request a file by traversing the symlink
|
// Request a file by traversing the symlink
|
||||||
buf := make([]byte, 10)
|
buf := make([]byte, 10)
|
||||||
err := m.Request(device1, "default", "symlink/requests_test.go", 0, nil, false, buf)
|
err := m.Request(device1, "default", "symlink/requests_test.go", 0, nil, 0, false, buf)
|
||||||
if err == nil || !bytes.Equal(buf, make([]byte, 10)) {
|
if err == nil || !bytes.Equal(buf, make([]byte, 10)) {
|
||||||
t.Error("Managed to traverse symlink")
|
t.Error("Managed to traverse symlink")
|
||||||
}
|
}
|
||||||
|
@ -464,6 +464,73 @@ func TestIssue4841(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRescanIfHaveInvalidContent(t *testing.T) {
|
||||||
|
m, fc, tmpDir := setupModelWithConnection()
|
||||||
|
defer m.Stop()
|
||||||
|
defer os.RemoveAll(tmpDir)
|
||||||
|
|
||||||
|
payload := []byte("hello")
|
||||||
|
|
||||||
|
if err := ioutil.WriteFile(filepath.Join(tmpDir, "foo"), payload, 0777); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
received := make(chan protocol.FileInfo)
|
||||||
|
fc.mut.Lock()
|
||||||
|
fc.indexFn = func(folder string, fs []protocol.FileInfo) {
|
||||||
|
if len(fs) != 1 {
|
||||||
|
t.Fatalf("Sent index with %d files, should be 1", len(fs))
|
||||||
|
}
|
||||||
|
if fs[0].Name != "foo" {
|
||||||
|
t.Fatalf(`Sent index with file %v, should be "foo"`, fs[0].Name)
|
||||||
|
}
|
||||||
|
received <- fs[0]
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fc.mut.Unlock()
|
||||||
|
|
||||||
|
// Scan without ignore patterns with "foo" not existing locally
|
||||||
|
if err := m.ScanFolder("default"); err != nil {
|
||||||
|
t.Fatal("Failed scanning:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
f := <-received
|
||||||
|
if f.Blocks[0].WeakHash != 103547413 {
|
||||||
|
t.Fatalf("unexpected weak hash: %d != 103547413", f.Blocks[0].WeakHash)
|
||||||
|
}
|
||||||
|
|
||||||
|
buf := make([]byte, len(payload))
|
||||||
|
|
||||||
|
err := m.Request(device2, "default", "foo", 0, f.Blocks[0].Hash, f.Blocks[0].WeakHash, false, buf)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(buf, payload) {
|
||||||
|
t.Errorf("%s != %s", buf, payload)
|
||||||
|
}
|
||||||
|
|
||||||
|
payload = []byte("bye")
|
||||||
|
buf = make([]byte, len(payload))
|
||||||
|
|
||||||
|
if err := ioutil.WriteFile(filepath.Join(tmpDir, "foo"), payload, 0777); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = m.Request(device2, "default", "foo", 0, f.Blocks[0].Hash, f.Blocks[0].WeakHash, false, buf)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("expected failure")
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case f := <-received:
|
||||||
|
if f.Blocks[0].WeakHash != 41943361 {
|
||||||
|
t.Fatalf("unexpected weak hash: %d != 41943361", f.Blocks[0].WeakHash)
|
||||||
|
}
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatalf("timed out")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func setupModelWithConnection() (*Model, *fakeConnection, string) {
|
func setupModelWithConnection() (*Model, *fakeConnection, string) {
|
||||||
tmpDir := createTmpDir()
|
tmpDir := createTmpDir()
|
||||||
cfg := defaultCfgWrapper.RawCopy()
|
cfg := defaultCfgWrapper.RawCopy()
|
||||||
|
|
|
@ -1186,36 +1186,32 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
|
||||||
var file fs.File
|
var file fs.File
|
||||||
var weakHashFinder *weakhash.Finder
|
var weakHashFinder *weakhash.Finder
|
||||||
|
|
||||||
if weakhash.Enabled {
|
blocksPercentChanged := 0
|
||||||
blocksPercentChanged := 0
|
if tot := len(state.file.Blocks); tot > 0 {
|
||||||
if tot := len(state.file.Blocks); tot > 0 {
|
blocksPercentChanged = (tot - state.have) * 100 / tot
|
||||||
blocksPercentChanged = (tot - state.have) * 100 / tot
|
}
|
||||||
|
|
||||||
|
if blocksPercentChanged >= f.WeakHashThresholdPct {
|
||||||
|
hashesToFind := make([]uint32, 0, len(state.blocks))
|
||||||
|
for _, block := range state.blocks {
|
||||||
|
if block.WeakHash != 0 {
|
||||||
|
hashesToFind = append(hashesToFind, block.WeakHash)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if blocksPercentChanged >= f.WeakHashThresholdPct {
|
if len(hashesToFind) > 0 {
|
||||||
hashesToFind := make([]uint32, 0, len(state.blocks))
|
file, err = f.fs.Open(state.file.Name)
|
||||||
for _, block := range state.blocks {
|
if err == nil {
|
||||||
if block.WeakHash != 0 {
|
weakHashFinder, err = weakhash.NewFinder(file, int(state.file.BlockSize()), hashesToFind)
|
||||||
hashesToFind = append(hashesToFind, block.WeakHash)
|
if err != nil {
|
||||||
|
l.Debugln("weak hasher", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(hashesToFind) > 0 {
|
|
||||||
file, err = f.fs.Open(state.file.Name)
|
|
||||||
if err == nil {
|
|
||||||
weakHashFinder, err = weakhash.NewFinder(file, int(state.file.BlockSize()), hashesToFind)
|
|
||||||
if err != nil {
|
|
||||||
l.Debugln("weak hasher", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
l.Debugf("not weak hashing %s. file did not contain any weak hashes", state.file.Name)
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
l.Debugf("not weak hashing %s. not enough changed %.02f < %d", state.file.Name, blocksPercentChanged, f.WeakHashThresholdPct)
|
l.Debugf("not weak hashing %s. file did not contain any weak hashes", state.file.Name)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
l.Debugf("not weak hashing %s. weak hashing disabled", state.file.Name)
|
l.Debugf("not weak hashing %s. not enough changed %.02f < %d", state.file.Name, blocksPercentChanged, f.WeakHashThresholdPct)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, block := range state.blocks {
|
for _, block := range state.blocks {
|
||||||
|
@ -1239,7 +1235,7 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
|
||||||
}
|
}
|
||||||
|
|
||||||
found, err := weakHashFinder.Iterate(block.WeakHash, buf, func(offset int64) bool {
|
found, err := weakHashFinder.Iterate(block.WeakHash, buf, func(offset int64) bool {
|
||||||
if _, err := verifyBuffer(buf, block); err != nil {
|
if verifyBuffer(buf, block) != nil {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1274,17 +1270,8 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
hash, err := verifyBuffer(buf, block)
|
if err := verifyBuffer(buf, block); err != nil {
|
||||||
if err != nil {
|
l.Debugln("Finder failed to verify buffer", err)
|
||||||
if hash != nil {
|
|
||||||
l.Debugf("Finder block mismatch in %s:%s:%d expected %q got %q", folder, path, index, block.Hash, hash)
|
|
||||||
err = f.model.finder.Fix(folder, path, index, block.Hash, hash)
|
|
||||||
if err != nil {
|
|
||||||
l.Warnln("finder fix:", err)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
l.Debugln("Finder failed to verify buffer", err)
|
|
||||||
}
|
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1324,22 +1311,22 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func verifyBuffer(buf []byte, block protocol.BlockInfo) ([]byte, error) {
|
func verifyBuffer(buf []byte, block protocol.BlockInfo) error {
|
||||||
if len(buf) != int(block.Size) {
|
if len(buf) != int(block.Size) {
|
||||||
return nil, fmt.Errorf("length mismatch %d != %d", len(buf), block.Size)
|
return fmt.Errorf("length mismatch %d != %d", len(buf), block.Size)
|
||||||
}
|
}
|
||||||
hf := sha256.New()
|
hf := sha256.New()
|
||||||
_, err := hf.Write(buf)
|
_, err := hf.Write(buf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return err
|
||||||
}
|
}
|
||||||
hash := hf.Sum(nil)
|
hash := hf.Sum(nil)
|
||||||
|
|
||||||
if !bytes.Equal(hash, block.Hash) {
|
if !bytes.Equal(hash, block.Hash) {
|
||||||
return hash, fmt.Errorf("hash mismatch %x != %x", hash, block.Hash)
|
return fmt.Errorf("hash mismatch %x != %x", hash, block.Hash)
|
||||||
}
|
}
|
||||||
|
|
||||||
return hash, nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *sendReceiveFolder) pullerRoutine(in <-chan pullBlockState, out chan<- *sharedPullerState) {
|
func (f *sendReceiveFolder) pullerRoutine(in <-chan pullBlockState, out chan<- *sharedPullerState) {
|
||||||
|
@ -1411,7 +1398,7 @@ func (f *sendReceiveFolder) pullBlock(state pullBlockState, out chan<- *sharedPu
|
||||||
// Fetch the block, while marking the selected device as in use so that
|
// Fetch the block, while marking the selected device as in use so that
|
||||||
// leastBusy can select another device when someone else asks.
|
// leastBusy can select another device when someone else asks.
|
||||||
activity.using(selected)
|
activity.using(selected)
|
||||||
buf, lastError := f.model.requestGlobal(selected.ID, f.folderID, state.file.Name, state.block.Offset, int(state.block.Size), state.block.Hash, selected.FromTemporary)
|
buf, lastError := f.model.requestGlobal(selected.ID, f.folderID, state.file.Name, state.block.Offset, int(state.block.Size), state.block.Hash, state.block.WeakHash, selected.FromTemporary)
|
||||||
activity.done(selected)
|
activity.done(selected)
|
||||||
if lastError != nil {
|
if lastError != nil {
|
||||||
l.Debugln("request:", f.folderID, state.file.Name, state.block.Offset, state.block.Size, "returned error:", lastError)
|
l.Debugln("request:", f.folderID, state.file.Name, state.block.Offset, state.block.Size, "returned error:", lastError)
|
||||||
|
@ -1420,7 +1407,7 @@ func (f *sendReceiveFolder) pullBlock(state pullBlockState, out chan<- *sharedPu
|
||||||
|
|
||||||
// Verify that the received block matches the desired hash, if not
|
// Verify that the received block matches the desired hash, if not
|
||||||
// try pulling it from another device.
|
// try pulling it from another device.
|
||||||
_, lastError = verifyBuffer(buf, state.block)
|
lastError = verifyBuffer(buf, state.block)
|
||||||
if lastError != nil {
|
if lastError != nil {
|
||||||
l.Debugln("request:", f.folderID, state.file.Name, state.block.Offset, state.block.Size, "hash mismatch")
|
l.Debugln("request:", f.folderID, state.file.Name, state.block.Offset, state.block.Size, "hash mismatch")
|
||||||
continue
|
continue
|
||||||
|
|
|
@ -434,52 +434,6 @@ func TestCopierCleanup(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Make sure that the copier routine hashes the content when asked, and pulls
|
|
||||||
// if it fails to find the block.
|
|
||||||
func TestLastResortPulling(t *testing.T) {
|
|
||||||
// Add a file to index (with the incorrect block representation, as content
|
|
||||||
// doesn't actually match the block list)
|
|
||||||
file := setUpFile("empty", []int{0})
|
|
||||||
m := setUpModel(file)
|
|
||||||
|
|
||||||
// Pretend that we are handling a new file of the same content but
|
|
||||||
// with a different name (causing to copy that particular block)
|
|
||||||
file.Name = "newfile"
|
|
||||||
|
|
||||||
iterFn := func(folder, file string, index int32) bool {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
f := setUpSendReceiveFolder(m)
|
|
||||||
|
|
||||||
copyChan := make(chan copyBlocksState)
|
|
||||||
pullChan := make(chan pullBlockState, 1)
|
|
||||||
finisherChan := make(chan *sharedPullerState, 1)
|
|
||||||
dbUpdateChan := make(chan dbUpdateJob, 1)
|
|
||||||
|
|
||||||
// Run a single copier routine
|
|
||||||
go f.copierRoutine(copyChan, pullChan, finisherChan)
|
|
||||||
|
|
||||||
f.handleFile(file, copyChan, finisherChan, dbUpdateChan)
|
|
||||||
|
|
||||||
// Copier should hash empty file, realise that the region it has read
|
|
||||||
// doesn't match the hash which was advertised by the block map, fix it
|
|
||||||
// and ask to pull the block.
|
|
||||||
<-pullChan
|
|
||||||
|
|
||||||
// Verify that it did fix the incorrect hash.
|
|
||||||
if m.finder.Iterate(folders, blocks[0].Hash, iterFn) {
|
|
||||||
t.Error("Found unexpected block")
|
|
||||||
}
|
|
||||||
|
|
||||||
if !m.finder.Iterate(folders, scanner.SHA256OfNothing, iterFn) {
|
|
||||||
t.Error("Expected block not found")
|
|
||||||
}
|
|
||||||
|
|
||||||
(<-finisherChan).fd.Close()
|
|
||||||
os.Remove(filepath.Join("testdata", fs.TempName("newfile")))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDeregisterOnFailInCopy(t *testing.T) {
|
func TestDeregisterOnFailInCopy(t *testing.T) {
|
||||||
file := setUpFile("filex", []int{0, 2, 0, 0, 5, 0, 0, 8})
|
file := setUpFile("filex", []int{0, 2, 0, 0, 5, 0, 0, 8})
|
||||||
defer os.Remove("testdata/" + fs.TempName("filex"))
|
defer os.Remove("testdata/" + fs.TempName("filex"))
|
||||||
|
|
|
@ -80,9 +80,9 @@ func benchmarkRequestsConnPair(b *testing.B, conn0, conn1 net.Conn) {
|
||||||
// Use c0 and c1 for each alternating request, so we get as much
|
// Use c0 and c1 for each alternating request, so we get as much
|
||||||
// data flowing in both directions.
|
// data flowing in both directions.
|
||||||
if i%2 == 0 {
|
if i%2 == 0 {
|
||||||
buf, err = c0.Request("folder", "file", int64(i), 128<<10, nil, false)
|
buf, err = c0.Request("folder", "file", int64(i), 128<<10, nil, 0, false)
|
||||||
} else {
|
} else {
|
||||||
buf, err = c1.Request("folder", "file", int64(i), 128<<10, nil, false)
|
buf, err = c1.Request("folder", "file", int64(i), 128<<10, nil, 0, false)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -171,7 +171,7 @@ func (m *fakeModel) Index(deviceID DeviceID, folder string, files []FileInfo) {
|
||||||
func (m *fakeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) {
|
func (m *fakeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *fakeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
|
func (m *fakeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, weakHAsh uint32, fromTemporary bool, buf []byte) error {
|
||||||
// We write the offset to the end of the buffer, so the receiver
|
// We write the offset to the end of the buffer, so the receiver
|
||||||
// can verify that it did in fact get some data back over the
|
// can verify that it did in fact get some data back over the
|
||||||
// connection.
|
// connection.
|
||||||
|
|
|
@ -353,6 +353,7 @@ type Request struct {
|
||||||
Size int32 `protobuf:"varint,5,opt,name=size,proto3" json:"size,omitempty"`
|
Size int32 `protobuf:"varint,5,opt,name=size,proto3" json:"size,omitempty"`
|
||||||
Hash []byte `protobuf:"bytes,6,opt,name=hash,proto3" json:"hash,omitempty"`
|
Hash []byte `protobuf:"bytes,6,opt,name=hash,proto3" json:"hash,omitempty"`
|
||||||
FromTemporary bool `protobuf:"varint,7,opt,name=from_temporary,json=fromTemporary,proto3" json:"from_temporary,omitempty"`
|
FromTemporary bool `protobuf:"varint,7,opt,name=from_temporary,json=fromTemporary,proto3" json:"from_temporary,omitempty"`
|
||||||
|
WeakHash uint32 `protobuf:"varint,8,opt,name=weak_hash,json=weakHash,proto3" json:"weak_hash,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *Request) Reset() { *m = Request{} }
|
func (m *Request) Reset() { *m = Request{} }
|
||||||
|
@ -1062,6 +1063,11 @@ func (m *Request) MarshalTo(dAtA []byte) (int, error) {
|
||||||
}
|
}
|
||||||
i++
|
i++
|
||||||
}
|
}
|
||||||
|
if m.WeakHash != 0 {
|
||||||
|
dAtA[i] = 0x40
|
||||||
|
i++
|
||||||
|
i = encodeVarintBep(dAtA, i, uint64(m.WeakHash))
|
||||||
|
}
|
||||||
return i, nil
|
return i, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1519,6 +1525,9 @@ func (m *Request) ProtoSize() (n int) {
|
||||||
if m.FromTemporary {
|
if m.FromTemporary {
|
||||||
n += 2
|
n += 2
|
||||||
}
|
}
|
||||||
|
if m.WeakHash != 0 {
|
||||||
|
n += 1 + sovBep(uint64(m.WeakHash))
|
||||||
|
}
|
||||||
return n
|
return n
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -3515,6 +3524,25 @@ func (m *Request) Unmarshal(dAtA []byte) error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
m.FromTemporary = bool(v != 0)
|
m.FromTemporary = bool(v != 0)
|
||||||
|
case 8:
|
||||||
|
if wireType != 0 {
|
||||||
|
return fmt.Errorf("proto: wrong wireType = %d for field WeakHash", wireType)
|
||||||
|
}
|
||||||
|
m.WeakHash = 0
|
||||||
|
for shift := uint(0); ; shift += 7 {
|
||||||
|
if shift >= 64 {
|
||||||
|
return ErrIntOverflowBep
|
||||||
|
}
|
||||||
|
if iNdEx >= l {
|
||||||
|
return io.ErrUnexpectedEOF
|
||||||
|
}
|
||||||
|
b := dAtA[iNdEx]
|
||||||
|
iNdEx++
|
||||||
|
m.WeakHash |= (uint32(b) & 0x7F) << shift
|
||||||
|
if b < 0x80 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
default:
|
default:
|
||||||
iNdEx = preIndex
|
iNdEx = preIndex
|
||||||
skippy, err := skipBep(dAtA[iNdEx:])
|
skippy, err := skipBep(dAtA[iNdEx:])
|
||||||
|
@ -4192,115 +4220,116 @@ var (
|
||||||
func init() { proto.RegisterFile("bep.proto", fileDescriptorBep) }
|
func init() { proto.RegisterFile("bep.proto", fileDescriptorBep) }
|
||||||
|
|
||||||
var fileDescriptorBep = []byte{
|
var fileDescriptorBep = []byte{
|
||||||
// 1757 bytes of a gzipped FileDescriptorProto
|
// 1762 bytes of a gzipped FileDescriptorProto
|
||||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4f, 0x8f, 0xdb, 0xc6,
|
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4f, 0x8f, 0xdb, 0xc6,
|
||||||
0x15, 0x17, 0x25, 0xea, 0xdf, 0x93, 0xb4, 0xe1, 0x8e, 0xed, 0x2d, 0xcb, 0x6c, 0x24, 0x5a, 0xb1,
|
0x15, 0x5f, 0x4a, 0x94, 0x44, 0x3d, 0x69, 0x37, 0xdc, 0xb1, 0xbd, 0x65, 0x99, 0x8d, 0x44, 0x2b,
|
||||||
0xe3, 0xcd, 0x22, 0x59, 0xbb, 0x49, 0xda, 0xa2, 0x45, 0x5b, 0x40, 0x7f, 0xb8, 0x6b, 0xa1, 0x32,
|
0x76, 0xbc, 0x59, 0x24, 0x6b, 0x37, 0x49, 0x5b, 0xb4, 0x68, 0x0b, 0xe8, 0x0f, 0x77, 0x2d, 0x54,
|
||||||
0xa5, 0x8e, 0xb4, 0x4e, 0x9d, 0x43, 0x09, 0x4a, 0x1c, 0x69, 0x09, 0x53, 0x1c, 0x95, 0xa4, 0xd6,
|
0xa6, 0xd4, 0x91, 0xd6, 0xa9, 0x73, 0x28, 0x41, 0x89, 0x23, 0x2d, 0x61, 0x8a, 0xa3, 0x92, 0xd4,
|
||||||
0x56, 0x3e, 0x82, 0x3e, 0x41, 0x2f, 0x02, 0x02, 0xf4, 0x54, 0xa0, 0xc7, 0x7e, 0x08, 0x1f, 0x83,
|
0xda, 0xca, 0x47, 0xd0, 0x27, 0xe8, 0x45, 0x40, 0x80, 0x9e, 0x0a, 0xf4, 0x83, 0xf8, 0x98, 0xf6,
|
||||||
0x1e, 0x7a, 0xe8, 0xc1, 0x68, 0xb6, 0x97, 0x1e, 0xfb, 0x09, 0x8a, 0x82, 0x33, 0xa4, 0x44, 0xed,
|
0xd0, 0x43, 0x0f, 0x46, 0xb3, 0xbd, 0xf4, 0xd8, 0x4f, 0x50, 0x14, 0x9c, 0x21, 0x25, 0x6a, 0xd7,
|
||||||
0xda, 0x81, 0x0f, 0x39, 0x71, 0xe6, 0xbd, 0xdf, 0xbc, 0x99, 0xf9, 0xcd, 0xef, 0xbd, 0x47, 0x28,
|
0x0e, 0x7c, 0xc8, 0x89, 0x33, 0xef, 0xfd, 0xe6, 0x0d, 0xdf, 0x6f, 0x7e, 0xef, 0xcd, 0x40, 0x71,
|
||||||
0x8e, 0xc8, 0xfc, 0x64, 0xee, 0xd1, 0x80, 0xa2, 0x02, 0xfb, 0x8c, 0xa9, 0xa3, 0x7c, 0x3a, 0xb5,
|
0x48, 0x66, 0x27, 0x33, 0x9f, 0x86, 0x14, 0x49, 0xec, 0x33, 0xa2, 0xae, 0xfa, 0xe9, 0xc4, 0x09,
|
||||||
0x83, 0x8b, 0xc5, 0xe8, 0x64, 0x4c, 0x67, 0x0f, 0xa7, 0x74, 0x4a, 0x1f, 0x32, 0xcf, 0x68, 0x31,
|
0x2f, 0xe6, 0xc3, 0x93, 0x11, 0x9d, 0x3e, 0x9c, 0xd0, 0x09, 0x7d, 0xc8, 0x3c, 0xc3, 0xf9, 0x98,
|
||||||
0x61, 0x33, 0x36, 0x61, 0x23, 0xbe, 0xb0, 0x3e, 0x87, 0xec, 0x63, 0xe2, 0x38, 0x14, 0xd5, 0xa0,
|
0xcd, 0xd8, 0x84, 0x8d, 0xf8, 0xc2, 0xda, 0x0c, 0x72, 0x8f, 0x89, 0xeb, 0x52, 0x54, 0x85, 0x92,
|
||||||
0x64, 0x91, 0x4b, 0x7b, 0x4c, 0x0c, 0xd7, 0x9c, 0x11, 0x59, 0x50, 0x85, 0xa3, 0x22, 0x06, 0x6e,
|
0x4d, 0x2e, 0x9d, 0x11, 0x31, 0x3d, 0x6b, 0x4a, 0x14, 0x41, 0x13, 0x8e, 0x8a, 0x18, 0xb8, 0xc9,
|
||||||
0xd2, 0xcd, 0x19, 0x09, 0x01, 0x63, 0xc7, 0x26, 0x6e, 0xc0, 0x01, 0x69, 0x0e, 0xe0, 0x26, 0x06,
|
0xb0, 0xa6, 0x24, 0x02, 0x8c, 0x5c, 0x87, 0x78, 0x21, 0x07, 0x64, 0x38, 0x80, 0x9b, 0x18, 0xe0,
|
||||||
0xb8, 0x0f, 0x7b, 0x11, 0xe0, 0x92, 0x78, 0xbe, 0x4d, 0x5d, 0x39, 0xc3, 0x30, 0x15, 0x6e, 0x7d,
|
0x3e, 0xec, 0xc5, 0x80, 0x4b, 0xe2, 0x07, 0x0e, 0xf5, 0x94, 0x2c, 0xc3, 0xec, 0x72, 0xeb, 0x53,
|
||||||
0xca, 0x8d, 0x75, 0x1f, 0x72, 0x8f, 0x89, 0x69, 0x11, 0x0f, 0x7d, 0x0c, 0x62, 0xb0, 0x9c, 0xf3,
|
0x6e, 0xac, 0x05, 0x90, 0x7f, 0x4c, 0x2c, 0x9b, 0xf8, 0xe8, 0x63, 0x10, 0xc3, 0xc5, 0x8c, 0xef,
|
||||||
0xbd, 0xf6, 0x3e, 0xbb, 0x73, 0x12, 0xdf, 0xe1, 0xe4, 0x09, 0xf1, 0x7d, 0x73, 0x4a, 0x86, 0xcb,
|
0xb5, 0xf7, 0xd9, 0x9d, 0x93, 0x24, 0x87, 0x93, 0x27, 0x24, 0x08, 0xac, 0x09, 0x19, 0x2c, 0x66,
|
||||||
0x39, 0xc1, 0x0c, 0x82, 0x7e, 0x03, 0xa5, 0x31, 0x9d, 0xcd, 0x3d, 0xe2, 0xb3, 0xc0, 0x69, 0xb6,
|
0x04, 0x33, 0x08, 0xfa, 0x0d, 0x94, 0x46, 0x74, 0x3a, 0xf3, 0x49, 0xc0, 0x02, 0x67, 0xd8, 0x8a,
|
||||||
0xe2, 0xf0, 0xc6, 0x8a, 0xd6, 0x16, 0x83, 0x93, 0x0b, 0xea, 0x0d, 0xa8, 0xb4, 0x9c, 0x85, 0x1f,
|
0xc3, 0x1b, 0x2b, 0x9a, 0x1b, 0x0c, 0x4e, 0x2f, 0xa8, 0xd5, 0x61, 0xb7, 0xe9, 0xce, 0x83, 0x90,
|
||||||
0x10, 0xaf, 0x45, 0xdd, 0x89, 0x3d, 0x45, 0x8f, 0x20, 0x3f, 0xa1, 0x8e, 0x45, 0x3c, 0x5f, 0x16,
|
0xf8, 0x4d, 0xea, 0x8d, 0x9d, 0x09, 0x7a, 0x04, 0x85, 0x31, 0x75, 0x6d, 0xe2, 0x07, 0x8a, 0xa0,
|
||||||
0xd4, 0xcc, 0x51, 0xe9, 0x33, 0x69, 0x1b, 0xec, 0x94, 0x39, 0x9a, 0xe2, 0xab, 0xd7, 0xb5, 0x14,
|
0x65, 0x8f, 0x4a, 0x9f, 0xc9, 0x9b, 0x60, 0xa7, 0xcc, 0xd1, 0x10, 0x5f, 0xbd, 0xae, 0xee, 0xe0,
|
||||||
0x8e, 0x61, 0xf5, 0x3f, 0xa7, 0x21, 0xc7, 0x3d, 0xe8, 0x00, 0xd2, 0xb6, 0xc5, 0x29, 0x6a, 0xe6,
|
0x04, 0x56, 0xfb, 0x73, 0x06, 0xf2, 0xdc, 0x83, 0x0e, 0x20, 0xe3, 0xd8, 0x9c, 0xa2, 0x46, 0xfe,
|
||||||
0xae, 0x5e, 0xd7, 0xd2, 0x9d, 0x36, 0x4e, 0xdb, 0x16, 0xba, 0x0d, 0x59, 0xc7, 0x1c, 0x11, 0x27,
|
0xea, 0x75, 0x35, 0xd3, 0x6e, 0xe1, 0x8c, 0x63, 0xa3, 0xdb, 0x90, 0x73, 0xad, 0x21, 0x71, 0x63,
|
||||||
0x22, 0x87, 0x4f, 0xd0, 0xfb, 0x50, 0xf4, 0x88, 0x69, 0x19, 0xd4, 0x75, 0x96, 0x8c, 0x92, 0x02,
|
0x72, 0xf8, 0x04, 0xbd, 0x0f, 0x45, 0x9f, 0x58, 0xb6, 0x49, 0x3d, 0x77, 0xc1, 0x28, 0x91, 0xb0,
|
||||||
0x2e, 0x84, 0x86, 0x9e, 0xeb, 0x2c, 0xd1, 0xa7, 0x80, 0xec, 0xa9, 0x4b, 0x3d, 0x62, 0xcc, 0x89,
|
0x14, 0x19, 0xba, 0x9e, 0xbb, 0x40, 0x9f, 0x02, 0x72, 0x26, 0x1e, 0xf5, 0x89, 0x39, 0x23, 0xfe,
|
||||||
0x37, 0xb3, 0xd9, 0x69, 0x7d, 0x59, 0x64, 0xa8, 0x7d, 0xee, 0xe9, 0x6f, 0x1d, 0xe8, 0x43, 0xa8,
|
0xd4, 0x61, 0x7f, 0x1b, 0x28, 0x22, 0x43, 0xed, 0x73, 0x4f, 0x6f, 0xe3, 0x40, 0x1f, 0xc2, 0x6e,
|
||||||
0x44, 0x70, 0x8b, 0x38, 0x24, 0x20, 0x72, 0x96, 0x21, 0xcb, 0xdc, 0xd8, 0x66, 0x36, 0xf4, 0x08,
|
0x0c, 0xb7, 0x89, 0x4b, 0x42, 0xa2, 0xe4, 0x18, 0xb2, 0xcc, 0x8d, 0x2d, 0x66, 0x43, 0x8f, 0xe0,
|
||||||
0x6e, 0x5b, 0xb6, 0x6f, 0x8e, 0x1c, 0x62, 0x04, 0x64, 0x36, 0x37, 0x6c, 0xd7, 0x22, 0x2f, 0x89,
|
0xb6, 0xed, 0x04, 0xd6, 0xd0, 0x25, 0x66, 0x48, 0xa6, 0x33, 0xd3, 0xf1, 0x6c, 0xf2, 0x92, 0x04,
|
||||||
0x2f, 0xe7, 0x18, 0x16, 0x45, 0xbe, 0x21, 0x99, 0xcd, 0x3b, 0xdc, 0x83, 0x0e, 0x20, 0x37, 0x37,
|
0x4a, 0x9e, 0x61, 0x51, 0xec, 0x1b, 0x90, 0xe9, 0xac, 0xcd, 0x3d, 0xe8, 0x00, 0xf2, 0x33, 0x6b,
|
||||||
0x17, 0x3e, 0xb1, 0xe4, 0x3c, 0xc3, 0x44, 0xb3, 0x90, 0x25, 0xae, 0x00, 0x5f, 0x96, 0xae, 0xb3,
|
0x1e, 0x10, 0x5b, 0x29, 0x30, 0x4c, 0x3c, 0x8b, 0x58, 0xe2, 0x0a, 0x08, 0x14, 0xf9, 0x3a, 0x4b,
|
||||||
0xd4, 0x66, 0x8e, 0x98, 0xa5, 0x08, 0x56, 0xff, 0x6f, 0x1a, 0x72, 0xdc, 0x83, 0x3e, 0xda, 0xb0,
|
0x2d, 0xe6, 0x48, 0x58, 0x8a, 0x61, 0xb5, 0xff, 0x66, 0x20, 0xcf, 0x3d, 0xe8, 0xa3, 0x35, 0x4b,
|
||||||
0x54, 0x6e, 0x1e, 0x84, 0xa8, 0x7f, 0xbe, 0xae, 0x15, 0xb8, 0xaf, 0xd3, 0x4e, 0xb0, 0x86, 0x40,
|
0xe5, 0xc6, 0x41, 0x84, 0xfa, 0xe7, 0xeb, 0xaa, 0xc4, 0x7d, 0xed, 0x56, 0x8a, 0x35, 0x04, 0x62,
|
||||||
0x4c, 0x28, 0x8a, 0x8d, 0xd1, 0x21, 0x14, 0x4d, 0xcb, 0x0a, 0x5f, 0x8f, 0xf8, 0x72, 0x46, 0xcd,
|
0x4a, 0x51, 0x6c, 0x8c, 0x0e, 0xa1, 0x68, 0xd9, 0x76, 0x74, 0x7a, 0x24, 0x50, 0xb2, 0x5a, 0xf6,
|
||||||
0x1c, 0x15, 0xf1, 0xd6, 0x80, 0x7e, 0xbe, 0xab, 0x06, 0xf1, 0xba, 0x7e, 0xde, 0x26, 0x83, 0xf0,
|
0xa8, 0x88, 0x37, 0x06, 0xf4, 0xf3, 0x6d, 0x35, 0x88, 0xd7, 0xf5, 0xf3, 0x36, 0x19, 0x44, 0x47,
|
||||||
0x29, 0xc6, 0xc4, 0x8b, 0x14, 0x9c, 0x65, 0xfb, 0x15, 0x42, 0x03, 0xd3, 0xef, 0x5d, 0x28, 0xcf,
|
0x31, 0x22, 0x7e, 0xac, 0xe0, 0x1c, 0xdb, 0x4f, 0x8a, 0x0c, 0x4c, 0xbf, 0x77, 0xa1, 0x3c, 0xb5,
|
||||||
0xcc, 0x97, 0x86, 0x4f, 0xfe, 0xb8, 0x20, 0xee, 0x98, 0x30, 0xba, 0x32, 0xb8, 0x34, 0x33, 0x5f,
|
0x5e, 0x9a, 0x01, 0xf9, 0xe3, 0x9c, 0x78, 0x23, 0xc2, 0xe8, 0xca, 0xe2, 0xd2, 0xd4, 0x7a, 0xd9,
|
||||||
0x0e, 0x22, 0x13, 0xaa, 0x02, 0xd8, 0x6e, 0xe0, 0x51, 0x6b, 0x31, 0x26, 0x5e, 0xc4, 0x55, 0xc2,
|
0x8f, 0x4d, 0xa8, 0x02, 0xe0, 0x78, 0xa1, 0x4f, 0xed, 0xf9, 0x88, 0xf8, 0x31, 0x57, 0x29, 0x0b,
|
||||||
0x82, 0x7e, 0x0a, 0x05, 0x46, 0xb6, 0x61, 0x5b, 0x72, 0x41, 0x15, 0x8e, 0xc4, 0xa6, 0x12, 0x5d,
|
0xfa, 0x29, 0x48, 0x8c, 0x6c, 0xd3, 0xb1, 0x15, 0x49, 0x13, 0x8e, 0xc4, 0x86, 0x1a, 0x27, 0x5e,
|
||||||
0x3c, 0xcf, 0xa8, 0x66, 0xf7, 0x8e, 0x87, 0x38, 0xcf, 0xb0, 0x1d, 0x0b, 0xfd, 0x0a, 0x14, 0xff,
|
0x60, 0x54, 0xb3, 0xbc, 0x93, 0x21, 0x2e, 0x30, 0x6c, 0xdb, 0x46, 0xbf, 0x02, 0x35, 0x78, 0xee,
|
||||||
0xb9, 0x1d, 0x3e, 0x14, 0x8f, 0x14, 0xd8, 0xd4, 0x35, 0x3c, 0x32, 0xa3, 0x97, 0xa6, 0xe3, 0xcb,
|
0x44, 0x07, 0xc5, 0x23, 0x85, 0x0e, 0xf5, 0x4c, 0x9f, 0x4c, 0xe9, 0xa5, 0xe5, 0x06, 0x4a, 0x91,
|
||||||
0x45, 0xb6, 0x8d, 0x1c, 0x22, 0x3a, 0x09, 0x00, 0x8e, 0xfc, 0xf5, 0x1e, 0x64, 0x59, 0xc4, 0xf0,
|
0x6d, 0xa3, 0x44, 0x88, 0x76, 0x0a, 0x80, 0x63, 0x7f, 0xad, 0x0b, 0x39, 0x16, 0x31, 0x3a, 0x45,
|
||||||
0x15, 0xb9, 0x58, 0xa3, 0xec, 0x8d, 0x66, 0xe8, 0x04, 0xb2, 0x13, 0xdb, 0x21, 0xbe, 0x9c, 0x66,
|
0x2e, 0xd6, 0xb8, 0x7a, 0xe3, 0x19, 0x3a, 0x81, 0xdc, 0xd8, 0x71, 0x49, 0xa0, 0x64, 0xd8, 0x19,
|
||||||
0x6f, 0x88, 0x12, 0x4a, 0xb7, 0x1d, 0xd2, 0x71, 0x27, 0x34, 0x7a, 0x45, 0x0e, 0xab, 0x9f, 0x43,
|
0xa2, 0x94, 0xd2, 0x1d, 0x97, 0xb4, 0xbd, 0x31, 0x8d, 0x4f, 0x91, 0xc3, 0x6a, 0xe7, 0x50, 0x62,
|
||||||
0x89, 0x05, 0x3c, 0x9f, 0x5b, 0x66, 0x40, 0x7e, 0xb0, 0xb0, 0x7f, 0x15, 0xa1, 0x10, 0x7b, 0x36,
|
0x01, 0xcf, 0x67, 0xb6, 0x15, 0x92, 0x1f, 0x2c, 0xec, 0x5f, 0x45, 0x90, 0x12, 0xcf, 0xfa, 0xd0,
|
||||||
0x8f, 0x2e, 0x24, 0x1e, 0xfd, 0x38, 0xaa, 0x07, 0x3c, 0xbb, 0x0f, 0x6e, 0xc6, 0x4b, 0x14, 0x04,
|
0x85, 0xd4, 0xa1, 0x1f, 0xc7, 0xfd, 0x80, 0x57, 0xf7, 0xc1, 0xcd, 0x78, 0xa9, 0x86, 0x80, 0x40,
|
||||||
0x04, 0xa2, 0x6f, 0x7f, 0x4d, 0x58, 0x3e, 0x65, 0x30, 0x1b, 0x23, 0x15, 0x4a, 0xd7, 0x93, 0xa8,
|
0x0c, 0x9c, 0xaf, 0x09, 0xab, 0xa7, 0x2c, 0x66, 0x63, 0xa4, 0x41, 0xe9, 0x7a, 0x11, 0xed, 0xe2,
|
||||||
0x82, 0x93, 0x26, 0xf4, 0x01, 0xc0, 0x8c, 0x5a, 0xf6, 0xc4, 0x26, 0x96, 0xe1, 0x33, 0x01, 0x64,
|
0xb4, 0x09, 0x7d, 0x00, 0x30, 0xa5, 0xb6, 0x33, 0x76, 0x88, 0x6d, 0x06, 0x4c, 0x00, 0x59, 0x5c,
|
||||||
0x70, 0x31, 0xb6, 0x0c, 0x90, 0x1c, 0xca, 0x3d, 0x4c, 0x21, 0x2b, 0xca, 0x95, 0x78, 0x1a, 0x7a,
|
0x4c, 0x2c, 0x7d, 0xa4, 0x44, 0x72, 0x8f, 0x4a, 0xc8, 0x8e, 0x6b, 0x25, 0x99, 0x46, 0x1e, 0xc7,
|
||||||
0x6c, 0xf7, 0xd2, 0x74, 0xec, 0x38, 0x43, 0xe2, 0x69, 0x58, 0xf5, 0x5c, 0xba, 0x93, 0xbc, 0x05,
|
0xbb, 0xb4, 0x5c, 0x27, 0xa9, 0x90, 0x64, 0x1a, 0x75, 0x3d, 0x8f, 0x6e, 0x15, 0xaf, 0xc4, 0x00,
|
||||||
0x06, 0xa8, 0xb8, 0x34, 0x99, 0xb8, 0x8f, 0x20, 0x1f, 0x57, 0xc5, 0xf0, 0x3d, 0x77, 0x32, 0xe9,
|
0xbb, 0x1e, 0x4d, 0x17, 0xee, 0x23, 0x28, 0x24, 0x5d, 0x31, 0x3a, 0xcf, 0xad, 0x4a, 0x7a, 0x4a,
|
||||||
0x29, 0x19, 0x07, 0x74, 0x53, 0x6f, 0x22, 0x18, 0x52, 0xa0, 0xb0, 0x91, 0x22, 0xb0, 0x93, 0x6e,
|
0x46, 0x21, 0x5d, 0xf7, 0x9b, 0x18, 0x86, 0x54, 0x90, 0xd6, 0x52, 0x04, 0xf6, 0xa7, 0xeb, 0x79,
|
||||||
0xe6, 0x61, 0x2d, 0xde, 0xdc, 0xc3, 0xf5, 0xe5, 0x92, 0x2a, 0x1c, 0x65, 0xf1, 0xe6, 0x6a, 0x7a,
|
0xd4, 0x8b, 0xd7, 0x79, 0x78, 0x81, 0x52, 0xd2, 0x84, 0xa3, 0x1c, 0x5e, 0xa7, 0x66, 0x44, 0xdb,
|
||||||
0xb8, 0xdd, 0x16, 0x30, 0x5a, 0xca, 0x65, 0xa6, 0xc5, 0xf7, 0x62, 0x2d, 0x0e, 0x2e, 0xa8, 0x17,
|
0x6d, 0x00, 0xc3, 0x85, 0x52, 0x66, 0x5a, 0x7c, 0x2f, 0xd1, 0x62, 0xff, 0x82, 0xfa, 0x61, 0xbb,
|
||||||
0x74, 0xda, 0xdb, 0x15, 0xcd, 0x25, 0x7a, 0x08, 0x30, 0x72, 0xe8, 0xf8, 0xb9, 0xc1, 0x68, 0xad,
|
0xb5, 0x59, 0xd1, 0x58, 0xa0, 0x87, 0x00, 0x43, 0x97, 0x8e, 0x9e, 0x9b, 0x8c, 0xd6, 0xdd, 0x28,
|
||||||
0x84, 0x11, 0x9b, 0xd2, 0xd5, 0xeb, 0x5a, 0x19, 0x9b, 0x2f, 0x9a, 0xa1, 0x63, 0x60, 0x7f, 0x4d,
|
0x62, 0x43, 0xbe, 0x7a, 0x5d, 0x2d, 0x63, 0xeb, 0x45, 0x23, 0x72, 0xf4, 0x9d, 0xaf, 0x09, 0x2e,
|
||||||
0x70, 0x71, 0x14, 0x0f, 0xd1, 0x4f, 0x20, 0xc7, 0xec, 0x71, 0x69, 0xb8, 0xb5, 0xbd, 0x10, 0xb3,
|
0x0e, 0x93, 0x21, 0xfa, 0x09, 0xe4, 0x99, 0x3d, 0x69, 0x0d, 0xb7, 0x36, 0x09, 0x31, 0x7b, 0x4a,
|
||||||
0x27, 0x04, 0x10, 0x01, 0x43, 0xae, 0xfc, 0xe5, 0xcc, 0xb1, 0xdd, 0xe7, 0x46, 0x60, 0x7a, 0x53,
|
0x00, 0x31, 0x30, 0xe2, 0x2a, 0x58, 0x4c, 0x5d, 0xc7, 0x7b, 0x6e, 0x86, 0x96, 0x3f, 0x21, 0xa1,
|
||||||
0x12, 0xc8, 0xfb, 0xbc, 0x43, 0x44, 0xd6, 0x21, 0x33, 0xfe, 0x52, 0xfc, 0xd3, 0x37, 0xb5, 0x54,
|
0xb2, 0xcf, 0x6f, 0x88, 0xd8, 0x3a, 0x60, 0xc6, 0x5f, 0x8a, 0x7f, 0xfa, 0xa6, 0xba, 0x53, 0xf3,
|
||||||
0xdd, 0x85, 0xe2, 0x26, 0x4e, 0xa8, 0x41, 0x3a, 0x99, 0xf8, 0x24, 0x60, 0x82, 0xc9, 0xe0, 0x68,
|
0xa0, 0xb8, 0x8e, 0x13, 0x69, 0x90, 0x8e, 0xc7, 0x01, 0x09, 0x99, 0x60, 0xb2, 0x38, 0x9e, 0xad,
|
||||||
0xb6, 0x91, 0x41, 0x9a, 0x31, 0xc0, 0x65, 0x80, 0x40, 0xbc, 0x30, 0xfd, 0x0b, 0x26, 0x8d, 0x32,
|
0x65, 0x90, 0x61, 0x0c, 0x70, 0x19, 0x20, 0x10, 0x2f, 0xac, 0xe0, 0x82, 0x49, 0xa3, 0x8c, 0xd9,
|
||||||
0x66, 0xe3, 0x30, 0xf1, 0x5f, 0x10, 0xf3, 0xb9, 0xc1, 0x1c, 0x5c, 0x18, 0x85, 0xd0, 0xf0, 0xd8,
|
0x38, 0x2a, 0xfc, 0x17, 0xc4, 0x7a, 0x6e, 0x32, 0x07, 0x17, 0x86, 0x14, 0x19, 0x1e, 0x5b, 0xc1,
|
||||||
0xf4, 0x2f, 0xa2, 0xfd, 0x7e, 0x0d, 0x39, 0xfe, 0x10, 0xe8, 0x73, 0x28, 0x8c, 0xe9, 0xc2, 0x0d,
|
0x45, 0xbc, 0xdf, 0xaf, 0x21, 0xcf, 0x0f, 0x02, 0x7d, 0x0e, 0xd2, 0x88, 0xce, 0xbd, 0x70, 0x73,
|
||||||
0xb6, 0xcd, 0x61, 0x3f, 0x59, 0x5b, 0x98, 0x27, 0xba, 0xd9, 0x06, 0x58, 0x3f, 0x85, 0x7c, 0xe4,
|
0x39, 0xec, 0xa7, 0x7b, 0x0b, 0xf3, 0xc4, 0x99, 0xad, 0x81, 0xb5, 0x53, 0x28, 0xc4, 0x2e, 0x74,
|
||||||
0x42, 0xf7, 0x37, 0x85, 0x4f, 0x6c, 0xde, 0xb9, 0xc6, 0xf9, 0x6e, 0xb7, 0xb8, 0x34, 0x9d, 0x05,
|
0x7f, 0xdd, 0xf8, 0xc4, 0xc6, 0x9d, 0x6b, 0x9c, 0x6f, 0xdf, 0x16, 0x97, 0x96, 0x3b, 0xe7, 0x3f,
|
||||||
0x3f, 0xbc, 0x88, 0xf9, 0xa4, 0xfe, 0x37, 0x01, 0xf2, 0x38, 0x7c, 0x67, 0x3f, 0x48, 0xf4, 0x99,
|
0x2f, 0x62, 0x3e, 0xa9, 0xfd, 0x4d, 0x80, 0x02, 0x8e, 0xce, 0x39, 0x08, 0x53, 0xf7, 0x4c, 0x6e,
|
||||||
0xec, 0x4e, 0x9f, 0xd9, 0x66, 0x64, 0x7a, 0x27, 0x23, 0xe3, 0xa4, 0xca, 0x24, 0x92, 0x6a, 0xcb,
|
0xeb, 0x9e, 0xd9, 0x54, 0x64, 0x66, 0xab, 0x22, 0x93, 0xa2, 0xca, 0xa6, 0x8a, 0x6a, 0xc3, 0x9c,
|
||||||
0x9c, 0xf8, 0x46, 0xe6, 0xb2, 0x6f, 0x60, 0x2e, 0x97, 0x60, 0xee, 0x3e, 0xec, 0x4d, 0x3c, 0x3a,
|
0xf8, 0x46, 0xe6, 0x72, 0x6f, 0x60, 0x2e, 0x9f, 0x62, 0xee, 0x3e, 0xec, 0x8d, 0x7d, 0x3a, 0x65,
|
||||||
0x63, 0x9d, 0x84, 0x7a, 0xa6, 0xb7, 0x8c, 0x12, 0xa0, 0x12, 0x5a, 0x87, 0xb1, 0xb1, 0x6e, 0x40,
|
0x37, 0x09, 0xf5, 0x2d, 0x7f, 0x11, 0x17, 0xc0, 0x6e, 0x64, 0x1d, 0x24, 0xc6, 0x6d, 0x82, 0xa5,
|
||||||
0x01, 0x13, 0x7f, 0x4e, 0x5d, 0x9f, 0xbc, 0xf5, 0xd8, 0x08, 0x44, 0xcb, 0x0c, 0x4c, 0x76, 0xe8,
|
0x6d, 0x82, 0x6b, 0x26, 0x48, 0x98, 0x04, 0x33, 0xea, 0x05, 0xe4, 0xad, 0x39, 0x21, 0x10, 0x6d,
|
||||||
0x32, 0x66, 0x63, 0xf4, 0x00, 0xc4, 0x31, 0xb5, 0xf8, 0x91, 0xf7, 0x92, 0x1a, 0xd2, 0x3c, 0x8f,
|
0x2b, 0xb4, 0x58, 0x46, 0x65, 0xcc, 0xc6, 0xe8, 0x01, 0x88, 0x23, 0x6a, 0xf3, 0x7c, 0xf6, 0xd2,
|
||||||
0x7a, 0x2d, 0x6a, 0x11, 0xcc, 0x00, 0xf5, 0x39, 0x48, 0x6d, 0xfa, 0xc2, 0x75, 0xa8, 0x69, 0xf5,
|
0x02, 0xd3, 0x7d, 0x9f, 0xfa, 0x4d, 0x6a, 0x13, 0xcc, 0x00, 0xb5, 0x19, 0xc8, 0x2d, 0xfa, 0xc2,
|
||||||
0x3d, 0x3a, 0x0d, 0x2b, 0xfa, 0x5b, 0x2b, 0x53, 0x1b, 0xf2, 0x0b, 0x56, 0xbb, 0xe2, 0xda, 0x74,
|
0x73, 0xa9, 0x65, 0xf7, 0x7c, 0x3a, 0x89, 0xda, 0xfd, 0x5b, 0xdb, 0x56, 0x0b, 0x0a, 0x73, 0xd6,
|
||||||
0x6f, 0xb7, 0x96, 0x5c, 0x0f, 0xc4, 0x0b, 0x5d, 0x9c, 0x80, 0xd1, 0xd2, 0xfa, 0x3f, 0x04, 0x50,
|
0xd8, 0x92, 0xc6, 0x75, 0x6f, 0xbb, 0xd1, 0x5c, 0x0f, 0xc4, 0xbb, 0x60, 0x52, 0x9d, 0xf1, 0xd2,
|
||||||
0xde, 0x8e, 0x46, 0x1d, 0x28, 0x71, 0xa4, 0x91, 0xf8, 0x89, 0x39, 0x7a, 0x97, 0x8d, 0x58, 0x19,
|
0xda, 0x3f, 0x04, 0x50, 0xdf, 0x8e, 0x46, 0x6d, 0x28, 0x71, 0xa4, 0x99, 0x7a, 0xe1, 0x1c, 0xbd,
|
||||||
0x83, 0xc5, 0x66, 0xfc, 0xc6, 0x0e, 0x98, 0x28, 0x18, 0x99, 0x77, 0x2b, 0x18, 0x0f, 0xa0, 0xc2,
|
0xcb, 0x46, 0xac, 0xc7, 0xc1, 0x7c, 0x3d, 0x7e, 0xe3, 0xf5, 0x98, 0xea, 0x26, 0xd9, 0x77, 0xeb,
|
||||||
0x33, 0x38, 0xee, 0xf7, 0xa2, 0x9a, 0x39, 0xca, 0x36, 0xd3, 0x52, 0x0a, 0x97, 0x47, 0x3c, 0x93,
|
0x26, 0x0f, 0x60, 0x97, 0x97, 0x77, 0xf2, 0x18, 0x10, 0xb5, 0xec, 0x51, 0xae, 0x91, 0x91, 0x77,
|
||||||
0x98, 0xbd, 0x9e, 0x03, 0xb1, 0x6f, 0xbb, 0xd3, 0x7a, 0x0d, 0xb2, 0x2d, 0x87, 0xb2, 0x07, 0xcb,
|
0x70, 0x79, 0xc8, 0xcb, 0x8c, 0xd9, 0x6b, 0x79, 0x10, 0x7b, 0x8e, 0x37, 0xa9, 0x55, 0x21, 0xd7,
|
||||||
0x79, 0xc4, 0xf4, 0xa9, 0x1b, 0xf3, 0xc8, 0x67, 0xc7, 0x7f, 0x4f, 0x43, 0x29, 0xf1, 0x2f, 0x86,
|
0x74, 0x29, 0x3b, 0xb0, 0xbc, 0x4f, 0xac, 0x80, 0x7a, 0x09, 0x8f, 0x7c, 0x76, 0xfc, 0xf7, 0x0c,
|
||||||
0x1e, 0xc1, 0x5e, 0xab, 0x7b, 0x3e, 0x18, 0x6a, 0xd8, 0x68, 0xf5, 0xf4, 0xd3, 0xce, 0x99, 0x94,
|
0x94, 0x52, 0x0f, 0x35, 0xf4, 0x08, 0xf6, 0x9a, 0x9d, 0xf3, 0xfe, 0x40, 0xc7, 0x66, 0xb3, 0x6b,
|
||||||
0x52, 0x0e, 0x57, 0x6b, 0x55, 0x9e, 0x6d, 0x41, 0xbb, 0xbf, 0x59, 0x35, 0xc8, 0x76, 0xf4, 0xb6,
|
0x9c, 0xb6, 0xcf, 0xe4, 0x1d, 0xf5, 0x70, 0xb9, 0xd2, 0x94, 0xe9, 0x06, 0xb4, 0xfd, 0x06, 0xab,
|
||||||
0xf6, 0x7b, 0x49, 0x50, 0x6e, 0xaf, 0xd6, 0xaa, 0x94, 0x00, 0xf2, 0x9e, 0xf5, 0x09, 0x94, 0x19,
|
0x42, 0xae, 0x6d, 0xb4, 0xf4, 0xdf, 0xcb, 0x82, 0x7a, 0x7b, 0xb9, 0xd2, 0xe4, 0x14, 0x90, 0x5f,
|
||||||
0xc0, 0x38, 0xef, 0xb7, 0x1b, 0x43, 0x4d, 0x4a, 0x2b, 0xca, 0x6a, 0xad, 0x1e, 0x5c, 0xc7, 0x45,
|
0x68, 0x9f, 0x40, 0x99, 0x01, 0xcc, 0xf3, 0x5e, 0xab, 0x3e, 0xd0, 0xe5, 0x8c, 0xaa, 0x2e, 0x57,
|
||||||
0x9c, 0x7f, 0x08, 0x79, 0xac, 0xfd, 0xee, 0x5c, 0x1b, 0x0c, 0xa5, 0x8c, 0x72, 0xb0, 0x5a, 0xab,
|
0xda, 0xc1, 0x75, 0x5c, 0xcc, 0xf9, 0x87, 0x50, 0xc0, 0xfa, 0xef, 0xce, 0xf5, 0xfe, 0x40, 0xce,
|
||||||
0x28, 0x01, 0x8c, 0xb3, 0xe6, 0x3e, 0x14, 0xb0, 0x36, 0xe8, 0xf7, 0xf4, 0x81, 0x26, 0x89, 0xca,
|
0xaa, 0x07, 0xcb, 0x95, 0x86, 0x52, 0xc0, 0xa4, 0xa4, 0xee, 0x83, 0x84, 0xf5, 0x7e, 0xaf, 0x6b,
|
||||||
0x8f, 0x56, 0x6b, 0xf5, 0xd6, 0x0e, 0x2a, 0x52, 0xe9, 0xcf, 0x60, 0xbf, 0xdd, 0xfb, 0x52, 0xef,
|
0xf4, 0x75, 0x59, 0x54, 0x7f, 0xb4, 0x5c, 0x69, 0xb7, 0xb6, 0x50, 0xb1, 0x4a, 0x7f, 0x06, 0xfb,
|
||||||
0xf6, 0x1a, 0x6d, 0xa3, 0x8f, 0x7b, 0x67, 0x58, 0x1b, 0x0c, 0xa4, 0xac, 0x52, 0x5b, 0xad, 0xd5,
|
0xad, 0xee, 0x97, 0x46, 0xa7, 0x5b, 0x6f, 0x99, 0x3d, 0xdc, 0x3d, 0xc3, 0x7a, 0xbf, 0x2f, 0xe7,
|
||||||
0xf7, 0x13, 0xf8, 0x1b, 0xa2, 0xfb, 0x00, 0xc4, 0x7e, 0x47, 0x3f, 0x93, 0x72, 0xca, 0xad, 0xd5,
|
0xd4, 0xea, 0x72, 0xa5, 0xbd, 0x9f, 0xc2, 0xdf, 0x10, 0xdd, 0x07, 0x20, 0xf6, 0xda, 0xc6, 0x99,
|
||||||
0x5a, 0x7d, 0x2f, 0x01, 0x0d, 0x49, 0x0d, 0x6f, 0xdc, 0xea, 0xf6, 0x06, 0x9a, 0x94, 0xbf, 0x71,
|
0x9c, 0x57, 0x6f, 0x2d, 0x57, 0xda, 0x7b, 0x29, 0x68, 0x44, 0x6a, 0x94, 0x71, 0xb3, 0xd3, 0xed,
|
||||||
0x63, 0x46, 0xf6, 0xf1, 0x1f, 0x00, 0xdd, 0xfc, 0x5b, 0x45, 0xf7, 0x40, 0xd4, 0x7b, 0xba, 0x26,
|
0xeb, 0x72, 0xe1, 0x46, 0xc6, 0x8c, 0xec, 0xe3, 0x3f, 0x00, 0xba, 0xf9, 0x94, 0x45, 0xf7, 0x40,
|
||||||
0xa5, 0xf8, 0xfd, 0x6f, 0x22, 0x74, 0xea, 0x12, 0x54, 0x87, 0x4c, 0xf7, 0xab, 0x2f, 0x24, 0x41,
|
0x34, 0xba, 0x86, 0x2e, 0xef, 0xf0, 0xfc, 0x6f, 0x22, 0x0c, 0xea, 0x11, 0x54, 0x83, 0x6c, 0xe7,
|
||||||
0xf9, 0xf1, 0x6a, 0xad, 0xde, 0xb9, 0x09, 0xea, 0x7e, 0xf5, 0xc5, 0x31, 0x85, 0x52, 0x32, 0x70,
|
0xab, 0x2f, 0x64, 0x41, 0xfd, 0xf1, 0x72, 0xa5, 0xdd, 0xb9, 0x09, 0xea, 0x7c, 0xf5, 0xc5, 0x31,
|
||||||
0x1d, 0x0a, 0x4f, 0xb4, 0x61, 0xa3, 0xdd, 0x18, 0x36, 0xa4, 0x14, 0x3f, 0x52, 0xec, 0x7e, 0x42,
|
0x85, 0x52, 0x3a, 0x70, 0x0d, 0xa4, 0x27, 0xfa, 0xa0, 0xde, 0xaa, 0x0f, 0xea, 0xf2, 0x0e, 0xff,
|
||||||
0x02, 0x93, 0x25, 0xe1, 0x21, 0x64, 0x75, 0xed, 0xa9, 0x86, 0x25, 0x41, 0xd9, 0x5f, 0xad, 0xd5,
|
0xa5, 0xc4, 0xfd, 0x84, 0x84, 0x16, 0x2b, 0xc2, 0x43, 0xc8, 0x19, 0xfa, 0x53, 0x1d, 0xcb, 0x82,
|
||||||
0x4a, 0x0c, 0xd0, 0xc9, 0x25, 0xf1, 0x50, 0x15, 0x72, 0x8d, 0xee, 0x97, 0x8d, 0x67, 0x03, 0x29,
|
0xba, 0xbf, 0x5c, 0x69, 0xbb, 0x09, 0xc0, 0x20, 0x97, 0xc4, 0x47, 0x15, 0xc8, 0xd7, 0x3b, 0x5f,
|
||||||
0xad, 0xa0, 0xd5, 0x5a, 0xdd, 0x8b, 0xdd, 0x0d, 0xe7, 0x85, 0xb9, 0xf4, 0x8f, 0xff, 0x27, 0x40,
|
0xd6, 0x9f, 0xf5, 0xe5, 0x8c, 0x8a, 0x96, 0x2b, 0x6d, 0x2f, 0x71, 0xd7, 0xdd, 0x17, 0xd6, 0x22,
|
||||||
0x39, 0xd9, 0xa1, 0x51, 0x15, 0xc4, 0xd3, 0x4e, 0x57, 0x8b, 0xb7, 0x4b, 0xfa, 0xc2, 0x31, 0x3a,
|
0x38, 0xfe, 0x9f, 0x00, 0xe5, 0xf4, 0xf5, 0x8d, 0x2a, 0x20, 0x9e, 0xb6, 0x3b, 0x7a, 0xb2, 0x5d,
|
||||||
0x82, 0x62, 0xbb, 0x83, 0xb5, 0xd6, 0xb0, 0x87, 0x9f, 0xc5, 0x77, 0x49, 0x82, 0xda, 0xb6, 0xc7,
|
0xda, 0x17, 0x8d, 0xd1, 0x11, 0x14, 0x5b, 0x6d, 0xac, 0x37, 0x07, 0x5d, 0xfc, 0x2c, 0xc9, 0x25,
|
||||||
0x04, 0xbe, 0x44, 0xbf, 0x80, 0xf2, 0xe0, 0xd9, 0x93, 0x6e, 0x47, 0xff, 0xad, 0xc1, 0x22, 0xa6,
|
0x0d, 0x6a, 0x39, 0x3e, 0x13, 0xf8, 0x02, 0xfd, 0x02, 0xca, 0xfd, 0x67, 0x4f, 0x3a, 0x6d, 0xe3,
|
||||||
0x95, 0x07, 0xab, 0xb5, 0x7a, 0x77, 0x07, 0x4c, 0xe6, 0x1e, 0x19, 0x9b, 0x01, 0xb1, 0x06, 0xbc,
|
0xb7, 0x26, 0x8b, 0x98, 0x51, 0x1f, 0x2c, 0x57, 0xda, 0xdd, 0x2d, 0x30, 0x99, 0xf9, 0x64, 0x64,
|
||||||
0x89, 0x84, 0xce, 0x82, 0x80, 0x5a, 0xb0, 0x1f, 0x2f, 0xdd, 0x6e, 0x96, 0x51, 0x3e, 0x59, 0xad,
|
0x85, 0xc4, 0xee, 0xf3, 0x1b, 0x26, 0x72, 0x4a, 0x02, 0x6a, 0xc2, 0x7e, 0xb2, 0x74, 0xb3, 0x59,
|
||||||
0xd5, 0x8f, 0xbe, 0x77, 0xfd, 0x66, 0xf7, 0x82, 0x80, 0xee, 0x41, 0x3e, 0x0a, 0x12, 0x2b, 0x29,
|
0x56, 0xfd, 0x64, 0xb9, 0xd2, 0x3e, 0xfa, 0xde, 0xf5, 0xeb, 0xdd, 0x25, 0x01, 0xdd, 0x83, 0x42,
|
||||||
0xb9, 0x34, 0x5a, 0x70, 0xfc, 0x17, 0x01, 0x8a, 0x9b, 0x72, 0x15, 0x12, 0xae, 0xf7, 0x0c, 0x0d,
|
0x1c, 0x24, 0x51, 0x52, 0x7a, 0x69, 0xbc, 0xe0, 0xf8, 0x2f, 0x02, 0x14, 0xd7, 0xed, 0x2a, 0x22,
|
||||||
0xe3, 0x1e, 0x8e, 0x19, 0xd8, 0x38, 0x75, 0xca, 0x86, 0xe8, 0x2e, 0xe4, 0xcf, 0x34, 0x5d, 0xc3,
|
0xdc, 0xe8, 0x9a, 0x3a, 0xc6, 0x5d, 0x9c, 0x30, 0xb0, 0x76, 0x1a, 0x94, 0x0d, 0xd1, 0x5d, 0x28,
|
||||||
0x9d, 0x56, 0x9c, 0x18, 0x1b, 0xc8, 0x19, 0x71, 0x89, 0x67, 0x8f, 0xd1, 0xc7, 0x50, 0xd6, 0x7b,
|
0x9c, 0xe9, 0x86, 0x8e, 0xdb, 0xcd, 0xa4, 0x30, 0xd6, 0x90, 0x33, 0xe2, 0x11, 0xdf, 0x19, 0xa1,
|
||||||
0xc6, 0xe0, 0xbc, 0xf5, 0x38, 0xbe, 0x3a, 0xdb, 0x3f, 0x11, 0x6a, 0xb0, 0x18, 0x5f, 0x30, 0x3e,
|
0x8f, 0xa1, 0x6c, 0x74, 0xcd, 0xfe, 0x79, 0xf3, 0x71, 0x92, 0x3a, 0xdb, 0x3f, 0x15, 0xaa, 0x3f,
|
||||||
0x8f, 0xc3, 0x1c, 0x7a, 0xda, 0xe8, 0x76, 0xda, 0x1c, 0x9a, 0x51, 0xe4, 0xd5, 0x5a, 0xbd, 0xbd,
|
0x1f, 0x5d, 0x30, 0x3e, 0x8f, 0xa3, 0x1a, 0x7a, 0x5a, 0xef, 0xb4, 0x5b, 0x1c, 0x9a, 0x55, 0x95,
|
||||||
0x81, 0x76, 0xf8, 0xaf, 0x4a, 0x88, 0x3d, 0xb6, 0xa0, 0xfa, 0xfd, 0x85, 0x09, 0xa9, 0x90, 0x6b,
|
0xe5, 0x4a, 0xbb, 0xbd, 0x86, 0xb6, 0xf9, 0x3b, 0x26, 0xc2, 0x1e, 0xdb, 0x50, 0xf9, 0xfe, 0xc6,
|
||||||
0xf4, 0xfb, 0x9a, 0xde, 0x8e, 0x4f, 0xbf, 0xf5, 0x35, 0xe6, 0x73, 0xe2, 0x5a, 0x21, 0xe2, 0xb4,
|
0x84, 0x34, 0xc8, 0xd7, 0x7b, 0x3d, 0xdd, 0x68, 0x25, 0x7f, 0xbf, 0xf1, 0xd5, 0x67, 0x33, 0xe2,
|
||||||
0x87, 0xcf, 0xb4, 0x61, 0x7c, 0xf8, 0x2d, 0xe2, 0x94, 0x86, 0x1d, 0xbc, 0x79, 0xf8, 0xea, 0xbb,
|
0xd9, 0x11, 0xe2, 0xb4, 0x8b, 0xcf, 0xf4, 0x41, 0xf2, 0xf3, 0x1b, 0xc4, 0x29, 0x8d, 0xae, 0xf7,
|
||||||
0x6a, 0xea, 0xdb, 0xef, 0xaa, 0xa9, 0x57, 0x57, 0x55, 0xe1, 0xdb, 0xab, 0xaa, 0xf0, 0xaf, 0xab,
|
0xc6, 0xe1, 0xab, 0xef, 0x2a, 0x3b, 0xdf, 0x7e, 0x57, 0xd9, 0x79, 0x75, 0x55, 0x11, 0xbe, 0xbd,
|
||||||
0x6a, 0xea, 0x3f, 0x57, 0x55, 0xe1, 0x9b, 0x7f, 0x57, 0x85, 0x51, 0x8e, 0x15, 0xb2, 0xcf, 0xff,
|
0xaa, 0x08, 0xff, 0xba, 0xaa, 0xec, 0xfc, 0xe7, 0xaa, 0x22, 0x7c, 0xf3, 0xef, 0x8a, 0x30, 0xcc,
|
||||||
0x1f, 0x00, 0x00, 0xff, 0xff, 0x0d, 0x36, 0x7c, 0x9b, 0xc0, 0x0e, 0x00, 0x00,
|
0xb3, 0x46, 0xf6, 0xf9, 0xff, 0x03, 0x00, 0x00, 0xff, 0xff, 0xe5, 0x17, 0x2b, 0x62, 0xdd, 0x0e,
|
||||||
|
0x00, 0x00,
|
||||||
}
|
}
|
||||||
|
|
|
@ -146,6 +146,7 @@ message Request {
|
||||||
int32 size = 5;
|
int32 size = 5;
|
||||||
bytes hash = 6;
|
bytes hash = 6;
|
||||||
bool from_temporary = 7;
|
bool from_temporary = 7;
|
||||||
|
uint32 weak_hash = 8;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Response
|
// Response
|
||||||
|
|
|
@ -11,6 +11,7 @@ type TestModel struct {
|
||||||
offset int64
|
offset int64
|
||||||
size int
|
size int
|
||||||
hash []byte
|
hash []byte
|
||||||
|
weakHash uint32
|
||||||
fromTemporary bool
|
fromTemporary bool
|
||||||
closedCh chan struct{}
|
closedCh chan struct{}
|
||||||
closedErr error
|
closedErr error
|
||||||
|
@ -28,12 +29,13 @@ func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo) {
|
||||||
func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) {
|
func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
|
func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, hash []byte, weakHash uint32, fromTemporary bool, buf []byte) error {
|
||||||
t.folder = folder
|
t.folder = folder
|
||||||
t.name = name
|
t.name = name
|
||||||
t.offset = offset
|
t.offset = offset
|
||||||
t.size = len(buf)
|
t.size = len(buf)
|
||||||
t.hash = hash
|
t.hash = hash
|
||||||
|
t.weakHash = weakHash
|
||||||
t.fromTemporary = fromTemporary
|
t.fromTemporary = fromTemporary
|
||||||
copy(buf, t.data)
|
copy(buf, t.data)
|
||||||
return nil
|
return nil
|
||||||
|
|
|
@ -26,7 +26,7 @@ func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileI
|
||||||
m.Model.IndexUpdate(deviceID, folder, files)
|
m.Model.IndexUpdate(deviceID, folder, files)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
|
func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, weakHash uint32, fromTemporary bool, buf []byte) error {
|
||||||
name = norm.NFD.String(name)
|
name = norm.NFD.String(name)
|
||||||
return m.Model.Request(deviceID, folder, name, offset, hash, fromTemporary, buf)
|
return m.Model.Request(deviceID, folder, name, offset, hash, weakHash, fromTemporary, buf)
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,14 +25,14 @@ func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileI
|
||||||
m.Model.IndexUpdate(deviceID, folder, files)
|
m.Model.IndexUpdate(deviceID, folder, files)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
|
func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, weakHash uint32, fromTemporary bool, buf []byte) error {
|
||||||
if strings.Contains(name, `\`) {
|
if strings.Contains(name, `\`) {
|
||||||
l.Warnf("Dropping request for %s, contains invalid path separator", name)
|
l.Warnf("Dropping request for %s, contains invalid path separator", name)
|
||||||
return ErrNoSuchFile
|
return ErrNoSuchFile
|
||||||
}
|
}
|
||||||
|
|
||||||
name = filepath.FromSlash(name)
|
name = filepath.FromSlash(name)
|
||||||
return m.Model.Request(deviceID, folder, name, offset, hash, fromTemporary, buf)
|
return m.Model.Request(deviceID, folder, name, offset, hash, weakHash, fromTemporary, buf)
|
||||||
}
|
}
|
||||||
|
|
||||||
func fixupFiles(files []FileInfo) []FileInfo {
|
func fixupFiles(files []FileInfo) []FileInfo {
|
||||||
|
|
|
@ -107,7 +107,7 @@ type Model interface {
|
||||||
// An index update was received from the peer device
|
// An index update was received from the peer device
|
||||||
IndexUpdate(deviceID DeviceID, folder string, files []FileInfo)
|
IndexUpdate(deviceID DeviceID, folder string, files []FileInfo)
|
||||||
// A request was made by the peer device
|
// A request was made by the peer device
|
||||||
Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error
|
Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, weakHash uint32, fromTemporary bool, buf []byte) error
|
||||||
// A cluster configuration message was received
|
// A cluster configuration message was received
|
||||||
ClusterConfig(deviceID DeviceID, config ClusterConfig)
|
ClusterConfig(deviceID DeviceID, config ClusterConfig)
|
||||||
// The peer device closed the connection
|
// The peer device closed the connection
|
||||||
|
@ -122,7 +122,7 @@ type Connection interface {
|
||||||
Name() string
|
Name() string
|
||||||
Index(folder string, files []FileInfo) error
|
Index(folder string, files []FileInfo) error
|
||||||
IndexUpdate(folder string, files []FileInfo) error
|
IndexUpdate(folder string, files []FileInfo) error
|
||||||
Request(folder string, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error)
|
Request(folder string, name string, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error)
|
||||||
ClusterConfig(config ClusterConfig)
|
ClusterConfig(config ClusterConfig)
|
||||||
DownloadProgress(folder string, updates []FileDownloadProgressUpdate)
|
DownloadProgress(folder string, updates []FileDownloadProgressUpdate)
|
||||||
Statistics() Statistics
|
Statistics() Statistics
|
||||||
|
@ -254,7 +254,7 @@ func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Request returns the bytes for the specified block after fetching them from the connected peer.
|
// Request returns the bytes for the specified block after fetching them from the connected peer.
|
||||||
func (c *rawConnection) Request(folder string, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
|
func (c *rawConnection) Request(folder string, name string, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) {
|
||||||
c.nextIDMut.Lock()
|
c.nextIDMut.Lock()
|
||||||
id := c.nextID
|
id := c.nextID
|
||||||
c.nextID++
|
c.nextID++
|
||||||
|
@ -275,6 +275,7 @@ func (c *rawConnection) Request(folder string, name string, offset int64, size i
|
||||||
Offset: offset,
|
Offset: offset,
|
||||||
Size: int32(size),
|
Size: int32(size),
|
||||||
Hash: hash,
|
Hash: hash,
|
||||||
|
WeakHash: weakHash,
|
||||||
FromTemporary: fromTemporary,
|
FromTemporary: fromTemporary,
|
||||||
}, nil)
|
}, nil)
|
||||||
if !ok {
|
if !ok {
|
||||||
|
@ -584,7 +585,7 @@ func (c *rawConnection) handleRequest(req Request) {
|
||||||
buf = make([]byte, size)
|
buf = make([]byte, size)
|
||||||
}
|
}
|
||||||
|
|
||||||
err := c.receiver.Request(c.id, req.Folder, req.Name, req.Offset, req.Hash, req.FromTemporary, buf)
|
err := c.receiver.Request(c.id, req.Folder, req.Name, req.Offset, req.Hash, req.WeakHash, req.FromTemporary, buf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.send(&Response{
|
c.send(&Response{
|
||||||
ID: req.ID,
|
ID: req.ID,
|
||||||
|
|
|
@ -72,7 +72,7 @@ func TestClose(t *testing.T) {
|
||||||
c0.Index("default", nil)
|
c0.Index("default", nil)
|
||||||
c0.Index("default", nil)
|
c0.Index("default", nil)
|
||||||
|
|
||||||
if _, err := c0.Request("default", "foo", 0, 0, nil, false); err == nil {
|
if _, err := c0.Request("default", "foo", 0, 0, nil, 0, false); err == nil {
|
||||||
t.Error("Request should return an error")
|
t.Error("Request should return an error")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -34,7 +34,7 @@ func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo) error {
|
||||||
return c.Connection.IndexUpdate(folder, myFs)
|
return c.Connection.IndexUpdate(folder, myFs)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c wireFormatConnection) Request(folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
|
func (c wireFormatConnection) Request(folder, name string, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) {
|
||||||
name = norm.NFC.String(filepath.ToSlash(name))
|
name = norm.NFC.String(filepath.ToSlash(name))
|
||||||
return c.Connection.Request(folder, name, offset, size, hash, fromTemporary)
|
return c.Connection.Request(folder, name, offset, size, hash, weakHash, fromTemporary)
|
||||||
}
|
}
|
||||||
|
|
|
@ -62,26 +62,24 @@ func HashFile(ctx context.Context, fs fs.Filesystem, path string, blockSize int,
|
||||||
// workers are used in parallel. The outbox will become closed when the inbox
|
// workers are used in parallel. The outbox will become closed when the inbox
|
||||||
// is closed and all items handled.
|
// is closed and all items handled.
|
||||||
type parallelHasher struct {
|
type parallelHasher struct {
|
||||||
fs fs.Filesystem
|
fs fs.Filesystem
|
||||||
workers int
|
workers int
|
||||||
outbox chan<- protocol.FileInfo
|
outbox chan<- protocol.FileInfo
|
||||||
inbox <-chan protocol.FileInfo
|
inbox <-chan protocol.FileInfo
|
||||||
counter Counter
|
counter Counter
|
||||||
done chan<- struct{}
|
done chan<- struct{}
|
||||||
useWeakHashes bool
|
wg sync.WaitGroup
|
||||||
wg sync.WaitGroup
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func newParallelHasher(ctx context.Context, fs fs.Filesystem, workers int, outbox chan<- protocol.FileInfo, inbox <-chan protocol.FileInfo, counter Counter, done chan<- struct{}, useWeakHashes bool) {
|
func newParallelHasher(ctx context.Context, fs fs.Filesystem, workers int, outbox chan<- protocol.FileInfo, inbox <-chan protocol.FileInfo, counter Counter, done chan<- struct{}) {
|
||||||
ph := ¶llelHasher{
|
ph := ¶llelHasher{
|
||||||
fs: fs,
|
fs: fs,
|
||||||
workers: workers,
|
workers: workers,
|
||||||
outbox: outbox,
|
outbox: outbox,
|
||||||
inbox: inbox,
|
inbox: inbox,
|
||||||
counter: counter,
|
counter: counter,
|
||||||
done: done,
|
done: done,
|
||||||
useWeakHashes: useWeakHashes,
|
wg: sync.NewWaitGroup(),
|
||||||
wg: sync.NewWaitGroup(),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for i := 0; i < workers; i++ {
|
for i := 0; i < workers; i++ {
|
||||||
|
@ -106,7 +104,7 @@ func (ph *parallelHasher) hashFiles(ctx context.Context) {
|
||||||
panic("Bug. Asked to hash a directory or a deleted file.")
|
panic("Bug. Asked to hash a directory or a deleted file.")
|
||||||
}
|
}
|
||||||
|
|
||||||
blocks, err := HashFile(ctx, ph.fs, f.Name, f.BlockSize(), ph.counter, ph.useWeakHashes)
|
blocks, err := HashFile(ctx, ph.fs, f.Name, f.BlockSize(), ph.counter, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
l.Debugln("hash error:", f.Name, err)
|
l.Debugln("hash error:", f.Name, err)
|
||||||
continue
|
continue
|
||||||
|
|
|
@ -7,6 +7,7 @@
|
||||||
package scanner
|
package scanner
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"hash"
|
"hash"
|
||||||
"io"
|
"io"
|
||||||
|
@ -107,6 +108,29 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou
|
||||||
return blocks, nil
|
return blocks, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func Validate(buf, hash []byte, weakHash uint32) bool {
|
||||||
|
rd := bytes.NewReader(buf)
|
||||||
|
if weakHash != 0 {
|
||||||
|
whf := adler32.New()
|
||||||
|
if _, err := io.Copy(whf, rd); err == nil && whf.Sum32() == weakHash {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
// Copy error or mismatch, go to next algo.
|
||||||
|
rd.Seek(0, io.SeekStart)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(hash) > 0 {
|
||||||
|
hf := sha256.New()
|
||||||
|
if _, err := io.Copy(hf, rd); err == nil {
|
||||||
|
// Sum allocates, so let's hope we don't hit this often.
|
||||||
|
return bytes.Equal(hf.Sum(nil), hash)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Both algos failed or no hashes were specified. Assume it's all good.
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
type noopHash struct{}
|
type noopHash struct{}
|
||||||
|
|
||||||
func (noopHash) Sum32() uint32 { return 0 }
|
func (noopHash) Sum32() uint32 { return 0 }
|
||||||
|
|
|
@ -64,8 +64,6 @@ type Config struct {
|
||||||
// Optional progress tick interval which defines how often FolderScanProgress
|
// Optional progress tick interval which defines how often FolderScanProgress
|
||||||
// events are emitted. Negative number means disabled.
|
// events are emitted. Negative number means disabled.
|
||||||
ProgressTickIntervalS int
|
ProgressTickIntervalS int
|
||||||
// Whether or not we should also compute weak hashes
|
|
||||||
UseWeakHashes bool
|
|
||||||
// Whether to use large blocks for large files or the old standard of 128KiB for everything.
|
// Whether to use large blocks for large files or the old standard of 128KiB for everything.
|
||||||
UseLargeBlocks bool
|
UseLargeBlocks bool
|
||||||
}
|
}
|
||||||
|
@ -120,7 +118,7 @@ func (w *walker) walk(ctx context.Context) chan protocol.FileInfo {
|
||||||
// We're not required to emit scan progress events, just kick off hashers,
|
// We're not required to emit scan progress events, just kick off hashers,
|
||||||
// and feed inputs directly from the walker.
|
// and feed inputs directly from the walker.
|
||||||
if w.ProgressTickIntervalS < 0 {
|
if w.ProgressTickIntervalS < 0 {
|
||||||
newParallelHasher(ctx, w.Filesystem, w.Hashers, finishedChan, toHashChan, nil, nil, w.UseWeakHashes)
|
newParallelHasher(ctx, w.Filesystem, w.Hashers, finishedChan, toHashChan, nil, nil)
|
||||||
return finishedChan
|
return finishedChan
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -151,7 +149,7 @@ func (w *walker) walk(ctx context.Context) chan protocol.FileInfo {
|
||||||
done := make(chan struct{})
|
done := make(chan struct{})
|
||||||
progress := newByteCounter()
|
progress := newByteCounter()
|
||||||
|
|
||||||
newParallelHasher(ctx, w.Filesystem, w.Hashers, finishedChan, realToHashChan, progress, done, w.UseWeakHashes)
|
newParallelHasher(ctx, w.Filesystem, w.Hashers, finishedChan, realToHashChan, progress, done)
|
||||||
|
|
||||||
// A routine which actually emits the FolderScanProgress events
|
// A routine which actually emits the FolderScanProgress events
|
||||||
// every w.ProgressTicker ticks, until the hasher routines terminate.
|
// every w.ProgressTicker ticks, until the hasher routines terminate.
|
||||||
|
|
|
@ -11,6 +11,10 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/chmduquesne/rollinghash/adler32"
|
"github.com/chmduquesne/rollinghash/adler32"
|
||||||
|
"github.com/chmduquesne/rollinghash/bozo32"
|
||||||
|
"github.com/chmduquesne/rollinghash/buzhash32"
|
||||||
|
"github.com/chmduquesne/rollinghash/buzhash64"
|
||||||
|
"github.com/chmduquesne/rollinghash/rabinkarp64"
|
||||||
)
|
)
|
||||||
|
|
||||||
const testFile = "../model/testdata/~syncthing~file.tmp"
|
const testFile = "../model/testdata/~syncthing~file.tmp"
|
||||||
|
@ -59,3 +63,115 @@ func BenchmarkWeakHashAdler32Roll(b *testing.B) {
|
||||||
|
|
||||||
b.SetBytes(size)
|
b.SetBytes(size)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func BenchmarkWeakHashRabinKarp64(b *testing.B) {
|
||||||
|
data := make([]byte, size)
|
||||||
|
hf := rabinkarp64.New()
|
||||||
|
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
hf.Write(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
_ = hf.Sum64()
|
||||||
|
b.SetBytes(size)
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkWeakHashRabinKarp64Roll(b *testing.B) {
|
||||||
|
data := make([]byte, size)
|
||||||
|
hf := rabinkarp64.New()
|
||||||
|
hf.Write(data)
|
||||||
|
|
||||||
|
b.ResetTimer()
|
||||||
|
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
for i := 0; i <= size; i++ {
|
||||||
|
hf.Roll('a')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
b.SetBytes(size)
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkWeakHashBozo32(b *testing.B) {
|
||||||
|
data := make([]byte, size)
|
||||||
|
hf := bozo32.New()
|
||||||
|
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
hf.Write(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
_ = hf.Sum32()
|
||||||
|
b.SetBytes(size)
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkWeakHashBozo32Roll(b *testing.B) {
|
||||||
|
data := make([]byte, size)
|
||||||
|
hf := bozo32.New()
|
||||||
|
hf.Write(data)
|
||||||
|
|
||||||
|
b.ResetTimer()
|
||||||
|
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
for i := 0; i <= size; i++ {
|
||||||
|
hf.Roll('a')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
b.SetBytes(size)
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkWeakHashBuzhash32(b *testing.B) {
|
||||||
|
data := make([]byte, size)
|
||||||
|
hf := buzhash32.New()
|
||||||
|
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
hf.Write(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
_ = hf.Sum32()
|
||||||
|
b.SetBytes(size)
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkWeakHashBuzhash32Roll(b *testing.B) {
|
||||||
|
data := make([]byte, size)
|
||||||
|
hf := buzhash32.New()
|
||||||
|
hf.Write(data)
|
||||||
|
|
||||||
|
b.ResetTimer()
|
||||||
|
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
for i := 0; i <= size; i++ {
|
||||||
|
hf.Roll('a')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
b.SetBytes(size)
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkWeakHashBuzhash64(b *testing.B) {
|
||||||
|
data := make([]byte, size)
|
||||||
|
hf := buzhash64.New()
|
||||||
|
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
hf.Write(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
_ = hf.Sum64()
|
||||||
|
b.SetBytes(size)
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkWeakHashBuzhash64Roll(b *testing.B) {
|
||||||
|
data := make([]byte, size)
|
||||||
|
hf := buzhash64.New()
|
||||||
|
hf.Write(data)
|
||||||
|
|
||||||
|
b.ResetTimer()
|
||||||
|
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
for i := 0; i <= size; i++ {
|
||||||
|
hf.Roll('a')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
b.SetBytes(size)
|
||||||
|
}
|
||||||
|
|
|
@ -20,10 +20,6 @@ const (
|
||||||
maxWeakhashFinderHits = 10
|
maxWeakhashFinderHits = 10
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
|
||||||
Enabled = true
|
|
||||||
)
|
|
||||||
|
|
||||||
// Find finds all the blocks of the given size within io.Reader that matches
|
// Find finds all the blocks of the given size within io.Reader that matches
|
||||||
// the hashes provided, and returns a hash -> slice of offsets within reader
|
// the hashes provided, and returns a hash -> slice of offsets within reader
|
||||||
// map, that produces the same weak hash.
|
// map, that produces the same weak hash.
|
||||||
|
|
|
@ -94,7 +94,7 @@
|
||||||
"importpath": "github.com/chmduquesne/rollinghash",
|
"importpath": "github.com/chmduquesne/rollinghash",
|
||||||
"repository": "https://github.com/chmduquesne/rollinghash",
|
"repository": "https://github.com/chmduquesne/rollinghash",
|
||||||
"vcs": "git",
|
"vcs": "git",
|
||||||
"revision": "3dc7875a1f890f9bcf0619adb5571fc6f7d516bb",
|
"revision": "abb8cbaf9915e48ee20cae94bcd94221b61707a2",
|
||||||
"branch": "master",
|
"branch": "master",
|
||||||
"notests": true
|
"notests": true
|
||||||
},
|
},
|
||||||
|
|
Loading…
Reference in New Issue