Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/model: Update db on scan/pull in folder (#5608)

This commit is contained in:
Simon Frei 2019-04-07 13:29:17 +02:00 committed by Jakob Borg
parent 48b1a2b264
commit 395e524e2d
8 changed files with 157 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
lib/model/folder.go
View File

@ -198,9 +198,9 @@ func (f *folder) Serve() {
func (f *folder) BringToFront(string) {}
func (f *folder) Override(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {}
func (f *folder) Override() {}
func (f *folder) Revert(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {}
func (f *folder) Revert() {}
func (f *folder) DelayScan(next time.Duration) {
f.Delay(next)
@ -345,7 +345,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
Subs: subDirs,
Matcher: f.ignores,
TempLifetime: time.Duration(f.model.cfg.Options().KeepTemporariesH) * time.Hour,
CurrentFiler: cFiler{f.model, f.ID},
CurrentFiler: cFiler{f.fset},
Filesystem: mtimefs,
IgnorePerms: f.IgnorePerms,
AutoNormalize: f.AutoNormalize,
@ -361,7 +361,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
l.Debugf("Stopping scan of folder %s due to: %s", f.Description(), err)
return err
}
f.model.updateLocalsFromScanning(f.ID, fs)
f.updateLocalsFromScanning(fs)
return nil
}
// Resolve items which are identical with the global state.
@ -737,6 +737,86 @@ func (f *folder) Errors() []FileError {
return append([]FileError{}, f.scanErrors...)
}
// ForceRescan marks the file such that it gets rehashed on next scan and then
// immediately executes that scan.
func (f *folder) ForceRescan(file protocol.FileInfo) error {
file.SetMustRescan(f.shortID)
f.fset.Update(protocol.LocalDeviceID, []protocol.FileInfo{file})
return f.Scan([]string{file.Name})
}
func (f *folder) updateLocalsFromScanning(fs []protocol.FileInfo) {
f.updateLocals(fs)
f.emitDiskChangeEvents(fs, events.LocalChangeDetected)
}
func (f *folder) updateLocalsFromPulling(fs []protocol.FileInfo) {
f.updateLocals(fs)
f.emitDiskChangeEvents(fs, events.RemoteChangeDetected)
}
func (f *folder) updateLocals(fs []protocol.FileInfo) {
f.fset.Update(protocol.LocalDeviceID, fs)
filenames := make([]string, len(fs))
for i, file := range fs {
filenames[i] = file.Name
}
events.Default.Log(events.LocalIndexUpdated, map[string]interface{}{
"folder": f.ID,
"items": len(fs),
"filenames": filenames,
"version": f.fset.Sequence(protocol.LocalDeviceID),
})
}
func (f *folder) emitDiskChangeEvents(fs []protocol.FileInfo, typeOfEvent events.EventType) {
for _, file := range fs {
if file.IsInvalid() {
continue
}
objType := "file"
action := "modified"
switch {
case file.IsDeleted():
action = "deleted"
// If our local vector is version 1 AND it is the only version
// vector so far seen for this file then it is a new file. Else if
// it is > 1 it's not new, and if it is 1 but another shortId
// version vector exists then it is new for us but created elsewhere
// so the file is still not new but modified by us. Only if it is
// truly new do we change this to 'added', else we leave it as
// 'modified'.
case len(file.Version.Counters) == 1 && file.Version.Counters[0].Value == 1:
action = "added"
}
if file.IsSymlink() {
objType = "symlink"
} else if file.IsDirectory() {
objType = "dir"
}
// Two different events can be fired here based on what EventType is passed into function
events.Default.Log(typeOfEvent, map[string]string{
"folder": f.ID,
"folderID": f.ID, // incorrect, deprecated, kept for historical compliance
"label": f.Label,
"action": action,
"type": objType,
"path": filepath.FromSlash(file.Name),
"modifiedBy": file.ModifiedBy.String(),
})
}
}
// The exists function is expected to return true for all known paths
// (excluding "" and ".")
func unifySubs(dirs []string, exists func(dir string) bool) []string {
@ -770,3 +850,12 @@ func unifySubs(dirs []string, exists func(dir string) bool) []string {
}
return dirs
}
type cFiler struct {
*db.FileSet
}
// Implements scanner.CurrentFiler
func (cf cFiler) CurrentFile(file string) (protocol.FileInfo, bool) {
return cf.Get(protocol.LocalDeviceID, file)
}

10
lib/model/folder_recvonly.go
View File

@ -62,7 +62,7 @@ func newReceiveOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matche
return &receiveOnlyFolder{sr}
}
func (f *receiveOnlyFolder) Revert(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {
func (f *receiveOnlyFolder) Revert() {
f.setState(FolderScanning)
defer f.setState(FolderIdle)
@ -78,7 +78,7 @@ func (f *receiveOnlyFolder) Revert(fs *db.FileSet, updateFn func([]protocol.File
batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles)
batchSizeBytes := 0
fs.WithHave(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
f.fset.WithHave(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
fi := intf.(protocol.FileInfo)
if !fi.IsReceiveOnlyChanged() {
// We're only interested in files that have changed locally in
@ -124,14 +124,14 @@ func (f *receiveOnlyFolder) Revert(fs *db.FileSet, updateFn func([]protocol.File
batchSizeBytes += fi.ProtoSize()
if len(batch) >= maxBatchSizeFiles || batchSizeBytes >= maxBatchSizeBytes {
updateFn(batch)
f.updateLocalsFromScanning(batch)
batch = batch[:0]
batchSizeBytes = 0
}
return true
})
if len(batch) > 0 {
updateFn(batch)
f.updateLocalsFromScanning(batch)
}
batch = batch[:0]
batchSizeBytes = 0
@ -153,7 +153,7 @@ func (f *receiveOnlyFolder) Revert(fs *db.FileSet, updateFn func([]protocol.File
})
}
if len(batch) > 0 {
updateFn(batch)
f.updateLocalsFromScanning(batch)
}
// We will likely have changed our local index, but that won't trigger a

31
lib/model/folder_recvonly_test.go
View File

@ -28,8 +28,8 @@ func TestRecvOnlyRevertDeletes(t *testing.T) {
// Get us a model up and running
m, fcfg := setupROFolder()
ffs := fcfg.Filesystem()
m, f := setupROFolder()
ffs := f.Filesystem()
defer os.Remove(m.cfg.ConfigPath())
defer os.Remove(ffs.URI())
defer m.Stop()
@ -48,7 +48,7 @@ func TestRecvOnlyRevertDeletes(t *testing.T) {
// Send and index update for the known stuff
m.Index(device1, "ro", knownFiles)
m.updateLocalsFromScanning("ro", knownFiles)
f.updateLocalsFromScanning(knownFiles)
size := m.GlobalSize("ro")
if size.Files != 1 || size.Directories != 1 {
@ -111,8 +111,8 @@ func TestRecvOnlyRevertNeeds(t *testing.T) {
// Get us a model up and running
m, fcfg := setupROFolder()
ffs := fcfg.Filesystem()
m, f := setupROFolder()
ffs := f.Filesystem()
defer os.Remove(m.cfg.ConfigPath())
defer os.Remove(ffs.URI())
defer m.Stop()
@ -126,7 +126,7 @@ func TestRecvOnlyRevertNeeds(t *testing.T) {
// Send and index update for the known stuff
m.Index(device1, "ro", knownFiles)
m.updateLocalsFromScanning("ro", knownFiles)
f.updateLocalsFromScanning(knownFiles)
// Start the folder. This will cause a scan.
@ -204,8 +204,8 @@ func TestRecvOnlyUndoChanges(t *testing.T) {
// Get us a model up and running
m, fcfg := setupROFolder()
ffs := fcfg.Filesystem()
m, f := setupROFolder()
ffs := f.Filesystem()
defer os.Remove(m.cfg.ConfigPath())
defer os.Remove(ffs.URI())
defer m.Stop()
@ -224,7 +224,7 @@ func TestRecvOnlyUndoChanges(t *testing.T) {
// Send and index update for the known stuff
m.Index(device1, "ro", knownFiles)
m.updateLocalsFromScanning("ro", knownFiles)
f.updateLocalsFromScanning(knownFiles)
// Start the folder. This will cause a scan.
@ -316,7 +316,7 @@ func setupKnownFiles(t *testing.T, ffs fs.Filesystem, data []byte) []protocol.Fi
return knownFiles
}
func setupROFolder() (*model, config.FolderConfiguration) {
func setupROFolder() (*model, *sendOnlyFolder) {
w := createTmpWrapper(defaultCfg)
fcfg := testFolderConfigTmp()
fcfg.ID = "ro"
@ -324,9 +324,16 @@ func setupROFolder() (*model, config.FolderConfiguration) {
w.SetFolder(fcfg)
m := newModel(w, myID, "syncthing", "dev", db.OpenMemory(), nil)
m.ServeBackground()
m.AddFolder(fcfg)
return m, fcfg
f := &sendOnlyFolder{
folder: folder{
fset: m.folderFiles[fcfg.ID],
FolderConfiguration: fcfg,
},
}
m.ServeBackground()
return m, f
}

14
lib/model/folder_sendonly.go
View File

@ -49,7 +49,7 @@ func (f *sendOnlyFolder) pull() bool {
f.fset.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes {
f.model.updateLocalsFromPulling(f.folderID, batch)
f.updateLocalsFromPulling(batch)
batch = batch[:0]
batchSizeBytes = 0
}
@ -85,25 +85,25 @@ func (f *sendOnlyFolder) pull() bool {
})
if len(batch) > 0 {
f.model.updateLocalsFromPulling(f.folderID, batch)
f.updateLocalsFromPulling(batch)
}
return true
}
func (f *sendOnlyFolder) Override(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {
func (f *sendOnlyFolder) Override() {
f.setState(FolderScanning)
batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles)
batchSizeBytes := 0
fs.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
f.fset.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
need := fi.(protocol.FileInfo)
if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes {
updateFn(batch)
f.updateLocalsFromScanning(batch)
batch = batch[:0]
batchSizeBytes = 0
}
have, ok := fs.Get(protocol.LocalDeviceID, need.Name)
have, ok := f.fset.Get(protocol.LocalDeviceID, need.Name)
// Don't override files that are in a bad state (ignored,
// unsupported, must rescan, ...).
if ok && have.IsInvalid() {
@ -126,7 +126,7 @@ func (f *sendOnlyFolder) Override(fs *db.FileSet, updateFn func([]protocol.FileI
return true
})
if len(batch) > 0 {
updateFn(batch)
f.updateLocalsFromScanning(batch)
}
f.setState(FolderIdle)
}

2
lib/model/folder_sendrecv.go
View File

@ -1604,7 +1604,7 @@ func (f *sendReceiveFolder) dbUpdaterRoutine(dbUpdateChan <-chan dbUpdateJob) {
// All updates to file/folder objects that originated remotely
// (across the network) use this call to updateLocals
f.model.updateLocalsFromPulling(f.folderID, files)
f.updateLocalsFromPulling(files)
if found {
f.ReceivedFile(lastFile.Name, lastFile.IsDeleted())

20
lib/model/folder_sendrecv_test.go
View File

@ -94,11 +94,6 @@ func setupSendReceiveFolder(files ...protocol.FileInfo) (*model, *sendReceiveFol
fcfg := testFolderConfigTmp()
model.AddFolder(fcfg)
// Update index
if files != nil {
model.updateLocalsFromScanning("default", files)
}
f := &sendReceiveFolder{
folder: folder{
stateTracker: newStateTracker("default"),
@ -115,6 +110,11 @@ func setupSendReceiveFolder(files ...protocol.FileInfo) (*model, *sendReceiveFol
}
f.fs = fs.NewMtimeFS(f.Filesystem(), db.NewNamespacedKV(model.db, "mtime"))
// Update index
if files != nil {
f.updateLocalsFromScanning(files)
}
// Folders are never actually started, so no initial scan will be done
close(f.initialScanFinished)
@ -362,7 +362,7 @@ func TestWeakHash(t *testing.T) {
ModifiedS: info.ModTime().Unix() + 1,
}
model.updateLocalsFromScanning("default", []protocol.FileInfo{existingFile})
fo.updateLocalsFromScanning([]protocol.FileInfo{existingFile})
copyChan := make(chan copyBlocksState)
pullChan := make(chan pullBlockState, expectBlocks)
@ -440,7 +440,7 @@ func TestCopierCleanup(t *testing.T) {
file.Blocks = []protocol.BlockInfo{blocks[1]}
file.Version = file.Version.Update(myID.Short())
// Update index (removing old blocks)
m.updateLocalsFromScanning("default", []protocol.FileInfo{file})
f.updateLocalsFromScanning([]protocol.FileInfo{file})
if m.finder.Iterate(folders, blocks[0].Hash, iterFn) {
t.Error("Unexpected block found")
@ -453,7 +453,7 @@ func TestCopierCleanup(t *testing.T) {
file.Blocks = []protocol.BlockInfo{blocks[0]}
file.Version = file.Version.Update(myID.Short())
// Update index (removing old blocks)
m.updateLocalsFromScanning("default", []protocol.FileInfo{file})
f.updateLocalsFromScanning([]protocol.FileInfo{file})
if !m.finder.Iterate(folders, blocks[0].Hash, iterFn) {
t.Error("Unexpected block found")
@ -878,7 +878,7 @@ func TestSRConflictReplaceFileByDir(t *testing.T) {
// create local file
file := createFile(t, name, ffs)
file.Version = protocol.Vector{}.Update(myID.Short())
m.updateLocalsFromScanning(f.ID, []protocol.FileInfo{file})
f.updateLocalsFromScanning([]protocol.FileInfo{file})
// Simulate remote creating a dir with the same name
file.Type = protocol.FileInfoTypeDirectory
@ -913,7 +913,7 @@ func TestSRConflictReplaceFileByLink(t *testing.T) {
// create local file
file := createFile(t, name, ffs)
file.Version = protocol.Vector{}.Update(myID.Short())
m.updateLocalsFromScanning(f.ID, []protocol.FileInfo{file})
f.updateLocalsFromScanning([]protocol.FileInfo{file})
// Simulate remote creating a symlink with the same name
file.Type = protocol.FileInfoTypeSymlink

139
lib/model/model.go
View File

@ -54,8 +54,8 @@ const (
type service interface {
BringToFront(string)
Override(*db.FileSet, func([]protocol.FileInfo))
Revert(*db.FileSet, func([]protocol.FileInfo))
Override()
Revert()
DelayScan(d time.Duration)
SchedulePull() // something relevant changed, we should try a pull
Jobs() ([]string, []string) // In progress, Queued
@ -65,6 +65,7 @@ type service interface {
CheckHealth() error
Errors() []FileError
WatchError() error
ForceRescan(file protocol.FileInfo) error
GetStatistics() stats.FolderStatistics
getState() (folderState, time.Time, error)
@ -1611,17 +1612,19 @@ func (m *model) recheckFile(deviceID protocol.DeviceID, folderFs fs.Filesystem,
// The hashes provided part of the request match what we expect to find according
// to what we have in the database, yet the content we've read off the filesystem doesn't
// Something is fishy, invalidate the file and rescan it.
cf.SetMustRescan(m.shortID)
// Update the index and tell others
// The file will temporarily become invalid, which is ok as the content is messed up.
m.updateLocalsFromScanning(folder, []protocol.FileInfo{cf})
if err := m.ScanFolderSubdirs(folder, []string{name}); err != nil {
l.Debugf("%v recheckFile: %s: %q / %q rescan: %s", m, deviceID, folder, name, err)
} else {
l.Debugf("%v recheckFile: %s: %q / %q", m, deviceID, folder, name)
m.fmut.Lock()
runner, ok := m.folderRunners[folder]
m.fmut.Unlock()
if !ok {
l.Debugf("%v recheckFile: %s: %q / %q: Folder stopped before rescan could be scheduled", m, deviceID, folder, name)
return
}
if err := runner.ForceRescan(cf); err != nil {
l.Debugf("%v recheckFile: %s: %q / %q rescan: %s", m, deviceID, folder, name, err)
return
}
l.Debugf("%v recheckFile: %s: %q / %q", m, deviceID, folder, name)
}
func (m *model) CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool) {
@ -1644,16 +1647,6 @@ func (m *model) CurrentGlobalFile(folder string, file string) (protocol.FileInfo
return fs.GetGlobal(file)
}
type cFiler struct {
m Model
r string
}
// Implements scanner.CurrentFiler
func (cf cFiler) CurrentFile(file string) (protocol.FileInfo, bool) {
return cf.m.CurrentFolderFile(cf.r, file)
}
// Connection returns the current connection for device, and a boolean whether a connection was found.
func (m *model) Connection(deviceID protocol.DeviceID) (connections.Connection, bool) {
m.pmut.RLock()
@ -1988,92 +1981,6 @@ func sendIndexTo(prevSequence int64, conn protocol.Connection, folder string, fs
return f.Sequence, err
}
func (m *model) updateLocalsFromScanning(folder string, fs []protocol.FileInfo) {
m.updateLocals(folder, fs)
m.fmut.RLock()
folderCfg := m.folderCfgs[folder]
m.fmut.RUnlock()
m.diskChangeDetected(folderCfg, fs, events.LocalChangeDetected)
}
func (m *model) updateLocalsFromPulling(folder string, fs []protocol.FileInfo) {
m.updateLocals(folder, fs)
m.fmut.RLock()
folderCfg := m.folderCfgs[folder]
m.fmut.RUnlock()
m.diskChangeDetected(folderCfg, fs, events.RemoteChangeDetected)
}
func (m *model) updateLocals(folder string, fs []protocol.FileInfo) {
m.fmut.RLock()
files := m.folderFiles[folder]
m.fmut.RUnlock()
if files == nil {
// The folder doesn't exist.
return
}
files.Update(protocol.LocalDeviceID, fs)
filenames := make([]string, len(fs))
for i, file := range fs {
filenames[i] = file.Name
}
events.Default.Log(events.LocalIndexUpdated, map[string]interface{}{
"folder": folder,
"items": len(fs),
"filenames": filenames,
"version": files.Sequence(protocol.LocalDeviceID),
})
}
func (m *model) diskChangeDetected(folderCfg config.FolderConfiguration, files []protocol.FileInfo, typeOfEvent events.EventType) {
for _, file := range files {
if file.IsInvalid() {
continue
}
objType := "file"
action := "modified"
switch {
case file.IsDeleted():
action = "deleted"
// If our local vector is version 1 AND it is the only version
// vector so far seen for this file then it is a new file. Else if
// it is > 1 it's not new, and if it is 1 but another shortId
// version vector exists then it is new for us but created elsewhere
// so the file is still not new but modified by us. Only if it is
// truly new do we change this to 'added', else we leave it as
// 'modified'.
case len(file.Version.Counters) == 1 && file.Version.Counters[0].Value == 1:
action = "added"
}
if file.IsSymlink() {
objType = "symlink"
} else if file.IsDirectory() {
objType = "dir"
}
// Two different events can be fired here based on what EventType is passed into function
events.Default.Log(typeOfEvent, map[string]string{
"folder": folderCfg.ID,
"folderID": folderCfg.ID, // incorrect, deprecated, kept for historical compliance
"label": folderCfg.Label,
"action": action,
"type": objType,
"path": filepath.FromSlash(file.Name),
"modifiedBy": file.ModifiedBy.String(),
})
}
}
func (m *model) requestGlobal(deviceID protocol.DeviceID, folder, name string, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) {
m.pmut.RLock()
nc, ok := m.conn[deviceID]
@ -2276,36 +2183,30 @@ func (m *model) Override(folder string) {
// Grab the runner and the file set.
m.fmut.RLock()
fs, fsOK := m.folderFiles[folder]
runner, runnerOK := m.folderRunners[folder]
runner, ok := m.folderRunners[folder]
m.fmut.RUnlock()
if !fsOK || !runnerOK {
if !ok {
return
}
// Run the override, taking updates as if they came from scanning.
runner.Override(fs, func(files []protocol.FileInfo) {
m.updateLocalsFromScanning(folder, files)
})
runner.Override()
}
func (m *model) Revert(folder string) {
// Grab the runner and the file set.
m.fmut.RLock()
fs, fsOK := m.folderFiles[folder]
runner, runnerOK := m.folderRunners[folder]
runner, ok := m.folderRunners[folder]
m.fmut.RUnlock()
if !fsOK || !runnerOK {
if !ok {
return
}
// Run the revert, taking updates as if they came from scanning.
runner.Revert(fs, func(files []protocol.FileInfo) {
m.updateLocalsFromScanning(folder, files)
})
runner.Revert()
}
// CurrentSequence returns the change version for the given folder.

3
lib/model/requests_test.go
View File

@ -448,7 +448,8 @@ func TestIssue4841(t *testing.T) {
fc.mut.Unlock()
// Setup file from remote that was ignored locally
m.updateLocals(defaultFolderConfig.ID, []protocol.FileInfo{{
folder := m.folderRunners[defaultFolderConfig.ID].(*sendReceiveFolder)
folder.updateLocals([]protocol.FileInfo{{
Name: "foo",
Type: protocol.FileInfoTypeFile,
LocalFlags: protocol.FlagLocalIgnored,