Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Reimplement quick startup scan

This commit is contained in:
Jakob Borg 2014-03-16 08:14:55 +01:00
parent d2d32f26c7
commit 2df78a9313
5 changed files with 63 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cmd/syncthing/main.go
View File

@ -239,8 +239,9 @@ func main() {
IgnoreFile: ".stignore",
FollowSymlinks: cfg.Options.FollowSymlinks,
BlockSize: BlockSize,
Suppressor: sup,
TempNamer: defTempNamer,
Suppressor: sup,
CurrentFiler: m,
}
updateLocalModel(m, w)

31
cmd/syncthing/model.go
View File

@ -397,7 +397,7 @@ func (m *Model) Request(nodeID, repo, name string, offset int64, size int) ([]by
warnf("SECURITY (nonexistent file) REQ(in): %s: %q o=%d s=%d", nodeID, name, offset, size)
return nil, ErrNoSuchFile
}
if lf.Flags&protocol.FlagInvalid != 0 {
if lf.Suppressed {
return nil, ErrInvalid
}
@ -480,6 +480,14 @@ func (m *Model) SeedLocal(fs []protocol.FileInfo) {
m.recomputeNeedForGlobal()
}
// Implements scanner.CurrentFiler
func (m *Model) CurrentFile(file string) scanner.File {
m.lmut.RLock()
f := m.local[file]
m.lmut.RUnlock()
return f
}
// ConnectedTo returns true if we are connected to the named node.
func (m *Model) ConnectedTo(nodeID string) bool {
m.pmut.RLock()
@ -810,7 +818,7 @@ func (m *Model) recomputeNeedForFile(gf scanner.File, toAdd []addOrder, toDelete
m.lmut.RUnlock()
if !ok || gf.NewerThan(lf) {
if gf.Flags&protocol.FlagInvalid != 0 {
if gf.Suppressed {
// Never attempt to sync invalid files
return toAdd, toDelete
}
@ -889,12 +897,13 @@ func fileFromFileInfo(f protocol.FileInfo) scanner.File {
offset += int64(b.Size)
}
return scanner.File{
Name: f.Name,
Size: offset,
Flags: f.Flags,
Modified: f.Modified,
Version: f.Version,
Blocks: blocks,
Name: f.Name,
Size: offset,
Flags: f.Flags &^ protocol.FlagInvalid,
Modified: f.Modified,
Version: f.Version,
Blocks: blocks,
Suppressed: f.Flags&protocol.FlagInvalid != 0,
}
}
@ -906,11 +915,15 @@ func fileInfoFromFile(f scanner.File) protocol.FileInfo {
Hash: b.Hash,
}
}
return protocol.FileInfo{
pf := protocol.FileInfo{
Name: f.Name,
Flags: f.Flags,
Modified: f.Modified,
Version: f.Version,
Blocks: blocks,
}
if f.Suppressed {
pf.Flags |= protocol.FlagInvalid
}
return pf
}

4
protocol/protocol.go
View File

@ -26,8 +26,8 @@ const (
)
const (
FlagDeleted = 1 << 12
FlagInvalid = 1 << 13
FlagDeleted uint32 = 1 << 12
FlagInvalid = 1 << 13
)
var (

13
scanner/file.go
View File

@ -3,12 +3,13 @@ package scanner
import "fmt"
type File struct {
Name string
Flags uint32
Modified int64
Version uint32
Size int64
Blocks []Block
Name string
Flags uint32
Modified int64
Version uint32
Size int64
Blocks []Block
Suppressed bool
}
func (f File) String() string {

66
scanner/walk.go
View File

@ -9,8 +9,6 @@ import (
"path/filepath"
"strings"
"time"
"github.com/calmh/syncthing/protocol"
)
type Walker struct {
@ -25,10 +23,13 @@ type Walker struct {
IgnoreFile string
// If TempNamer is not nil, it is used to ignore tempory files when walking.
TempNamer TempNamer
// If CurrentFiler is not nil, it is queried for the current file before rescanning.
CurrentFiler CurrentFiler
// If Suppressor is not nil, it is queried for supression of modified files.
// Suppressed files will be returned with empty metadata and the Suppressed flag set.
// Requires CurrentFiler to be set.
Suppressor Suppressor
previous map[string]File // file name -> last seen file state
suppressed map[string]bool // file name -> suppression status
}
@ -44,6 +45,11 @@ type Suppressor interface {
Suppress(name string, fi os.FileInfo) bool
}
type CurrentFiler interface {
// CurrentFile returns the file as seen at last scan.
CurrentFile(name string) File
}
// Walk returns the list of files found in the local repository by scanning the
// file system. Files are blockwise hashed.
func (w *Walker) Walk() (files []File, ignore map[string][]string) {
@ -95,8 +101,7 @@ func (w *Walker) CleanTempFiles() {
}
func (w *Walker) lazyInit() {
if w.previous == nil {
w.previous = make(map[string]File)
if w.suppressed == nil {
w.suppressed = make(map[string]bool)
}
}
@ -171,40 +176,30 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
}
if info.Mode()&os.ModeType == 0 {
modified := info.ModTime().Unix()
pf := w.previous[rn]
if pf.Modified == modified {
if nf := uint32(info.Mode()); nf != pf.Flags {
if w.CurrentFiler != nil {
cf := w.CurrentFiler.CurrentFile(rn)
if cf.Modified == info.ModTime().Unix() {
if debug {
dlog.Println("new flags:", rn)
dlog.Println("unchanged:", rn)
}
pf.Flags = nf
pf.Version++
w.previous[rn] = pf
} else if debug {
dlog.Println("unchanged:", rn)
*res = append(*res, cf)
return nil
}
*res = append(*res, pf)
return nil
}
if w.Suppressor != nil && w.Suppressor.Suppress(rn, info) {
if debug {
dlog.Println("suppressed:", rn)
if w.Suppressor != nil && w.Suppressor.Suppress(rn, info) {
if debug {
dlog.Println("suppressed:", rn)
}
if !w.suppressed[rn] {
w.suppressed[rn] = true
log.Printf("INFO: Changes to %q are being temporarily suppressed because it changes too frequently.", p)
}
cf.Suppressed = true
*res = append(*res, cf)
} else if w.suppressed[rn] {
log.Printf("INFO: Changes to %q are no longer suppressed.", p)
delete(w.suppressed, rn)
}
if !w.suppressed[rn] {
w.suppressed[rn] = true
log.Printf("INFO: Changes to %q are being temporarily suppressed because it changes too frequently.", p)
}
f := pf
f.Flags = protocol.FlagInvalid
f.Blocks = nil
*res = append(*res, f)
return nil
} else if w.suppressed[rn] {
log.Printf("INFO: Changes to %q are no longer suppressed.", p)
delete(w.suppressed, rn)
}
fd, err := os.Open(p)
@ -232,10 +227,9 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
Name: rn,
Size: info.Size(),
Flags: uint32(info.Mode()),
Modified: modified,
Modified: info.ModTime().Unix(),
Blocks: blocks,
}
w.previous[rn] = f
*res = append(*res, f)
}