From 986b15573a66f95beae53b10371ba50ab147124a Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Mon, 6 Jan 2014 21:17:18 +0100 Subject: [PATCH] Ignore files matching patterns in .stignore (fixes #7) --- README.md | 9 +++++++ main.go | 2 +- model/model_test.go | 14 +++++----- model/walk.go | 66 ++++++++++++++++++++++++++++++++++++--------- model/walk_test.go | 62 +++++++++++++++++++++++++++++++++++++++++- 5 files changed, 132 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index e7e4a3981..ad1ccd0f4 100644 --- a/README.md +++ b/README.md @@ -186,6 +186,15 @@ $ syncthing --gui 127.0.0.1:8080 You then point your browser to the given address. +Excluding Files +--------------- + +syncthing looks for files named `.stignore` while walking the +repository. The file is expected to contain glob patterns of file names +to ignore. Patterns are matched on file name only and apply to files in +the same directory as the `.stignore` file and in directories lower down +in the hierarchy. + License ======= diff --git a/main.go b/main.go index 6778e6ff0..194d03173 100644 --- a/main.go +++ b/main.go @@ -333,7 +333,7 @@ func connect(myID string, addr string, nodeAddrs map[string][]string, m *model.M } func updateLocalModel(m *model.Model) { - files := m.Walk(!opts.NoSymlinks) + files := m.FilteredWalk(!opts.NoSymlinks) m.ReplaceLocal(files) saveIndex(m) } diff --git a/model/model_test.go b/model/model_test.go index 797ca76b8..31cd2857f 100644 --- a/model/model_test.go +++ b/model/model_test.go @@ -58,7 +58,7 @@ func init() { func TestUpdateLocal(t *testing.T) { m := NewModel("testdata") - fs := m.Walk(false) + fs, _ := m.Walk(false) m.ReplaceLocal(fs) if len(m.need) > 0 { @@ -100,7 +100,7 @@ func TestUpdateLocal(t *testing.T) { func TestRemoteUpdateExisting(t *testing.T) { m := NewModel("testdata") - fs := m.Walk(false) + fs, _ := m.Walk(false) m.ReplaceLocal(fs) newFile := protocol.FileInfo{ @@ -117,7 +117,7 @@ func TestRemoteUpdateExisting(t *testing.T) { func TestRemoteAddNew(t *testing.T) { m := NewModel("testdata") - fs := m.Walk(false) + fs, _ := m.Walk(false) m.ReplaceLocal(fs) newFile := protocol.FileInfo{ @@ -134,7 +134,7 @@ func TestRemoteAddNew(t *testing.T) { func TestRemoteUpdateOld(t *testing.T) { m := NewModel("testdata") - fs := m.Walk(false) + fs, _ := m.Walk(false) m.ReplaceLocal(fs) oldTimeStamp := int64(1234) @@ -152,7 +152,7 @@ func TestRemoteUpdateOld(t *testing.T) { func TestRemoteIndexUpdate(t *testing.T) { m := NewModel("testdata") - fs := m.Walk(false) + fs, _ := m.Walk(false) m.ReplaceLocal(fs) foo := protocol.FileInfo{ @@ -185,7 +185,7 @@ func TestRemoteIndexUpdate(t *testing.T) { func TestDelete(t *testing.T) { m := NewModel("testdata") - fs := m.Walk(false) + fs, _ := m.Walk(false) m.ReplaceLocal(fs) if l1, l2 := len(m.local), len(fs); l1 != l2 { @@ -275,7 +275,7 @@ func TestDelete(t *testing.T) { func TestForgetNode(t *testing.T) { m := NewModel("testdata") - fs := m.Walk(false) + fs, _ := m.Walk(false) m.ReplaceLocal(fs) if l1, l2 := len(m.local), len(fs); l1 != l2 { diff --git a/model/walk.go b/model/walk.go index 7377c62f7..7c19c3236 100644 --- a/model/walk.go +++ b/model/walk.go @@ -1,7 +1,9 @@ package model import ( + "bytes" "fmt" + "io/ioutil" "log" "os" "path" @@ -35,7 +37,7 @@ func tempName(name string, modified int64) string { return path.Join(tdir, tname) } -func (m *Model) genWalker(res *[]File) filepath.WalkFunc { +func (m *Model) genWalker(res *[]File, ign map[string][]string) filepath.WalkFunc { return func(p string, info os.FileInfo, err error) error { if err != nil { return nil @@ -45,12 +47,26 @@ func (m *Model) genWalker(res *[]File) filepath.WalkFunc { return nil } - if info.Mode()&os.ModeType == 0 { - rn, err := filepath.Rel(m.dir, p) - if err != nil { - return nil - } + rn, err := filepath.Rel(m.dir, p) + if err != nil { + return nil + } + if pn, sn := path.Split(rn); sn == ".stignore" { + pn := strings.Trim(pn, "/") + bs, _ := ioutil.ReadFile(p) + lines := bytes.Split(bs, []byte("\n")) + var patterns []string + for _, line := range lines { + if len(line) > 0 { + patterns = append(patterns, string(line)) + } + } + ign[pn] = patterns + return nil + } + + if info.Mode()&os.ModeType == 0 { fi, err := os.Stat(p) if err != nil { return nil @@ -94,21 +110,21 @@ func (m *Model) genWalker(res *[]File) filepath.WalkFunc { // Walk returns the list of files found in the local repository by scanning the // file system. Files are blockwise hashed. -func (m *Model) Walk(followSymlinks bool) []File { - var files []File - fn := m.genWalker(&files) +func (m *Model) Walk(followSymlinks bool) (files []File, ignore map[string][]string) { + ignore = make(map[string][]string) + fn := m.genWalker(&files, ignore) filepath.Walk(m.dir, fn) if followSymlinks { d, err := os.Open(m.dir) if err != nil { - return files + return } defer d.Close() fis, err := d.Readdir(-1) if err != nil { - return files + return } for _, fi := range fis { @@ -118,7 +134,15 @@ func (m *Model) Walk(followSymlinks bool) []File { } } - return files + return +} + +// Walk returns the list of files found in the local repository by scanning the +// file system. Files are blockwise hashed. Patterns marked in .stignore files +// are removed from the results. +func (m *Model) FilteredWalk(followSymlinks bool) []File { + var files, ignored = m.Walk(followSymlinks) + return ignoreFilter(ignored, files) } func (m *Model) cleanTempFile(path string, info os.FileInfo, err error) error { @@ -137,3 +161,21 @@ func (m *Model) cleanTempFile(path string, info os.FileInfo, err error) error { func (m *Model) cleanTempFiles() { filepath.Walk(m.dir, m.cleanTempFile) } + +func ignoreFilter(patterns map[string][]string, files []File) (filtered []File) { +nextFile: + for _, f := range files { + first, last := path.Split(f.Name) + for prefix, pats := range patterns { + if len(prefix) == 0 || prefix == first || strings.HasPrefix(first, prefix+"/") { + for _, pattern := range pats { + if match, _ := path.Match(pattern, last); match { + continue nextFile + } + } + } + } + filtered = append(filtered, f) + } + return filtered +} diff --git a/model/walk_test.go b/model/walk_test.go index b48bd432c..36929be43 100644 --- a/model/walk_test.go +++ b/model/walk_test.go @@ -2,6 +2,7 @@ package model import ( "fmt" + "reflect" "testing" "time" ) @@ -16,9 +17,13 @@ var testdata = []struct { {"foo", 7, "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f"}, } +var correctIgnores = map[string][]string{ + "": {".*", "quux"}, +} + func TestWalk(t *testing.T) { m := NewModel("testdata") - files := m.Walk(false) + files, ignores := m.Walk(false) if l1, l2 := len(files), len(testdata); l1 != l2 { t.Fatalf("Incorrect number of walked files %d != %d", l1, l2) @@ -39,4 +44,59 @@ func TestWalk(t *testing.T) { t.Errorf("Unrealistic modtime %d for test %d", mt, i) } } + + if !reflect.DeepEqual(ignores, correctIgnores) { + t.Errorf("Incorrect ignores\n %v\n %v", correctIgnores, ignores) + } +} + +func TestFilteredWalk(t *testing.T) { + m := NewModel("testdata") + files := m.FilteredWalk(false) + + if len(files) != 2 { + t.Fatalf("Incorrect number of walked filtered files %d != 2", len(files)) + } + if files[0].Name != "bar" { + t.Error("Incorrect first file", files[0]) + } + if files[1].Name != "foo" { + t.Error("Incorrect second file", files[1]) + } +} + +func TestIgnore(t *testing.T) { + var patterns = map[string][]string{ + "": {"t2"}, + "foo": {"bar", "z*"}, + "foo/baz": {"quux", ".*"}, + } + var files = []File{ + {Name: "foo/bar"}, + {Name: "foo/quux"}, + {Name: "foo/zuux"}, + {Name: "foo/qzuux"}, + {Name: "foo/baz/t1"}, + {Name: "foo/baz/t2"}, + {Name: "foo/baz/bar"}, + {Name: "foo/baz/quuxa"}, + {Name: "foo/baz/aquux"}, + {Name: "foo/baz/.quux"}, + {Name: "foo/baz/zquux"}, + {Name: "foo/baz/quux"}, + {Name: "foo/bazz/quux"}, + } + var remaining = []File{ + {Name: "foo/quux"}, + {Name: "foo/qzuux"}, + {Name: "foo/baz/t1"}, + {Name: "foo/baz/quuxa"}, + {Name: "foo/baz/aquux"}, + {Name: "foo/bazz/quux"}, + } + + var filtered = ignoreFilter(patterns, files) + if !reflect.DeepEqual(filtered, remaining) { + t.Errorf("Filtering mismatch\n %v\n %v", remaining, filtered) + } }