Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/db, lib/model: Remove filesystem state from FileSet (fixes #7850) (#8151)

This commit is contained in:
Simon Frei 2022-01-31 10:12:52 +01:00 committed by GitHub
parent d1e81a0acf
commit 635085d139
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 70 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
lib/db/benchmark_test.go
View File

@ -11,7 +11,6 @@ import (
"testing" "testing"
"github.com/syncthing/syncthing/lib/db" "github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/protocol"
) )
@ -47,7 +46,7 @@ func getBenchFileSet(b testing.TB) (*db.Lowlevel, *db.FileSet) {
lazyInitBenchFiles() lazyInitBenchFiles()
ldb := newLowlevelMemory(b) ldb := newLowlevelMemory(b)
benchS := newFileSet(b, "test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) benchS := newFileSet(b, "test)", ldb)
replace(benchS, remoteDevice0, files) replace(benchS, remoteDevice0, files)
replace(benchS, protocol.LocalDeviceID, firstHalf) replace(benchS, protocol.LocalDeviceID, firstHalf)
@ -60,7 +59,7 @@ func BenchmarkReplaceAll(b *testing.B) {
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
m := newFileSet(b, "test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) m := newFileSet(b, "test)", ldb)
replace(m, protocol.LocalDeviceID, files) replace(m, protocol.LocalDeviceID, files)
} }
@ -202,7 +201,7 @@ func BenchmarkNeedHalf(b *testing.B) {
func BenchmarkNeedHalfRemote(b *testing.B) { func BenchmarkNeedHalfRemote(b *testing.B) {
ldb := newLowlevelMemory(b) ldb := newLowlevelMemory(b)
defer ldb.Close() defer ldb.Close()
fset := newFileSet(b, "test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) fset := newFileSet(b, "test)", ldb)
replace(fset, remoteDevice0, firstHalf) replace(fset, remoteDevice0, firstHalf)
replace(fset, protocol.LocalDeviceID, files) replace(fset, protocol.LocalDeviceID, files)

17
lib/db/db_test.go
View File

@ -14,7 +14,6 @@ import (
"github.com/syncthing/syncthing/lib/db/backend" "github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/events" "github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/protocol"
) )
@ -42,11 +41,11 @@ func TestIgnoredFiles(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
fs := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db) fs := newFileSet(t, "test", db)
// The contents of the database are like this: // The contents of the database are like this:
// //
// fs := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db) // fs := newFileSet(t, "test", db)
// fs.Update(protocol.LocalDeviceID, []protocol.FileInfo{ // fs.Update(protocol.LocalDeviceID, []protocol.FileInfo{
// { // invalid (ignored) file // { // invalid (ignored) file
// Name: "foo", // Name: "foo",
@ -498,7 +497,7 @@ func TestCheckGlobals(t *testing.T) {
db := newLowlevelMemory(t) db := newLowlevelMemory(t)
defer db.Close() defer db.Close()
fs := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), db) fs := newFileSet(t, "test", db)
// Add any file // Add any file
name := "foo" name := "foo"
@ -854,7 +853,7 @@ func TestCheckLocalNeed(t *testing.T) {
defer db.Close() defer db.Close()
folderStr := "test" folderStr := "test"
fs := newFileSet(t, folderStr, fs.NewFilesystem(fs.FilesystemTypeFake, ""), db) fs := newFileSet(t, folderStr, db)
// Add files such that we are in sync for a and b, and need c and d. // Add files such that we are in sync for a and b, and need c and d.
files := []protocol.FileInfo{ files := []protocol.FileInfo{
@ -929,9 +928,8 @@ func TestDuplicateNeedCount(t *testing.T) {
defer db.Close() defer db.Close()
folder := "test" folder := "test"
testFs := fs.NewFilesystem(fs.FilesystemTypeFake, "")
fs := newFileSet(t, folder, testFs, db) fs := newFileSet(t, folder, db)
files := []protocol.FileInfo{{Name: "foo", Version: protocol.Vector{}.Update(myID), Sequence: 1}} files := []protocol.FileInfo{{Name: "foo", Version: protocol.Vector{}.Update(myID), Sequence: 1}}
fs.Update(protocol.LocalDeviceID, files) fs.Update(protocol.LocalDeviceID, files)
files[0].Version = files[0].Version.Update(remoteDevice0.Short()) files[0].Version = files[0].Version.Update(remoteDevice0.Short())
@ -939,7 +937,7 @@ func TestDuplicateNeedCount(t *testing.T) {
db.checkRepair() db.checkRepair()
fs = newFileSet(t, folder, testFs, db) fs = newFileSet(t, folder, db)
found := false found := false
for _, c := range fs.meta.counts.Counts { for _, c := range fs.meta.counts.Counts {
if bytes.Equal(protocol.LocalDeviceID[:], c.DeviceID) && c.LocalFlags == needFlag { if bytes.Equal(protocol.LocalDeviceID[:], c.DeviceID) && c.LocalFlags == needFlag {
@ -959,9 +957,8 @@ func TestNeedAfterDropGlobal(t *testing.T) {
defer db.Close() defer db.Close()
folder := "test" folder := "test"
testFs := fs.NewFilesystem(fs.FilesystemTypeFake, "")
fs := newFileSet(t, folder, testFs, db) fs := newFileSet(t, folder, db)
// Initial: // Initial:
// Three devices and a file "test": local has Version 1, remoteDevice0 // Three devices and a file "test": local has Version 1, remoteDevice0

5
lib/db/meta_test.go
View File

@ -12,7 +12,6 @@ import (
"testing" "testing"
"github.com/syncthing/syncthing/lib/events" "github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/protocol"
) )
@ -109,7 +108,7 @@ func TestRecalcMeta(t *testing.T) {
defer ldb.Close() defer ldb.Close()
// Add some files // Add some files
s1 := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeFake, "fake"), ldb) s1 := newFileSet(t, "test", ldb)
files := []protocol.FileInfo{ files := []protocol.FileInfo{
{Name: "a", Size: 1000}, {Name: "a", Size: 1000},
{Name: "b", Size: 2000}, {Name: "b", Size: 2000},
@ -161,7 +160,7 @@ func TestRecalcMeta(t *testing.T) {
} }
// Create a new fileset, which will realize the inconsistency and recalculate // Create a new fileset, which will realize the inconsistency and recalculate
s2 := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeFake, "fake"), ldb) s2 := newFileSet(t, "test", ldb)
// Verify local/global size // Verify local/global size
snap = snapshot(t, s2) snap = snapshot(t, s2)

8
lib/db/set.go
View File

@ -24,7 +24,6 @@ import (
type FileSet struct { type FileSet struct {
folder string folder string
fs fs.Filesystem
db *Lowlevel db *Lowlevel
meta *metadataTracker meta *metadataTracker
@ -36,7 +35,7 @@ type FileSet struct {
// continue iteration, false to stop. // continue iteration, false to stop.
type Iterator func(f protocol.FileIntf) bool type Iterator func(f protocol.FileIntf) bool
func NewFileSet(folder string, fs fs.Filesystem, db *Lowlevel) (*FileSet, error) { func NewFileSet(folder string, db *Lowlevel) (*FileSet, error) {
select { select {
case <-db.oneFileSetCreated: case <-db.oneFileSetCreated:
default: default:
@ -49,7 +48,6 @@ func NewFileSet(folder string, fs fs.Filesystem, db *Lowlevel) (*FileSet, error)
} }
s := &FileSet{ s := &FileSet{
folder: folder, folder: folder,
fs: fs,
db: db, db: db,
meta: meta, meta: meta,
updateMutex: sync.NewMutex(), updateMutex: sync.NewMutex(),
@ -405,7 +403,7 @@ func (s *FileSet) SetIndexID(device protocol.DeviceID, id protocol.IndexID) {
} }
} }
func (s *FileSet) MtimeFS() fs.Filesystem { func (s *FileSet) MtimeFS(filesystem fs.Filesystem) fs.Filesystem {
opStr := fmt.Sprintf("%s MtimeFS()", s.folder) opStr := fmt.Sprintf("%s MtimeFS()", s.folder)
l.Debugf(opStr) l.Debugf(opStr)
prefix, err := s.db.keyer.GenerateMtimesKey(nil, []byte(s.folder)) prefix, err := s.db.keyer.GenerateMtimesKey(nil, []byte(s.folder))
@ -415,7 +413,7 @@ func (s *FileSet) MtimeFS() fs.Filesystem {
fatalError(err, opStr, s.db) fatalError(err, opStr, s.db)
} }
kv := NewNamespacedKV(s.db, string(prefix)) kv := NewNamespacedKV(s.db, string(prefix))
return fs.NewMtimeFS(s.fs, kv) return fs.NewMtimeFS(filesystem, kv)
} }
func (s *FileSet) ListDevices() []protocol.DeviceID { func (s *FileSet) ListDevices() []protocol.DeviceID {

78
lib/db/set_test.go
View File

@ -19,7 +19,6 @@ import (
"github.com/syncthing/syncthing/lib/db" "github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/db/backend" "github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/events" "github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/protocol"
) )
@ -146,7 +145,7 @@ func TestGlobalSet(t *testing.T) {
ldb := newLowlevelMemory(t) ldb := newLowlevelMemory(t)
defer ldb.Close() defer ldb.Close()
m := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) m := newFileSet(t, "test", ldb)
local0 := fileList{ local0 := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)}, protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
@ -452,7 +451,7 @@ func TestNeedWithInvalid(t *testing.T) {
ldb := newLowlevelMemory(t) ldb := newLowlevelMemory(t)
defer ldb.Close() defer ldb.Close()
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) s := newFileSet(t, "test", ldb)
localHave := fileList{ localHave := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)}, protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
@ -493,7 +492,7 @@ func TestUpdateToInvalid(t *testing.T) {
defer ldb.Close() defer ldb.Close()
folder := "test" folder := "test"
s := newFileSet(t, folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) s := newFileSet(t, folder, ldb)
f := db.NewBlockFinder(ldb) f := db.NewBlockFinder(ldb)
localHave := fileList{ localHave := fileList{
@ -549,7 +548,7 @@ func TestInvalidAvailability(t *testing.T) {
ldb := newLowlevelMemory(t) ldb := newLowlevelMemory(t)
defer ldb.Close() defer ldb.Close()
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) s := newFileSet(t, "test", ldb)
remote0Have := fileList{ remote0Have := fileList{
protocol.FileInfo{Name: "both", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)}, protocol.FileInfo{Name: "both", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
@ -591,7 +590,7 @@ func TestGlobalReset(t *testing.T) {
ldb := newLowlevelMemory(t) ldb := newLowlevelMemory(t)
defer ldb.Close() defer ldb.Close()
m := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) m := newFileSet(t, "test", ldb)
local := []protocol.FileInfo{ local := []protocol.FileInfo{
{Name: "a", Sequence: 1, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}}, {Name: "a", Sequence: 1, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
@ -630,7 +629,7 @@ func TestNeed(t *testing.T) {
ldb := newLowlevelMemory(t) ldb := newLowlevelMemory(t)
defer ldb.Close() defer ldb.Close()
m := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) m := newFileSet(t, "test", ldb)
local := []protocol.FileInfo{ local := []protocol.FileInfo{
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}}, {Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
@ -671,7 +670,7 @@ func TestSequence(t *testing.T) {
ldb := newLowlevelMemory(t) ldb := newLowlevelMemory(t)
defer ldb.Close() defer ldb.Close()
m := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) m := newFileSet(t, "test", ldb)
local1 := []protocol.FileInfo{ local1 := []protocol.FileInfo{
{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}}, {Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
@ -702,7 +701,7 @@ func TestListDropFolder(t *testing.T) {
ldb := newLowlevelMemory(t) ldb := newLowlevelMemory(t)
defer ldb.Close() defer ldb.Close()
s0 := newFileSet(t, "test0", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) s0 := newFileSet(t, "test0", ldb)
local1 := []protocol.FileInfo{ local1 := []protocol.FileInfo{
{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}}, {Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}}, {Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
@ -710,7 +709,7 @@ func TestListDropFolder(t *testing.T) {
} }
replace(s0, protocol.LocalDeviceID, local1) replace(s0, protocol.LocalDeviceID, local1)
s1 := newFileSet(t, "test1", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) s1 := newFileSet(t, "test1", ldb)
local2 := []protocol.FileInfo{ local2 := []protocol.FileInfo{
{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}}, {Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}}, {Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
@ -753,7 +752,7 @@ func TestGlobalNeedWithInvalid(t *testing.T) {
ldb := newLowlevelMemory(t) ldb := newLowlevelMemory(t)
defer ldb.Close() defer ldb.Close()
s := newFileSet(t, "test1", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) s := newFileSet(t, "test1", ldb)
rem0 := fileList{ rem0 := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)}, protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
@ -796,7 +795,7 @@ func TestLongPath(t *testing.T) {
ldb := newLowlevelMemory(t) ldb := newLowlevelMemory(t)
defer ldb.Close() defer ldb.Close()
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) s := newFileSet(t, "test", ldb)
var b bytes.Buffer var b bytes.Buffer
for i := 0; i < 100; i++ { for i := 0; i < 100; i++ {
@ -840,7 +839,7 @@ func BenchmarkUpdateOneFile(b *testing.B) {
os.RemoveAll("testdata/benchmarkupdate.db") os.RemoveAll("testdata/benchmarkupdate.db")
}() }()
m := newFileSet(b, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) m := newFileSet(b, "test", ldb)
replace(m, protocol.LocalDeviceID, local0) replace(m, protocol.LocalDeviceID, local0)
l := local0[4:5] l := local0[4:5]
@ -856,7 +855,7 @@ func TestIndexID(t *testing.T) {
ldb := newLowlevelMemory(t) ldb := newLowlevelMemory(t)
defer ldb.Close() defer ldb.Close()
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) s := newFileSet(t, "test", ldb)
// The Index ID for some random device is zero by default. // The Index ID for some random device is zero by default.
id := s.IndexID(remoteDevice0) id := s.IndexID(remoteDevice0)
@ -888,7 +887,7 @@ func TestIndexID(t *testing.T) {
func TestDropFiles(t *testing.T) { func TestDropFiles(t *testing.T) {
ldb := newLowlevelMemory(t) ldb := newLowlevelMemory(t)
m := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) m := newFileSet(t, "test", ldb)
local0 := fileList{ local0 := fileList{
protocol.FileInfo{Name: "a", Sequence: 1, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)}, protocol.FileInfo{Name: "a", Sequence: 1, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
@ -952,7 +951,7 @@ func TestIssue4701(t *testing.T) {
ldb := newLowlevelMemory(t) ldb := newLowlevelMemory(t)
defer ldb.Close() defer ldb.Close()
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) s := newFileSet(t, "test", ldb)
localHave := fileList{ localHave := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}}, protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
@ -995,7 +994,7 @@ func TestWithHaveSequence(t *testing.T) {
defer ldb.Close() defer ldb.Close()
folder := "test" folder := "test"
s := newFileSet(t, folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) s := newFileSet(t, folder, ldb)
// The files must not be in alphabetical order // The files must not be in alphabetical order
localHave := fileList{ localHave := fileList{
@ -1033,7 +1032,7 @@ func TestStressWithHaveSequence(t *testing.T) {
defer ldb.Close() defer ldb.Close()
folder := "test" folder := "test"
s := newFileSet(t, folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) s := newFileSet(t, folder, ldb)
var localHave []protocol.FileInfo var localHave []protocol.FileInfo
for i := 0; i < 100; i++ { for i := 0; i < 100; i++ {
@ -1078,7 +1077,7 @@ func TestIssue4925(t *testing.T) {
defer ldb.Close() defer ldb.Close()
folder := "test" folder := "test"
s := newFileSet(t, folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) s := newFileSet(t, folder, ldb)
localHave := fileList{ localHave := fileList{
protocol.FileInfo{Name: "dir"}, protocol.FileInfo{Name: "dir"},
@ -1106,7 +1105,7 @@ func TestMoveGlobalBack(t *testing.T) {
folder := "test" folder := "test"
file := "foo" file := "foo"
s := newFileSet(t, folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) s := newFileSet(t, folder, ldb)
localHave := fileList{{Name: file, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}, Blocks: genBlocks(1), ModifiedS: 10, Size: 1}} localHave := fileList{{Name: file, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}, Blocks: genBlocks(1), ModifiedS: 10, Size: 1}}
remote0Have := fileList{{Name: file, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}, {ID: remoteDevice0.Short(), Value: 1}}}, Blocks: genBlocks(2), ModifiedS: 0, Size: 2}} remote0Have := fileList{{Name: file, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}, {ID: remoteDevice0.Short(), Value: 1}}}, Blocks: genBlocks(2), ModifiedS: 0, Size: 2}}
@ -1175,7 +1174,7 @@ func TestIssue5007(t *testing.T) {
folder := "test" folder := "test"
file := "foo" file := "foo"
s := newFileSet(t, folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) s := newFileSet(t, folder, ldb)
fs := fileList{{Name: file, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}}} fs := fileList{{Name: file, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}}}
@ -1205,7 +1204,7 @@ func TestNeedDeleted(t *testing.T) {
folder := "test" folder := "test"
file := "foo" file := "foo"
s := newFileSet(t, folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) s := newFileSet(t, folder, ldb)
fs := fileList{{Name: file, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}, Deleted: true}} fs := fileList{{Name: file, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}, Deleted: true}}
@ -1242,7 +1241,7 @@ func TestReceiveOnlyAccounting(t *testing.T) {
defer ldb.Close() defer ldb.Close()
folder := "test" folder := "test"
s := newFileSet(t, folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) s := newFileSet(t, folder, ldb)
local := protocol.DeviceID{1} local := protocol.DeviceID{1}
remote := protocol.DeviceID{2} remote := protocol.DeviceID{2}
@ -1348,7 +1347,7 @@ func TestNeedAfterUnignore(t *testing.T) {
folder := "test" folder := "test"
file := "foo" file := "foo"
s := newFileSet(t, folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) s := newFileSet(t, folder, ldb)
remID := remoteDevice0.Short() remID := remoteDevice0.Short()
@ -1379,7 +1378,7 @@ func TestRemoteInvalidNotAccounted(t *testing.T) {
ldb := newLowlevelMemory(t) ldb := newLowlevelMemory(t)
defer ldb.Close() defer ldb.Close()
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) s := newFileSet(t, "test", ldb)
files := []protocol.FileInfo{ files := []protocol.FileInfo{
{Name: "a", Size: 1234, Sequence: 42, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}}, // valid, should count {Name: "a", Size: 1234, Sequence: 42, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}}, // valid, should count
@ -1400,7 +1399,7 @@ func TestNeedWithNewerInvalid(t *testing.T) {
ldb := newLowlevelMemory(t) ldb := newLowlevelMemory(t)
defer ldb.Close() defer ldb.Close()
s := newFileSet(t, "default", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) s := newFileSet(t, "default", ldb)
rem0ID := remoteDevice0.Short() rem0ID := remoteDevice0.Short()
rem1ID := remoteDevice1.Short() rem1ID := remoteDevice1.Short()
@ -1442,7 +1441,7 @@ func TestNeedAfterDeviceRemove(t *testing.T) {
defer ldb.Close() defer ldb.Close()
file := "foo" file := "foo"
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) s := newFileSet(t, "test", ldb)
fs := fileList{{Name: file, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}}} fs := fileList{{Name: file, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}}}
@ -1469,7 +1468,7 @@ func TestCaseSensitive(t *testing.T) {
ldb := newLowlevelMemory(t) ldb := newLowlevelMemory(t)
defer ldb.Close() defer ldb.Close()
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) s := newFileSet(t, "test", ldb)
local := []protocol.FileInfo{ local := []protocol.FileInfo{
{Name: filepath.FromSlash("D1/f1"), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}}, {Name: filepath.FromSlash("D1/f1"), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
@ -1507,7 +1506,7 @@ func TestSequenceIndex(t *testing.T) {
ldb := newLowlevelMemory(t) ldb := newLowlevelMemory(t)
defer ldb.Close() defer ldb.Close()
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) s := newFileSet(t, "test", ldb)
local := []protocol.FileInfo{ local := []protocol.FileInfo{
{Name: filepath.FromSlash("banana"), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}}, {Name: filepath.FromSlash("banana"), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
@ -1603,7 +1602,7 @@ func TestIgnoreAfterReceiveOnly(t *testing.T) {
defer ldb.Close() defer ldb.Close()
file := "foo" file := "foo"
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) s := newFileSet(t, "test", ldb)
fs := fileList{{ fs := fileList{{
Name: file, Name: file,
@ -1634,7 +1633,7 @@ func TestUpdateWithOneFileTwice(t *testing.T) {
defer ldb.Close() defer ldb.Close()
file := "foo" file := "foo"
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), ldb) s := newFileSet(t, "test", ldb)
fs := fileList{{ fs := fileList{{
Name: file, Name: file,
@ -1671,7 +1670,7 @@ func TestNeedRemoteOnly(t *testing.T) {
ldb := newLowlevelMemory(t) ldb := newLowlevelMemory(t)
defer ldb.Close() defer ldb.Close()
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), ldb) s := newFileSet(t, "test", ldb)
remote0Have := fileList{ remote0Have := fileList{
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)}, protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
@ -1689,7 +1688,7 @@ func TestNeedRemoteAfterReset(t *testing.T) {
ldb := newLowlevelMemory(t) ldb := newLowlevelMemory(t)
defer ldb.Close() defer ldb.Close()
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), ldb) s := newFileSet(t, "test", ldb)
files := fileList{ files := fileList{
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)}, protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
@ -1715,7 +1714,7 @@ func TestIgnoreLocalChanged(t *testing.T) {
ldb := newLowlevelMemory(t) ldb := newLowlevelMemory(t)
defer ldb.Close() defer ldb.Close()
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), ldb) s := newFileSet(t, "test", ldb)
// Add locally changed file // Add locally changed file
files := fileList{ files := fileList{
@ -1749,7 +1748,7 @@ func TestNoIndexIDResetOnDrop(t *testing.T) {
ldb := newLowlevelMemory(t) ldb := newLowlevelMemory(t)
defer ldb.Close() defer ldb.Close()
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), ldb) s := newFileSet(t, "test", ldb)
s.SetIndexID(remoteDevice0, 1) s.SetIndexID(remoteDevice0, 1)
s.Drop(remoteDevice0) s.Drop(remoteDevice0)
@ -1772,7 +1771,7 @@ func TestConcurrentIndexID(t *testing.T) {
} }
for i := 0; i < max; i++ { for i := 0; i < max; i++ {
ldb := newLowlevelMemory(t) ldb := newLowlevelMemory(t)
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), ldb) s := newFileSet(t, "test", ldb)
go setID(s, 0) go setID(s, 0)
go setID(s, 1) go setID(s, 1)
<-done <-done
@ -1789,9 +1788,8 @@ func TestNeedRemoveLastValid(t *testing.T) {
defer db.Close() defer db.Close()
folder := "test" folder := "test"
testFs := fs.NewFilesystem(fs.FilesystemTypeFake, "")
fs := newFileSet(t, folder, testFs, db) fs := newFileSet(t, folder, db)
files := []protocol.FileInfo{ files := []protocol.FileInfo{
{Name: "foo", Version: protocol.Vector{}.Update(myID), Sequence: 1}, {Name: "foo", Version: protocol.Vector{}.Update(myID), Sequence: 1},
@ -1887,9 +1885,9 @@ func newLowlevelMemory(t testing.TB) *db.Lowlevel {
return newLowlevel(t, backend.OpenMemory()) return newLowlevel(t, backend.OpenMemory())
} }
func newFileSet(t testing.TB, folder string, fs fs.Filesystem, ll *db.Lowlevel) *db.FileSet { func newFileSet(t testing.TB, folder string, ll *db.Lowlevel) *db.FileSet {
t.Helper() t.Helper()
fset, err := db.NewFileSet(folder, fs, ll) fset, err := db.NewFileSet(folder, ll)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

6
lib/db/util_test.go
View File

@ -14,8 +14,6 @@ import (
"github.com/syncthing/syncthing/lib/db/backend" "github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/events" "github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
// "github.com/syncthing/syncthing/lib/protocol"
) )
// writeJSONS serializes the database to a JSON stream that can be checked // writeJSONS serializes the database to a JSON stream that can be checked
@ -83,9 +81,9 @@ func newLowlevelMemory(t testing.TB) *Lowlevel {
return newLowlevel(t, backend.OpenMemory()) return newLowlevel(t, backend.OpenMemory())
} }
func newFileSet(t testing.TB, folder string, fs fs.Filesystem, db *Lowlevel) *FileSet { func newFileSet(t testing.TB, folder string, db *Lowlevel) *FileSet {
t.Helper() t.Helper()
fset, err := NewFileSet(folder, fs, db) fset, err := NewFileSet(folder, db)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

2
lib/model/folder.go
View File

@ -103,7 +103,7 @@ func newFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg conf
shortID: model.shortID, shortID: model.shortID,
fset: fset, fset: fset,
ignores: ignores, ignores: ignores,
mtimefs: fset.MtimeFS(), mtimefs: fset.MtimeFS(cfg.Filesystem()),
modTimeWindow: cfg.ModTimeWindow(), modTimeWindow: cfg.ModTimeWindow(),
done: make(chan struct{}), done: make(chan struct{}),

4
lib/model/folder_sendrecv_test.go
View File

@ -805,8 +805,8 @@ func TestCopyOwner(t *testing.T) {
f.folder.FolderConfiguration = newFolderConfiguration(m.cfg, f.ID, f.Label, fs.FilesystemTypeFake, "/TestCopyOwner") f.folder.FolderConfiguration = newFolderConfiguration(m.cfg, f.ID, f.Label, fs.FilesystemTypeFake, "/TestCopyOwner")
f.folder.FolderConfiguration.CopyOwnershipFromParent = true f.folder.FolderConfiguration.CopyOwnershipFromParent = true
f.fset = newFileSet(t, f.ID, f.Filesystem(), m.db) f.fset = newFileSet(t, f.ID, m.db)
f.mtimefs = f.fset.MtimeFS() f.mtimefs = f.fset.MtimeFS(f.Filesystem())
// Create a parent dir with a certain owner/group. // Create a parent dir with a certain owner/group.

7
lib/model/model.go
View File

@ -553,7 +553,7 @@ func (m *model) restartFolder(from, to config.FolderConfiguration, cacheIgnoredF
// locking, but it's unsafe to create fset:s concurrently so // locking, but it's unsafe to create fset:s concurrently so
// that's the price we pay. // that's the price we pay.
var err error var err error
fset, err = db.NewFileSet(folder, to.Filesystem(), m.db) fset, err = db.NewFileSet(folder, m.db)
if err != nil { if err != nil {
return fmt.Errorf("restarting %v: %w", to.Description(), err) return fmt.Errorf("restarting %v: %w", to.Description(), err)
} }
@ -586,7 +586,7 @@ func (m *model) restartFolder(from, to config.FolderConfiguration, cacheIgnoredF
func (m *model) newFolder(cfg config.FolderConfiguration, cacheIgnoredFiles bool) error { func (m *model) newFolder(cfg config.FolderConfiguration, cacheIgnoredFiles bool) error {
// Creating the fileset can take a long time (metadata calculation) so // Creating the fileset can take a long time (metadata calculation) so
// we do it outside of the lock. // we do it outside of the lock.
fset, err := db.NewFileSet(cfg.ID, cfg.Filesystem(), m.db) fset, err := db.NewFileSet(cfg.ID, m.db)
if err != nil { if err != nil {
return fmt.Errorf("adding %v: %w", cfg.Description(), err) return fmt.Errorf("adding %v: %w", cfg.Description(), err)
} }
@ -2003,11 +2003,12 @@ func (m *model) CurrentGlobalFile(folder string, file string) (protocol.FileInfo
func (m *model) GetMtimeMapping(folder string, file string) (fs.MtimeMapping, error) { func (m *model) GetMtimeMapping(folder string, file string) (fs.MtimeMapping, error) {
m.fmut.RLock() m.fmut.RLock()
ffs, ok := m.folderFiles[folder] ffs, ok := m.folderFiles[folder]
fcfg := m.folderCfgs[folder]
m.fmut.RUnlock() m.fmut.RUnlock()
if !ok { if !ok {
return fs.MtimeMapping{}, ErrFolderMissing return fs.MtimeMapping{}, ErrFolderMissing
} }
return fs.GetMtimeMapping(ffs.MtimeFS(), file) return fs.GetMtimeMapping(ffs.MtimeFS(fcfg.Filesystem()), file)
} }
// Connection returns the current connection for device, and a boolean whether a connection was found. // Connection returns the current connection for device, and a boolean whether a connection was found.

10
lib/model/model_test.go
View File

@ -1621,7 +1621,7 @@ func TestROScanRecovery(t *testing.T) {
defer cancel() defer cancel()
m := newModel(t, cfg, myID, "syncthing", "dev", nil) m := newModel(t, cfg, myID, "syncthing", "dev", nil)
set := newFileSet(t, "default", defaultFs, m.db) set := newFileSet(t, "default", m.db)
set.Update(protocol.LocalDeviceID, []protocol.FileInfo{ set.Update(protocol.LocalDeviceID, []protocol.FileInfo{
{Name: "dummyfile", Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1}}}}, {Name: "dummyfile", Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1}}}},
}) })
@ -1676,7 +1676,7 @@ func TestRWScanRecovery(t *testing.T) {
testOs.RemoveAll(fcfg.Path) testOs.RemoveAll(fcfg.Path)
set := newFileSet(t, "default", defaultFs, m.db) set := newFileSet(t, "default", m.db)
set.Update(protocol.LocalDeviceID, []protocol.FileInfo{ set.Update(protocol.LocalDeviceID, []protocol.FileInfo{
{Name: "dummyfile", Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1}}}}, {Name: "dummyfile", Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1}}}},
}) })
@ -2177,7 +2177,7 @@ func TestIssue2782(t *testing.T) {
func TestIndexesForUnknownDevicesDropped(t *testing.T) { func TestIndexesForUnknownDevicesDropped(t *testing.T) {
m := newModel(t, defaultCfgWrapper, myID, "syncthing", "dev", nil) m := newModel(t, defaultCfgWrapper, myID, "syncthing", "dev", nil)
files := newFileSet(t, "default", defaultFs, m.db) files := newFileSet(t, "default", m.db)
files.Drop(device1) files.Drop(device1)
files.Update(device1, genFiles(1)) files.Update(device1, genFiles(1))
files.Drop(device2) files.Drop(device2)
@ -2191,7 +2191,7 @@ func TestIndexesForUnknownDevicesDropped(t *testing.T) {
defer cleanupModel(m) defer cleanupModel(m)
// Remote sequence is cached, hence need to recreated. // Remote sequence is cached, hence need to recreated.
files = newFileSet(t, "default", defaultFs, m.db) files = newFileSet(t, "default", m.db)
if l := len(files.ListDevices()); l != 1 { if l := len(files.ListDevices()); l != 1 {
t.Errorf("Expected one device got %v", l) t.Errorf("Expected one device got %v", l)
@ -2622,7 +2622,7 @@ func TestCustomMarkerName(t *testing.T) {
testOs.RemoveAll(fcfg.Path) testOs.RemoveAll(fcfg.Path)
m := newModel(t, cfg, myID, "syncthing", "dev", nil) m := newModel(t, cfg, myID, "syncthing", "dev", nil)
set := newFileSet(t, "default", defaultFs, m.db) set := newFileSet(t, "default", m.db)
set.Update(protocol.LocalDeviceID, []protocol.FileInfo{ set.Update(protocol.LocalDeviceID, []protocol.FileInfo{
{Name: "dummyfile"}, {Name: "dummyfile"},
}) })

6
lib/model/testutils_test.go
View File

@ -307,7 +307,7 @@ func fsetSnapshot(t *testing.T, fset *db.FileSet) *db.Snapshot {
func folderIgnoresAlwaysReload(t testing.TB, m *testModel, fcfg config.FolderConfiguration) { func folderIgnoresAlwaysReload(t testing.TB, m *testModel, fcfg config.FolderConfiguration) {
t.Helper() t.Helper()
m.removeFolder(fcfg) m.removeFolder(fcfg)
fset := newFileSet(t, fcfg.ID, fcfg.Filesystem(), m.db) fset := newFileSet(t, fcfg.ID, m.db)
ignores := ignore.New(fcfg.Filesystem(), ignore.WithCache(true), ignore.WithChangeDetector(newAlwaysChanged())) ignores := ignore.New(fcfg.Filesystem(), ignore.WithCache(true), ignore.WithChangeDetector(newAlwaysChanged()))
m.fmut.Lock() m.fmut.Lock()
m.addAndStartFolderLockedWithIgnores(fcfg, fset, ignores) m.addAndStartFolderLockedWithIgnores(fcfg, fset, ignores)
@ -359,9 +359,9 @@ func newDeviceConfiguration(defaultCfg config.DeviceConfiguration, id protocol.D
return cfg return cfg
} }
func newFileSet(t testing.TB, folder string, fs fs.Filesystem, ldb *db.Lowlevel) *db.FileSet { func newFileSet(t testing.TB, folder string, ldb *db.Lowlevel) *db.FileSet {
t.Helper() t.Helper()
fset, err := db.NewFileSet(folder, fs, ldb) fset, err := db.NewFileSet(folder, ldb)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }