lib/fs, lib/api, lib/model: Expose mtime remappings as part of /db/file (#7624)

* lib/fs, lib/api, lib/model: Expose mtime remappings as part of /db/file

* Fix wrong error returned by CLI

* Gofmt

* Better names

* Review comments

* Review comments
This commit is contained in:
Audrius Butkevicius 2021-05-03 11:28:25 +01:00 committed by GitHub
parent f09dcb98eb
commit 87a0eecc31
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 305 additions and 111 deletions

View File

@ -135,11 +135,11 @@ func (c *apiClient) Post(url, body string) (*http.Response, error) {
} }
func checkResponse(response *http.Response) error { func checkResponse(response *http.Response) error {
if response.StatusCode == 404 { if response.StatusCode == http.StatusNotFound {
return errors.New("invalid endpoint or API call") return errors.New("invalid endpoint or API call")
} else if response.StatusCode == 403 { } else if response.StatusCode == http.StatusUnauthorized {
return errors.New("invalid API key") return errors.New("invalid API key")
} else if response.StatusCode != 200 { } else if response.StatusCode != http.StatusOK {
data, err := responseToBArray(response) data, err := responseToBArray(response)
if err != nil { if err != nil {
return err return err

View File

@ -32,7 +32,7 @@ import (
"unicode" "unicode"
"github.com/julienschmidt/httprouter" "github.com/julienschmidt/httprouter"
metrics "github.com/rcrowley/go-metrics" "github.com/rcrowley/go-metrics"
"github.com/thejerf/suture/v4" "github.com/thejerf/suture/v4"
"github.com/vitrun/qart/qr" "github.com/vitrun/qart/qr"
"golang.org/x/text/runes" "golang.org/x/text/runes"
@ -915,11 +915,16 @@ func (s *service) getDBFile(w http.ResponseWriter, r *http.Request) {
if err != nil { if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError) http.Error(w, err.Error(), http.StatusInternalServerError)
} }
mtimeMapping, mtimeErr := s.model.GetMtimeMapping(folder, file)
sendJSON(w, map[string]interface{}{ sendJSON(w, map[string]interface{}{
"global": jsonFileInfo(gf), "global": jsonFileInfo(gf),
"local": jsonFileInfo(lf), "local": jsonFileInfo(lf),
"availability": av, "availability": av,
"mtime": map[string]interface{}{
"err": mtimeErr,
"value": mtimeMapping,
},
}) })
} }
@ -934,6 +939,8 @@ func (s *service) getDebugFile(w http.ResponseWriter, r *http.Request) {
return return
} }
mtimeMapping, mtimeErr := s.model.GetMtimeMapping(folder, file)
lf, _ := snap.Get(protocol.LocalDeviceID, file) lf, _ := snap.Get(protocol.LocalDeviceID, file)
gf, _ := snap.GetGlobal(file) gf, _ := snap.GetGlobal(file)
av := snap.Availability(file) av := snap.Availability(file)
@ -944,6 +951,10 @@ func (s *service) getDebugFile(w http.ResponseWriter, r *http.Request) {
"local": jsonFileInfo(lf), "local": jsonFileInfo(lf),
"availability": av, "availability": av,
"globalVersions": vl.String(), "globalVersions": vl.String(),
"mtime": map[string]interface{}{
"err": mtimeErr,
"value": mtimeMapping,
},
}) })
} }

View File

@ -36,7 +36,7 @@ const (
// KeyTypeFolderStatistic <folder ID as string> <some string> = some value // KeyTypeFolderStatistic <folder ID as string> <some string> = some value
KeyTypeFolderStatistic byte = 4 KeyTypeFolderStatistic byte = 4
// KeyTypeVirtualMtime <int32 folder ID> <file name> = dbMtime // KeyTypeVirtualMtime <int32 folder ID> <file name> = mtimeMapping
KeyTypeVirtualMtime byte = 5 KeyTypeVirtualMtime byte = 5
// KeyTypeFolderIdx <int32 id> = string value // KeyTypeFolderIdx <int32 id> = string value

View File

@ -330,6 +330,14 @@ func (f *BasicFilesystem) SameFile(fi1, fi2 FileInfo) bool {
return os.SameFile(f1.osFileInfo(), f2.osFileInfo()) return os.SameFile(f1.osFileInfo(), f2.osFileInfo())
} }
func (f *BasicFilesystem) underlying() (Filesystem, bool) {
return nil, false
}
func (f *BasicFilesystem) wrapperType() filesystemWrapperType {
return filesystemWrapperTypeNone
}
// basicFile implements the fs.File interface on top of an os.File // basicFile implements the fs.File interface on top of an os.File
type basicFile struct { type basicFile struct {
*os.File *os.File

View File

@ -339,6 +339,14 @@ func (f *caseFilesystem) Unhide(name string) error {
return f.Filesystem.Unhide(name) return f.Filesystem.Unhide(name)
} }
func (f *caseFilesystem) underlying() (Filesystem, bool) {
return f.Filesystem, true
}
func (f *caseFilesystem) wrapperType() filesystemWrapperType {
return filesystemWrapperTypeCase
}
func (f *caseFilesystem) checkCase(name string) error { func (f *caseFilesystem) checkCase(name string) error {
var err error var err error
if name, err = Canonicalize(name); err != nil { if name, err = Canonicalize(name); err != nil {

View File

@ -161,7 +161,10 @@ func BenchmarkWalkCaseFakeFS100k(b *testing.B) {
b.Fatal(err) b.Fatal(err)
} }
b.Run("rawfs", func(b *testing.B) { b.Run("rawfs", func(b *testing.B) {
fakefs := unwrapFilesystem(fsys).(*fakefs) var fakefs *fakeFS
if ffs, ok := unwrapFilesystem(fsys, filesystemWrapperTypeNone); ok {
fakefs = ffs.(*fakeFS)
}
fakefs.resetCounters() fakefs.resetCounters()
benchmarkWalkFakeFS(b, fsys, paths, 0, "") benchmarkWalkFakeFS(b, fsys, paths, 0, "")
fakefs.reportMetricsPerOp(b) fakefs.reportMetricsPerOp(b)
@ -174,7 +177,10 @@ func BenchmarkWalkCaseFakeFS100k(b *testing.B) {
Filesystem: fsys, Filesystem: fsys,
realCaser: newDefaultRealCaser(fsys), realCaser: newDefaultRealCaser(fsys),
} }
fakefs := unwrapFilesystem(fsys).(*fakefs) var fakefs *fakeFS
if ffs, ok := unwrapFilesystem(fsys, filesystemWrapperTypeNone); ok {
fakefs = ffs.(*fakeFS)
}
fakefs.resetCounters() fakefs.resetCounters()
benchmarkWalkFakeFS(b, casefs, paths, 0, "") benchmarkWalkFakeFS(b, casefs, paths, 0, "")
fakefs.reportMetricsPerOp(b) fakefs.reportMetricsPerOp(b)
@ -197,7 +203,10 @@ func BenchmarkWalkCaseFakeFS100k(b *testing.B) {
Filesystem: fsys, Filesystem: fsys,
realCaser: newDefaultRealCaser(fsys), realCaser: newDefaultRealCaser(fsys),
} }
fakefs := unwrapFilesystem(fsys).(*fakefs) var fakefs *fakeFS
if ffs, ok := unwrapFilesystem(fsys, filesystemWrapperTypeNone); ok {
fakefs = ffs.(*fakeFS)
}
fakefs.resetCounters() fakefs.resetCounters()
benchmarkWalkFakeFS(b, casefs, paths, otherOpEvery, otherOpPath) benchmarkWalkFakeFS(b, casefs, paths, otherOpEvery, otherOpPath)
fakefs.reportMetricsPerOp(b) fakefs.reportMetricsPerOp(b)

View File

@ -18,7 +18,8 @@ import (
// reason for existence is the Windows version, which allows creating // reason for existence is the Windows version, which allows creating
// symlinks when non-elevated. // symlinks when non-elevated.
func DebugSymlinkForTestsOnly(oldFs, newFs Filesystem, oldname, newname string) error { func DebugSymlinkForTestsOnly(oldFs, newFs Filesystem, oldname, newname string) error {
if caseFs, ok := unwrapFilesystem(newFs).(*caseFilesystem); ok { if fs, ok := unwrapFilesystem(newFs, filesystemWrapperTypeCase); ok {
caseFs := fs.(*caseFilesystem)
if err := caseFs.checkCase(newname); err != nil { if err := caseFs.checkCase(newname); err != nil {
return err return err
} }

View File

@ -52,3 +52,11 @@ func (fs *errorFilesystem) SameFile(fi1, fi2 FileInfo) bool { return false }
func (fs *errorFilesystem) Watch(path string, ignore Matcher, ctx context.Context, ignorePerms bool) (<-chan Event, <-chan error, error) { func (fs *errorFilesystem) Watch(path string, ignore Matcher, ctx context.Context, ignorePerms bool) (<-chan Event, <-chan error, error) {
return nil, nil, fs.err return nil, nil, fs.err
} }
func (fs *errorFilesystem) underlying() (Filesystem, bool) {
return nil, false
}
func (fs *errorFilesystem) wrapperType() filesystemWrapperType {
return filesystemWrapperTypeError
}

View File

@ -27,7 +27,7 @@ import (
// see readShortAt() // see readShortAt()
const randomBlockShift = 14 // 128k const randomBlockShift = 14 // 128k
// fakefs is a fake filesystem for testing and benchmarking. It has the // fakeFS is a fake filesystem for testing and benchmarking. It has the
// following properties: // following properties:
// //
// - File metadata is kept in RAM. Specifically, we remember which files and // - File metadata is kept in RAM. Specifically, we remember which files and
@ -37,7 +37,7 @@ const randomBlockShift = 14 // 128k
// - File contents are generated pseudorandomly with just the file name as // - File contents are generated pseudorandomly with just the file name as
// seed. Writes are discarded, other than having the effect of increasing // seed. Writes are discarded, other than having the effect of increasing
// the file size. If you only write data that you've read from a file with // the file size. If you only write data that you've read from a file with
// the same name on a different fakefs, you'll never know the difference... // the same name on a different fakeFS, you'll never know the difference...
// //
// - We totally ignore permissions - pretend you are root. // - We totally ignore permissions - pretend you are root.
// //
@ -51,10 +51,10 @@ const randomBlockShift = 14 // 128k
// insens=b "true" makes filesystem case-insensitive Windows- or OSX-style (default false) // insens=b "true" makes filesystem case-insensitive Windows- or OSX-style (default false)
// latency=d to set the amount of time each "disk" operation takes, where d is time.ParseDuration format // latency=d to set the amount of time each "disk" operation takes, where d is time.ParseDuration format
// //
// - Two fakefs:s pointing at the same root path see the same files. // - Two fakeFS:s pointing at the same root path see the same files.
// //
type fakefs struct { type fakeFS struct {
counters fakefsCounters counters fakeFSCounters
uri string uri string
mut sync.Mutex mut sync.Mutex
root *fakeEntry root *fakeEntry
@ -63,7 +63,7 @@ type fakefs struct {
latency time.Duration latency time.Duration
} }
type fakefsCounters struct { type fakeFSCounters struct {
Chmod int64 Chmod int64
Lchown int64 Lchown int64
Chtimes int64 Chtimes int64
@ -81,13 +81,13 @@ type fakefsCounters struct {
} }
var ( var (
fakefsMut sync.Mutex fakeFSMut sync.Mutex
fakefsFs = make(map[string]*fakefs) fakeFSCache = make(map[string]*fakeFS)
) )
func newFakeFilesystem(rootURI string, _ ...Option) *fakefs { func newFakeFilesystem(rootURI string, _ ...Option) *fakeFS {
fakefsMut.Lock() fakeFSMut.Lock()
defer fakefsMut.Unlock() defer fakeFSMut.Unlock()
root := rootURI root := rootURI
var params url.Values var params url.Values
@ -97,12 +97,12 @@ func newFakeFilesystem(rootURI string, _ ...Option) *fakefs {
params = uri.Query() params = uri.Query()
} }
if fs, ok := fakefsFs[rootURI]; ok { if fs, ok := fakeFSCache[rootURI]; ok {
// Already have an fs at this path // Already have an fs at this path
return fs return fs
} }
fs := &fakefs{ fs := &fakeFS{
uri: "fake://" + rootURI, uri: "fake://" + rootURI,
root: &fakeEntry{ root: &fakeEntry{
name: "/", name: "/",
@ -157,7 +157,7 @@ func newFakeFilesystem(rootURI string, _ ...Option) *fakefs {
// the filesystem initially. // the filesystem initially.
fs.latency, _ = time.ParseDuration(params.Get("latency")) fs.latency, _ = time.ParseDuration(params.Get("latency"))
fakefsFs[root] = fs fakeFSCache[root] = fs
return fs return fs
} }
@ -183,7 +183,7 @@ type fakeEntry struct {
content []byte content []byte
} }
func (fs *fakefs) entryForName(name string) *fakeEntry { func (fs *fakeFS) entryForName(name string) *fakeEntry {
// bug: lookup doesn't work through symlinks. // bug: lookup doesn't work through symlinks.
if fs.insens { if fs.insens {
name = UnicodeLowercase(name) name = UnicodeLowercase(name)
@ -210,7 +210,7 @@ func (fs *fakefs) entryForName(name string) *fakeEntry {
return entry return entry
} }
func (fs *fakefs) Chmod(name string, mode FileMode) error { func (fs *fakeFS) Chmod(name string, mode FileMode) error {
fs.mut.Lock() fs.mut.Lock()
defer fs.mut.Unlock() defer fs.mut.Unlock()
fs.counters.Chmod++ fs.counters.Chmod++
@ -223,7 +223,7 @@ func (fs *fakefs) Chmod(name string, mode FileMode) error {
return nil return nil
} }
func (fs *fakefs) Lchown(name string, uid, gid int) error { func (fs *fakeFS) Lchown(name string, uid, gid int) error {
fs.mut.Lock() fs.mut.Lock()
defer fs.mut.Unlock() defer fs.mut.Unlock()
fs.counters.Lchown++ fs.counters.Lchown++
@ -237,7 +237,7 @@ func (fs *fakefs) Lchown(name string, uid, gid int) error {
return nil return nil
} }
func (fs *fakefs) Chtimes(name string, atime time.Time, mtime time.Time) error { func (fs *fakeFS) Chtimes(name string, atime time.Time, mtime time.Time) error {
fs.mut.Lock() fs.mut.Lock()
defer fs.mut.Unlock() defer fs.mut.Unlock()
fs.counters.Chtimes++ fs.counters.Chtimes++
@ -250,7 +250,7 @@ func (fs *fakefs) Chtimes(name string, atime time.Time, mtime time.Time) error {
return nil return nil
} }
func (fs *fakefs) create(name string) (*fakeEntry, error) { func (fs *fakeFS) create(name string) (*fakeEntry, error) {
fs.mut.Lock() fs.mut.Lock()
defer fs.mut.Unlock() defer fs.mut.Unlock()
fs.counters.Create++ fs.counters.Create++
@ -296,7 +296,7 @@ func (fs *fakefs) create(name string) (*fakeEntry, error) {
return new, nil return new, nil
} }
func (fs *fakefs) Create(name string) (File, error) { func (fs *fakeFS) Create(name string) (File, error) {
entry, err := fs.create(name) entry, err := fs.create(name)
if err != nil { if err != nil {
return nil, err return nil, err
@ -307,7 +307,7 @@ func (fs *fakefs) Create(name string) (File, error) {
return &fakeFile{fakeEntry: entry}, nil return &fakeFile{fakeEntry: entry}, nil
} }
func (fs *fakefs) CreateSymlink(target, name string) error { func (fs *fakeFS) CreateSymlink(target, name string) error {
entry, err := fs.create(name) entry, err := fs.create(name)
if err != nil { if err != nil {
return err return err
@ -317,7 +317,7 @@ func (fs *fakefs) CreateSymlink(target, name string) error {
return nil return nil
} }
func (fs *fakefs) DirNames(name string) ([]string, error) { func (fs *fakeFS) DirNames(name string) ([]string, error) {
fs.mut.Lock() fs.mut.Lock()
defer fs.mut.Unlock() defer fs.mut.Unlock()
fs.counters.DirNames++ fs.counters.DirNames++
@ -336,7 +336,7 @@ func (fs *fakefs) DirNames(name string) ([]string, error) {
return names, nil return names, nil
} }
func (fs *fakefs) Lstat(name string) (FileInfo, error) { func (fs *fakeFS) Lstat(name string) (FileInfo, error) {
fs.mut.Lock() fs.mut.Lock()
defer fs.mut.Unlock() defer fs.mut.Unlock()
fs.counters.Lstat++ fs.counters.Lstat++
@ -355,7 +355,7 @@ func (fs *fakefs) Lstat(name string) (FileInfo, error) {
return info, nil return info, nil
} }
func (fs *fakefs) Mkdir(name string, perm FileMode) error { func (fs *fakeFS) Mkdir(name string, perm FileMode) error {
fs.mut.Lock() fs.mut.Lock()
defer fs.mut.Unlock() defer fs.mut.Unlock()
fs.counters.Mkdir++ fs.counters.Mkdir++
@ -389,7 +389,7 @@ func (fs *fakefs) Mkdir(name string, perm FileMode) error {
return nil return nil
} }
func (fs *fakefs) MkdirAll(name string, perm FileMode) error { func (fs *fakeFS) MkdirAll(name string, perm FileMode) error {
fs.mut.Lock() fs.mut.Lock()
defer fs.mut.Unlock() defer fs.mut.Unlock()
fs.counters.MkdirAll++ fs.counters.MkdirAll++
@ -426,7 +426,7 @@ func (fs *fakefs) MkdirAll(name string, perm FileMode) error {
return nil return nil
} }
func (fs *fakefs) Open(name string) (File, error) { func (fs *fakeFS) Open(name string) (File, error) {
fs.mut.Lock() fs.mut.Lock()
defer fs.mut.Unlock() defer fs.mut.Unlock()
fs.counters.Open++ fs.counters.Open++
@ -443,7 +443,7 @@ func (fs *fakefs) Open(name string) (File, error) {
return &fakeFile{fakeEntry: entry}, nil return &fakeFile{fakeEntry: entry}, nil
} }
func (fs *fakefs) OpenFile(name string, flags int, mode FileMode) (File, error) { func (fs *fakeFS) OpenFile(name string, flags int, mode FileMode) (File, error) {
if flags&os.O_CREATE == 0 { if flags&os.O_CREATE == 0 {
return fs.Open(name) return fs.Open(name)
} }
@ -486,7 +486,7 @@ func (fs *fakefs) OpenFile(name string, flags int, mode FileMode) (File, error)
return &fakeFile{fakeEntry: newEntry}, nil return &fakeFile{fakeEntry: newEntry}, nil
} }
func (fs *fakefs) ReadSymlink(name string) (string, error) { func (fs *fakeFS) ReadSymlink(name string) (string, error) {
fs.mut.Lock() fs.mut.Lock()
defer fs.mut.Unlock() defer fs.mut.Unlock()
fs.counters.ReadSymlink++ fs.counters.ReadSymlink++
@ -501,7 +501,7 @@ func (fs *fakefs) ReadSymlink(name string) (string, error) {
return entry.dest, nil return entry.dest, nil
} }
func (fs *fakefs) Remove(name string) error { func (fs *fakeFS) Remove(name string) error {
fs.mut.Lock() fs.mut.Lock()
defer fs.mut.Unlock() defer fs.mut.Unlock()
fs.counters.Remove++ fs.counters.Remove++
@ -524,7 +524,7 @@ func (fs *fakefs) Remove(name string) error {
return nil return nil
} }
func (fs *fakefs) RemoveAll(name string) error { func (fs *fakeFS) RemoveAll(name string) error {
fs.mut.Lock() fs.mut.Lock()
defer fs.mut.Unlock() defer fs.mut.Unlock()
fs.counters.RemoveAll++ fs.counters.RemoveAll++
@ -536,7 +536,7 @@ func (fs *fakefs) RemoveAll(name string) error {
entry := fs.entryForName(filepath.Dir(name)) entry := fs.entryForName(filepath.Dir(name))
if entry == nil { if entry == nil {
return nil // all tested real systems exibit this behaviour return nil // all tested real systems exhibit this behaviour
} }
// RemoveAll is easy when the file system uses garbage collection under // RemoveAll is easy when the file system uses garbage collection under
@ -545,7 +545,7 @@ func (fs *fakefs) RemoveAll(name string) error {
return nil return nil
} }
func (fs *fakefs) Rename(oldname, newname string) error { func (fs *fakeFS) Rename(oldname, newname string) error {
fs.mut.Lock() fs.mut.Lock()
defer fs.mut.Unlock() defer fs.mut.Unlock()
fs.counters.Rename++ fs.counters.Rename++
@ -595,56 +595,56 @@ func (fs *fakefs) Rename(oldname, newname string) error {
return nil return nil
} }
func (fs *fakefs) Stat(name string) (FileInfo, error) { func (fs *fakeFS) Stat(name string) (FileInfo, error) {
return fs.Lstat(name) return fs.Lstat(name)
} }
func (fs *fakefs) SymlinksSupported() bool { func (fs *fakeFS) SymlinksSupported() bool {
return false return false
} }
func (fs *fakefs) Walk(name string, walkFn WalkFunc) error { func (fs *fakeFS) Walk(name string, walkFn WalkFunc) error {
return errors.New("not implemented") return errors.New("not implemented")
} }
func (fs *fakefs) Watch(path string, ignore Matcher, ctx context.Context, ignorePerms bool) (<-chan Event, <-chan error, error) { func (fs *fakeFS) Watch(path string, ignore Matcher, ctx context.Context, ignorePerms bool) (<-chan Event, <-chan error, error) {
return nil, nil, ErrWatchNotSupported return nil, nil, ErrWatchNotSupported
} }
func (fs *fakefs) Hide(name string) error { func (fs *fakeFS) Hide(name string) error {
return nil return nil
} }
func (fs *fakefs) Unhide(name string) error { func (fs *fakeFS) Unhide(name string) error {
return nil return nil
} }
func (fs *fakefs) Glob(pattern string) ([]string, error) { func (fs *fakeFS) Glob(pattern string) ([]string, error) {
// gnnh we don't seem to actually require this in practice // gnnh we don't seem to actually require this in practice
return nil, errors.New("not implemented") return nil, errors.New("not implemented")
} }
func (fs *fakefs) Roots() ([]string, error) { func (fs *fakeFS) Roots() ([]string, error) {
return []string{"/"}, nil return []string{"/"}, nil
} }
func (fs *fakefs) Usage(name string) (Usage, error) { func (fs *fakeFS) Usage(name string) (Usage, error) {
return Usage{}, errors.New("not implemented") return Usage{}, errors.New("not implemented")
} }
func (fs *fakefs) Type() FilesystemType { func (fs *fakeFS) Type() FilesystemType {
return FilesystemTypeFake return FilesystemTypeFake
} }
func (fs *fakefs) URI() string { func (fs *fakeFS) URI() string {
return fs.uri return fs.uri
} }
func (fs *fakefs) Options() []Option { func (fs *fakeFS) Options() []Option {
return nil return nil
} }
func (fs *fakefs) SameFile(fi1, fi2 FileInfo) bool { func (fs *fakeFS) SameFile(fi1, fi2 FileInfo) bool {
// BUG: real systems base file sameness on path, inodes, etc // BUG: real systems base file sameness on path, inodes, etc
// we try our best, but FileInfo just doesn't have enough data // we try our best, but FileInfo just doesn't have enough data
// so there be false positives, especially on Windows // so there be false positives, especially on Windows
@ -659,17 +659,25 @@ func (fs *fakefs) SameFile(fi1, fi2 FileInfo) bool {
return ok && fi1.ModTime().Equal(fi2.ModTime()) && fi1.Mode() == fi2.Mode() && fi1.IsDir() == fi2.IsDir() && fi1.IsRegular() == fi2.IsRegular() && fi1.IsSymlink() == fi2.IsSymlink() && fi1.Owner() == fi2.Owner() && fi1.Group() == fi2.Group() return ok && fi1.ModTime().Equal(fi2.ModTime()) && fi1.Mode() == fi2.Mode() && fi1.IsDir() == fi2.IsDir() && fi1.IsRegular() == fi2.IsRegular() && fi1.IsSymlink() == fi2.IsSymlink() && fi1.Owner() == fi2.Owner() && fi1.Group() == fi2.Group()
} }
func (fs *fakefs) resetCounters() { func (fs *fakeFS) underlying() (Filesystem, bool) {
return nil, false
}
func (fs *fakeFS) wrapperType() filesystemWrapperType {
return filesystemWrapperTypeNone
}
func (fs *fakeFS) resetCounters() {
fs.mut.Lock() fs.mut.Lock()
fs.counters = fakefsCounters{} fs.counters = fakeFSCounters{}
fs.mut.Unlock() fs.mut.Unlock()
} }
func (fs *fakefs) reportMetricsPerOp(b *testing.B) { func (fs *fakeFS) reportMetricsPerOp(b *testing.B) {
fs.reportMetricsPer(b, 1, "op") fs.reportMetricsPer(b, 1, "op")
} }
func (fs *fakefs) reportMetricsPer(b *testing.B, divisor float64, unit string) { func (fs *fakeFS) reportMetricsPer(b *testing.B, divisor float64, unit string) {
fs.mut.Lock() fs.mut.Lock()
defer fs.mut.Unlock() defer fs.mut.Unlock()
b.ReportMetric(float64(fs.counters.Lstat)/divisor/float64(b.N), "Lstat/"+unit) b.ReportMetric(float64(fs.counters.Lstat)/divisor/float64(b.N), "Lstat/"+unit)

View File

@ -21,7 +21,7 @@ import (
) )
func TestFakeFS(t *testing.T) { func TestFakeFS(t *testing.T) {
// Test some basic aspects of the fakefs // Test some basic aspects of the fakeFS
fs := newFakeFilesystem("/foo/bar/baz") fs := newFakeFilesystem("/foo/bar/baz")
@ -131,7 +131,7 @@ func TestFakeFS(t *testing.T) {
} }
func testFakeFSRead(t *testing.T, fs Filesystem) { func testFakeFSRead(t *testing.T, fs Filesystem) {
// Test some basic aspects of the fakefs // Test some basic aspects of the fakeFS
// Create // Create
fd, _ := fs.Create("test") fd, _ := fs.Create("test")
defer fd.Close() defer fd.Close()
@ -201,7 +201,7 @@ func TestFakeFSCaseSensitive(t *testing.T) {
{"FileName", testFakeFSFileName}, {"FileName", testFakeFSFileName},
} }
var filesystems = []testFS{ var filesystems = []testFS{
{"fakefs", newFakeFilesystem("/foo")}, {"fakeFS", newFakeFilesystem("/foo")},
} }
testDir, sensitive := createTestDir(t) testDir, sensitive := createTestDir(t)
@ -237,7 +237,7 @@ func TestFakeFSCaseInsensitive(t *testing.T) {
} }
var filesystems = []testFS{ var filesystems = []testFS{
{"fakefs", newFakeFilesystem("/foobar?insens=true")}, {"fakeFS", newFakeFilesystem("/foobar?insens=true")},
} }
testDir, sensitive := createTestDir(t) testDir, sensitive := createTestDir(t)
@ -891,7 +891,7 @@ func testFakeFSCreateInsens(t *testing.T, fs Filesystem) {
t.Errorf("name of created file \"fOo\" is %s", fd2.Name()) t.Errorf("name of created file \"fOo\" is %s", fd2.Name())
} }
// one would expect DirNames to show the last variant, but in fact it shows // one would expect DirNames to show the last wrapperType, but in fact it shows
// the original one // the original one
assertDir(t, fs, "/", []string{"FOO"}) assertDir(t, fs, "/", []string{"FOO"})
} }

View File

@ -16,6 +16,17 @@ import (
"time" "time"
) )
type filesystemWrapperType int32
const (
filesystemWrapperTypeNone filesystemWrapperType = iota
filesystemWrapperTypeMtime
filesystemWrapperTypeCase
filesystemWrapperTypeError
filesystemWrapperTypeWalk
filesystemWrapperTypeLog
)
// The Filesystem interface abstracts access to the file system. // The Filesystem interface abstracts access to the file system.
type Filesystem interface { type Filesystem interface {
Chmod(name string, mode FileMode) error Chmod(name string, mode FileMode) error
@ -49,6 +60,10 @@ type Filesystem interface {
URI() string URI() string
Options() []Option Options() []Option
SameFile(fi1, fi2 FileInfo) bool SameFile(fi1, fi2 FileInfo) bool
// Used for unwrapping things
underlying() (Filesystem, bool)
wrapperType() filesystemWrapperType
} }
// The File interface abstracts access to a regular file, being a somewhat // The File interface abstracts access to a regular file, being a somewhat
@ -284,18 +299,16 @@ func wrapFilesystem(fs Filesystem, wrapFn func(Filesystem) Filesystem) Filesyste
return fs return fs
} }
// unwrapFilesystem removes "wrapping" filesystems to expose the underlying filesystem. // unwrapFilesystem removes "wrapping" filesystems to expose the filesystem of the requested wrapperType, if it exists.
func unwrapFilesystem(fs Filesystem) Filesystem { func unwrapFilesystem(fs Filesystem, wrapperType filesystemWrapperType) (Filesystem, bool) {
var ok bool
for { for {
switch sfs := fs.(type) { if fs.wrapperType() == wrapperType {
case *logFilesystem: return fs, true
fs = sfs.Filesystem }
case *walkFilesystem: fs, ok = fs.underlying()
fs = sfs.Filesystem if !ok {
case *mtimeFS: return nil, false
fs = sfs.Filesystem
default:
return sfs
} }
} }
} }

View File

@ -163,3 +163,11 @@ func (fs *logFilesystem) Usage(name string) (Usage, error) {
l.Debugln(getCaller(), fs.Type(), fs.URI(), "Usage", name, usage, err) l.Debugln(getCaller(), fs.Type(), fs.URI(), "Usage", name, usage, err)
return usage, err return usage, err
} }
func (fs *logFilesystem) underlying() (Filesystem, bool) {
return fs.Filesystem, true
}
func (fs *logFilesystem) wrapperType() filesystemWrapperType {
return filesystemWrapperTypeLog
}

View File

@ -7,6 +7,7 @@
package fs package fs
import ( import (
"errors"
"time" "time"
) )
@ -69,14 +70,14 @@ func (f *mtimeFS) Stat(name string) (FileInfo, error) {
return nil, err return nil, err
} }
real, virtual, err := f.load(name) mtimeMapping, err := f.load(name)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if real == info.ModTime() { if mtimeMapping.Real == info.ModTime() {
info = mtimeFileInfo{ info = mtimeFileInfo{
FileInfo: info, FileInfo: info,
mtime: virtual, mtime: mtimeMapping.Virtual,
} }
} }
@ -89,14 +90,14 @@ func (f *mtimeFS) Lstat(name string) (FileInfo, error) {
return nil, err return nil, err
} }
real, virtual, err := f.load(name) mtimeMapping, err := f.load(name)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if real == info.ModTime() { if mtimeMapping.Real == info.ModTime() {
info = mtimeFileInfo{ info = mtimeFileInfo{
FileInfo: info, FileInfo: info,
mtime: virtual, mtime: mtimeMapping.Virtual,
} }
} }
@ -106,15 +107,15 @@ func (f *mtimeFS) Lstat(name string) (FileInfo, error) {
func (f *mtimeFS) Walk(root string, walkFn WalkFunc) error { func (f *mtimeFS) Walk(root string, walkFn WalkFunc) error {
return f.Filesystem.Walk(root, func(path string, info FileInfo, err error) error { return f.Filesystem.Walk(root, func(path string, info FileInfo, err error) error {
if info != nil { if info != nil {
real, virtual, loadErr := f.load(path) mtimeMapping, loadErr := f.load(path)
if loadErr != nil && err == nil { if loadErr != nil && err == nil {
// The iterator gets to deal with the error // The iterator gets to deal with the error
err = loadErr err = loadErr
} }
if real == info.ModTime() { if mtimeMapping.Real == info.ModTime() {
info = mtimeFileInfo{ info = mtimeFileInfo{
FileInfo: info, FileInfo: info,
mtime: virtual, mtime: mtimeMapping.Virtual,
} }
} }
} }
@ -146,8 +147,13 @@ func (f *mtimeFS) OpenFile(name string, flags int, mode FileMode) (File, error)
return mtimeFile{fd, f}, nil return mtimeFile{fd, f}, nil
} }
// "real" is the on disk timestamp func (f *mtimeFS) underlying() (Filesystem, bool) {
// "virtual" is what want the timestamp to be return f.Filesystem, true
}
func (f *mtimeFS) wrapperType() filesystemWrapperType {
return filesystemWrapperTypeMtime
}
func (f *mtimeFS) save(name string, real, virtual time.Time) { func (f *mtimeFS) save(name string, real, virtual time.Time) {
if f.caseInsensitive { if f.caseInsensitive {
@ -161,32 +167,31 @@ func (f *mtimeFS) save(name string, real, virtual time.Time) {
return return
} }
mtime := dbMtime{ mtime := MtimeMapping{
real: real, Real: real,
virtual: virtual, Virtual: virtual,
} }
bs, _ := mtime.Marshal() // Can't fail bs, _ := mtime.Marshal() // Can't fail
f.db.PutBytes(name, bs) f.db.PutBytes(name, bs)
} }
func (f *mtimeFS) load(name string) (real, virtual time.Time, err error) { func (f *mtimeFS) load(name string) (MtimeMapping, error) {
if f.caseInsensitive { if f.caseInsensitive {
name = UnicodeLowercase(name) name = UnicodeLowercase(name)
} }
data, exists, err := f.db.Bytes(name) data, exists, err := f.db.Bytes(name)
if err != nil { if err != nil {
return time.Time{}, time.Time{}, err return MtimeMapping{}, err
} else if !exists { } else if !exists {
return time.Time{}, time.Time{}, nil return MtimeMapping{}, nil
} }
var mtime dbMtime var mtime MtimeMapping
if err := mtime.Unmarshal(data); err != nil { if err := mtime.Unmarshal(data); err != nil {
return time.Time{}, time.Time{}, err return MtimeMapping{}, err
} }
return mtime, nil
return mtime.real, mtime.virtual, nil
} }
// The mtimeFileInfo is an os.FileInfo that lies about the ModTime(). // The mtimeFileInfo is an os.FileInfo that lies about the ModTime().
@ -211,43 +216,57 @@ func (f mtimeFile) Stat() (FileInfo, error) {
return nil, err return nil, err
} }
real, virtual, err := f.fs.load(f.Name()) mtimeMapping, err := f.fs.load(f.Name())
if err != nil { if err != nil {
return nil, err return nil, err
} }
if real == info.ModTime() { if mtimeMapping.Real == info.ModTime() {
info = mtimeFileInfo{ info = mtimeFileInfo{
FileInfo: info, FileInfo: info,
mtime: virtual, mtime: mtimeMapping.Virtual,
} }
} }
return info, nil return info, nil
} }
// Used by copyRange to unwrap to the real file and access SyscallConn
func (f mtimeFile) unwrap() File { func (f mtimeFile) unwrap() File {
return f.File return f.File
} }
// The dbMtime is our database representation // MtimeMapping represents the mapping as stored in the database
type MtimeMapping struct {
type dbMtime struct { // "Real" is the on disk timestamp
real time.Time Real time.Time `json:"real"`
virtual time.Time // "Virtual" is what want the timestamp to be
Virtual time.Time `json:"virtual"`
} }
func (t *dbMtime) Marshal() ([]byte, error) { func (t *MtimeMapping) Marshal() ([]byte, error) {
bs0, _ := t.real.MarshalBinary() bs0, _ := t.Real.MarshalBinary()
bs1, _ := t.virtual.MarshalBinary() bs1, _ := t.Virtual.MarshalBinary()
return append(bs0, bs1...), nil return append(bs0, bs1...), nil
} }
func (t *dbMtime) Unmarshal(bs []byte) error { func (t *MtimeMapping) Unmarshal(bs []byte) error {
if err := t.real.UnmarshalBinary(bs[:len(bs)/2]); err != nil { if err := t.Real.UnmarshalBinary(bs[:len(bs)/2]); err != nil {
return err return err
} }
if err := t.virtual.UnmarshalBinary(bs[len(bs)/2:]); err != nil { if err := t.Virtual.UnmarshalBinary(bs[len(bs)/2:]); err != nil {
return err return err
} }
return nil return nil
} }
func GetMtimeMapping(fs Filesystem, file string) (MtimeMapping, error) {
fs, ok := unwrapFilesystem(fs, filesystemWrapperTypeMtime)
if !ok {
return MtimeMapping{}, errors.New("failed to unwrap")
}
mtimeFs, ok := fs.(*mtimeFS)
if !ok {
return MtimeMapping{}, errors.New("unwrapping failed")
}
return mtimeFs.load(file)
}

View File

@ -149,3 +149,11 @@ func (f *walkFilesystem) Walk(root string, walkFn WalkFunc) error {
} }
return f.walk(root, info, walkFn, ancestors) return f.walk(root, info, walkFn, ancestors)
} }
func (f *walkFilesystem) underlying() (Filesystem, bool) {
return f.Filesystem, true
}
func (f *walkFilesystem) wrapperType() filesystemWrapperType {
return filesystemWrapperTypeWalk
}

View File

@ -8,6 +8,7 @@ import (
"time" "time"
"github.com/syncthing/syncthing/lib/db" "github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/model" "github.com/syncthing/syncthing/lib/model"
"github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/stats" "github.com/syncthing/syncthing/lib/stats"
@ -249,6 +250,20 @@ type Model struct {
getHelloReturnsOnCall map[int]struct { getHelloReturnsOnCall map[int]struct {
result1 protocol.HelloIntf result1 protocol.HelloIntf
} }
GetMtimeMappingStub func(string, string) (fs.MtimeMapping, error)
getMtimeMappingMutex sync.RWMutex
getMtimeMappingArgsForCall []struct {
arg1 string
arg2 string
}
getMtimeMappingReturns struct {
result1 fs.MtimeMapping
result2 error
}
getMtimeMappingReturnsOnCall map[int]struct {
result1 fs.MtimeMapping
result2 error
}
GlobalDirectoryTreeStub func(string, string, int, bool) ([]*model.TreeEntry, error) GlobalDirectoryTreeStub func(string, string, int, bool) ([]*model.TreeEntry, error)
globalDirectoryTreeMutex sync.RWMutex globalDirectoryTreeMutex sync.RWMutex
globalDirectoryTreeArgsForCall []struct { globalDirectoryTreeArgsForCall []struct {
@ -1691,6 +1706,71 @@ func (fake *Model) GetHelloReturnsOnCall(i int, result1 protocol.HelloIntf) {
}{result1} }{result1}
} }
func (fake *Model) GetMtimeMapping(arg1 string, arg2 string) (fs.MtimeMapping, error) {
fake.getMtimeMappingMutex.Lock()
ret, specificReturn := fake.getMtimeMappingReturnsOnCall[len(fake.getMtimeMappingArgsForCall)]
fake.getMtimeMappingArgsForCall = append(fake.getMtimeMappingArgsForCall, struct {
arg1 string
arg2 string
}{arg1, arg2})
stub := fake.GetMtimeMappingStub
fakeReturns := fake.getMtimeMappingReturns
fake.recordInvocation("GetMtimeMapping", []interface{}{arg1, arg2})
fake.getMtimeMappingMutex.Unlock()
if stub != nil {
return stub(arg1, arg2)
}
if specificReturn {
return ret.result1, ret.result2
}
return fakeReturns.result1, fakeReturns.result2
}
func (fake *Model) GetMtimeMappingCallCount() int {
fake.getMtimeMappingMutex.RLock()
defer fake.getMtimeMappingMutex.RUnlock()
return len(fake.getMtimeMappingArgsForCall)
}
func (fake *Model) GetMtimeMappingCalls(stub func(string, string) (fs.MtimeMapping, error)) {
fake.getMtimeMappingMutex.Lock()
defer fake.getMtimeMappingMutex.Unlock()
fake.GetMtimeMappingStub = stub
}
func (fake *Model) GetMtimeMappingArgsForCall(i int) (string, string) {
fake.getMtimeMappingMutex.RLock()
defer fake.getMtimeMappingMutex.RUnlock()
argsForCall := fake.getMtimeMappingArgsForCall[i]
return argsForCall.arg1, argsForCall.arg2
}
func (fake *Model) GetMtimeMappingReturns(result1 fs.MtimeMapping, result2 error) {
fake.getMtimeMappingMutex.Lock()
defer fake.getMtimeMappingMutex.Unlock()
fake.GetMtimeMappingStub = nil
fake.getMtimeMappingReturns = struct {
result1 fs.MtimeMapping
result2 error
}{result1, result2}
}
func (fake *Model) GetMtimeMappingReturnsOnCall(i int, result1 fs.MtimeMapping, result2 error) {
fake.getMtimeMappingMutex.Lock()
defer fake.getMtimeMappingMutex.Unlock()
fake.GetMtimeMappingStub = nil
if fake.getMtimeMappingReturnsOnCall == nil {
fake.getMtimeMappingReturnsOnCall = make(map[int]struct {
result1 fs.MtimeMapping
result2 error
})
}
fake.getMtimeMappingReturnsOnCall[i] = struct {
result1 fs.MtimeMapping
result2 error
}{result1, result2}
}
func (fake *Model) GlobalDirectoryTree(arg1 string, arg2 string, arg3 int, arg4 bool) ([]*model.TreeEntry, error) { func (fake *Model) GlobalDirectoryTree(arg1 string, arg2 string, arg3 int, arg4 bool) ([]*model.TreeEntry, error) {
fake.globalDirectoryTreeMutex.Lock() fake.globalDirectoryTreeMutex.Lock()
ret, specificReturn := fake.globalDirectoryTreeReturnsOnCall[len(fake.globalDirectoryTreeArgsForCall)] ret, specificReturn := fake.globalDirectoryTreeReturnsOnCall[len(fake.globalDirectoryTreeArgsForCall)]
@ -3186,6 +3266,8 @@ func (fake *Model) Invocations() map[string][][]interface{} {
defer fake.getFolderVersionsMutex.RUnlock() defer fake.getFolderVersionsMutex.RUnlock()
fake.getHelloMutex.RLock() fake.getHelloMutex.RLock()
defer fake.getHelloMutex.RUnlock() defer fake.getHelloMutex.RUnlock()
fake.getMtimeMappingMutex.RLock()
defer fake.getMtimeMappingMutex.RUnlock()
fake.globalDirectoryTreeMutex.RLock() fake.globalDirectoryTreeMutex.RLock()
defer fake.globalDirectoryTreeMutex.RUnlock() defer fake.globalDirectoryTreeMutex.RUnlock()
fake.indexMutex.RLock() fake.indexMutex.RLock()

View File

@ -95,6 +95,7 @@ type Model interface {
CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool, error) CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool, error)
CurrentGlobalFile(folder string, file string) (protocol.FileInfo, bool, error) CurrentGlobalFile(folder string, file string) (protocol.FileInfo, bool, error)
GetMtimeMapping(folder string, file string) (fs.MtimeMapping, error)
Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) ([]Availability, error) Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) ([]Availability, error)
Completion(device protocol.DeviceID, folder string) (FolderCompletion, error) Completion(device protocol.DeviceID, folder string) (FolderCompletion, error)
@ -2038,12 +2039,12 @@ func (m *model) CurrentFolderFile(folder string, file string) (protocol.FileInfo
func (m *model) CurrentGlobalFile(folder string, file string) (protocol.FileInfo, bool, error) { func (m *model) CurrentGlobalFile(folder string, file string) (protocol.FileInfo, bool, error) {
m.fmut.RLock() m.fmut.RLock()
fs, ok := m.folderFiles[folder] ffs, ok := m.folderFiles[folder]
m.fmut.RUnlock() m.fmut.RUnlock()
if !ok { if !ok {
return protocol.FileInfo{}, false, ErrFolderMissing return protocol.FileInfo{}, false, ErrFolderMissing
} }
snap, err := fs.Snapshot() snap, err := ffs.Snapshot()
if err != nil { if err != nil {
return protocol.FileInfo{}, false, err return protocol.FileInfo{}, false, err
} }
@ -2052,6 +2053,16 @@ func (m *model) CurrentGlobalFile(folder string, file string) (protocol.FileInfo
return f, ok, nil return f, ok, nil
} }
func (m *model) GetMtimeMapping(folder string, file string) (fs.MtimeMapping, error) {
m.fmut.RLock()
ffs, ok := m.folderFiles[folder]
m.fmut.RUnlock()
if !ok {
return fs.MtimeMapping{}, ErrFolderMissing
}
return fs.GetMtimeMapping(ffs.MtimeFS(), file)
}
// Connection returns the current connection for device, and a boolean whether a connection was found. // Connection returns the current connection for device, and a boolean whether a connection was found.
func (m *model) Connection(deviceID protocol.DeviceID) (protocol.Connection, bool) { func (m *model) Connection(deviceID protocol.DeviceID) (protocol.Connection, bool) {
m.pmut.RLock() m.pmut.RLock()