Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Move FileIntf to files package, expose Iterator type 

This is where FileIntf is used, so it should be defined here (it's not
a protocol thing, really).
This commit is contained in:
Jakob Borg 2015-01-09 08:18:42 +01:00
parent 2d047fa428
commit 8bc7d259f4
7 changed files with 55 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/stindex/main.go
View File

@ -43,7 +43,7 @@ func main() {
if *device == "" {
log.Printf("*** Global index for folder %q", *folder)
fs.WithGlobalTruncated(func(fi protocol.FileIntf) bool {
fs.WithGlobalTruncated(func(fi files.FileIntf) bool {
f := fi.(protocol.FileInfoTruncated)
fmt.Println(f)
fmt.Println("\t", fs.Availability(f.Name))
@ -55,7 +55,7 @@ func main() {
log.Fatal(err)
}
log.Printf("*** Have index for folder %q device %q", *folder, n)
fs.WithHaveTruncated(n, func(fi protocol.FileIntf) bool {
fs.WithHaveTruncated(n, func(fi files.FileIntf) bool {
f := fi.(protocol.FileInfoTruncated)
fmt.Println(f)
return true

10
internal/files/leveldb.go
View File

@ -166,8 +166,6 @@ func globalKeyFolder(key []byte) []byte {
type deletionHandler func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) uint64
type fileIterator func(f protocol.FileIntf) bool
func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, deleteFn deletionHandler) uint64 {
runtime.GC()
@ -528,7 +526,7 @@ func ldbRemoveFromGlobal(db dbReader, batch dbWriter, folder, device, file []byt
}
}
func ldbWithHave(db *leveldb.DB, folder, device []byte, truncate bool, fn fileIterator) {
func ldbWithHave(db *leveldb.DB, folder, device []byte, truncate bool, fn Iterator) {
start := deviceKey(folder, device, nil) // before all folder/device files
limit := deviceKey(folder, device, []byte{0xff, 0xff, 0xff, 0xff}) // after all folder/device files
snap, err := db.GetSnapshot()
@ -666,7 +664,7 @@ func ldbGetGlobal(db *leveldb.DB, folder, file []byte) (protocol.FileInfo, bool)
return f, true
}
func ldbWithGlobal(db *leveldb.DB, folder []byte, truncate bool, fn fileIterator) {
func ldbWithGlobal(db *leveldb.DB, folder []byte, truncate bool, fn Iterator) {
runtime.GC()
start := globalKey(folder, nil)
@ -754,7 +752,7 @@ func ldbAvailability(db *leveldb.DB, folder, file []byte) []protocol.DeviceID {
return devices
}
func ldbWithNeed(db *leveldb.DB, folder, device []byte, truncate bool, fn fileIterator) {
func ldbWithNeed(db *leveldb.DB, folder, device []byte, truncate bool, fn Iterator) {
runtime.GC()
start := globalKey(folder, nil)
@ -938,7 +936,7 @@ func ldbDropFolder(db *leveldb.DB, folder []byte) {
dbi.Release()
}
func unmarshalTrunc(bs []byte, truncate bool) (protocol.FileIntf, error) {
func unmarshalTrunc(bs []byte, truncate bool) (FileIntf, error) {
if truncate {
var tf protocol.FileInfoTruncated
err := tf.UnmarshalXDR(bs)

32
internal/files/set.go
View File

@ -38,6 +38,22 @@ type Set struct {
blockmap *BlockMap
}
// FileIntf is the set of methods implemented by both protocol.FileInfo and
// protocol.FileInfoTruncated.
type FileIntf interface {
Size() int64
IsDeleted() bool
IsInvalid() bool
IsDirectory() bool
IsSymlink() bool
HasPermissionBits() bool
}
// The Iterator is called with either a protocol.FileInfo or a
// protocol.FileInfoTruncated (depending on the method) and returns true to
// continue iteration, false to stop.
type Iterator func(f FileIntf) bool
func NewSet(folder string, db *leveldb.DB) *Set {
var s = Set{
localVersion: make(map[protocol.DeviceID]uint64),
@ -124,42 +140,42 @@ func (s *Set) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
}
}
func (s *Set) WithNeed(device protocol.DeviceID, fn fileIterator) {
func (s *Set) WithNeed(device protocol.DeviceID, fn Iterator) {
if debug {
l.Debugf("%s WithNeed(%v)", s.folder, device)
}
ldbWithNeed(s.db, []byte(s.folder), device[:], false, nativeFileIterator(fn))
}
func (s *Set) WithNeedTruncated(device protocol.DeviceID, fn fileIterator) {
func (s *Set) WithNeedTruncated(device protocol.DeviceID, fn Iterator) {
if debug {
l.Debugf("%s WithNeedTruncated(%v)", s.folder, device)
}
ldbWithNeed(s.db, []byte(s.folder), device[:], true, nativeFileIterator(fn))
}
func (s *Set) WithHave(device protocol.DeviceID, fn fileIterator) {
func (s *Set) WithHave(device protocol.DeviceID, fn Iterator) {
if debug {
l.Debugf("%s WithHave(%v)", s.folder, device)
}
ldbWithHave(s.db, []byte(s.folder), device[:], false, nativeFileIterator(fn))
}
func (s *Set) WithHaveTruncated(device protocol.DeviceID, fn fileIterator) {
func (s *Set) WithHaveTruncated(device protocol.DeviceID, fn Iterator) {
if debug {
l.Debugf("%s WithHaveTruncated(%v)", s.folder, device)
}
ldbWithHave(s.db, []byte(s.folder), device[:], true, nativeFileIterator(fn))
}
func (s *Set) WithGlobal(fn fileIterator) {
func (s *Set) WithGlobal(fn Iterator) {
if debug {
l.Debugf("%s WithGlobal()", s.folder)
}
ldbWithGlobal(s.db, []byte(s.folder), false, nativeFileIterator(fn))
}
func (s *Set) WithGlobalTruncated(fn fileIterator) {
func (s *Set) WithGlobalTruncated(fn Iterator) {
if debug {
l.Debugf("%s WithGlobalTruncated()", s.folder)
}
@ -210,8 +226,8 @@ func normalizeFilenames(fs []protocol.FileInfo) {
}
}
func nativeFileIterator(fn fileIterator) fileIterator {
return func(fi protocol.FileIntf) bool {
func nativeFileIterator(fn Iterator) Iterator {
return func(fi FileIntf) bool {
switch f := fi.(type) {
case protocol.FileInfo:
f.Name = osutil.NativeFilename(f.Name)

6
internal/files/set_test.go
View File

@ -51,7 +51,7 @@ func genBlocks(n int) []protocol.BlockInfo {
func globalList(s *files.Set) []protocol.FileInfo {
var fs []protocol.FileInfo
s.WithGlobal(func(fi protocol.FileIntf) bool {
s.WithGlobal(func(fi FileIntf) bool {
f := fi.(protocol.FileInfo)
fs = append(fs, f)
return true
@ -61,7 +61,7 @@ func globalList(s *files.Set) []protocol.FileInfo {
func haveList(s *files.Set, n protocol.DeviceID) []protocol.FileInfo {
var fs []protocol.FileInfo
s.WithHave(n, func(fi protocol.FileIntf) bool {
s.WithHave(n, func(fi FileIntf) bool {
f := fi.(protocol.FileInfo)
fs = append(fs, f)
return true
@ -71,7 +71,7 @@ func haveList(s *files.Set, n protocol.DeviceID) []protocol.FileInfo {
func needList(s *files.Set, n protocol.DeviceID) []protocol.FileInfo {
var fs []protocol.FileInfo
s.WithNeed(n, func(fi protocol.FileIntf) bool {
s.WithNeed(n, func(fi FileIntf) bool {
f := fi.(protocol.FileInfo)
fs = append(fs, f)
return true

38
internal/model/model.go
View File

@ -310,7 +310,7 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) float64 {
return 0 // Folder doesn't exist, so we hardly have any of it
}
rf.WithGlobalTruncated(func(f protocol.FileIntf) bool {
rf.WithGlobalTruncated(func(f files.FileIntf) bool {
if !f.IsDeleted() {
tot += f.Size()
}
@ -322,7 +322,7 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) float64 {
}
var need int64
rf.WithNeedTruncated(device, func(f protocol.FileIntf) bool {
rf.WithNeedTruncated(device, func(f files.FileIntf) bool {
if !f.IsDeleted() {
need += f.Size()
}
@ -347,7 +347,7 @@ func sizeOf(fs []protocol.FileInfo) (files, deleted int, bytes int64) {
return
}
func sizeOfFile(f protocol.FileIntf) (files, deleted int, bytes int64) {
func sizeOfFile(f files.FileIntf) (files, deleted int, bytes int64) {
if !f.IsDeleted() {
files++
} else {
@ -359,15 +359,15 @@ func sizeOfFile(f protocol.FileIntf) (files, deleted int, bytes int64) {
// GlobalSize returns the number of files, deleted files and total bytes for all
// files in the global model.
func (m *Model) GlobalSize(folder string) (files, deleted int, bytes int64) {
func (m *Model) GlobalSize(folder string) (nfiles, deleted int, bytes int64) {
defer m.leveldbPanicWorkaround()
m.fmut.RLock()
defer m.fmut.RUnlock()
if rf, ok := m.folderFiles[folder]; ok {
rf.WithGlobalTruncated(func(f protocol.FileIntf) bool {
rf.WithGlobalTruncated(func(f files.FileIntf) bool {
fs, de, by := sizeOfFile(f)
files += fs
nfiles += fs
deleted += de
bytes += by
return true
@ -378,18 +378,18 @@ func (m *Model) GlobalSize(folder string) (files, deleted int, bytes int64) {
// LocalSize returns the number of files, deleted files and total bytes for all
// files in the local folder.
func (m *Model) LocalSize(folder string) (files, deleted int, bytes int64) {
func (m *Model) LocalSize(folder string) (nfiles, deleted int, bytes int64) {
defer m.leveldbPanicWorkaround()
m.fmut.RLock()
defer m.fmut.RUnlock()
if rf, ok := m.folderFiles[folder]; ok {
rf.WithHaveTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool {
rf.WithHaveTruncated(protocol.LocalDeviceID, func(f files.FileIntf) bool {
if f.IsInvalid() {
return true
}
fs, de, by := sizeOfFile(f)
files += fs
nfiles += fs
deleted += de
bytes += by
return true
@ -399,22 +399,22 @@ func (m *Model) LocalSize(folder string) (files, deleted int, bytes int64) {
}
// NeedSize returns the number and total size of currently needed files.
func (m *Model) NeedSize(folder string) (files int, bytes int64) {
func (m *Model) NeedSize(folder string) (nfiles int, bytes int64) {
defer m.leveldbPanicWorkaround()
m.fmut.RLock()
defer m.fmut.RUnlock()
if rf, ok := m.folderFiles[folder]; ok {
rf.WithNeedTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool {
rf.WithNeedTruncated(protocol.LocalDeviceID, func(f files.FileIntf) bool {
fs, de, by := sizeOfFile(f)
files += fs + de
nfiles += fs + de
bytes += by
return true
})
}
bytes -= m.progressEmitter.BytesCompleted(folder)
if debug {
l.Debugf("%v NeedSize(%q): %d %d", m, folder, files, bytes)
l.Debugf("%v NeedSize(%q): %d %d", m, folder, nfiles, bytes)
}
return
}
@ -441,21 +441,21 @@ func (m *Model) NeedFolderFiles(folder string, max int) ([]protocol.FileInfoTrun
for i, name := range progressNames {
if f, ok := rf.GetGlobal(name); ok {
progress[i] = f.ToTruncated() /// XXX: Should implement GetGlobalTruncated directly
progress[i] = protocol.Truncate(f) /// XXX: Should implement GetGlobalTruncated directly
seen[name] = true
}
}
for i, name := range queuedNames {
if f, ok := rf.GetGlobal(name); ok {
queued[i] = f.ToTruncated() /// XXX: Should implement GetGlobalTruncated directly
queued[i] = protocol.Truncate(f) /// XXX: Should implement GetGlobalTruncated directly
seen[name] = true
}
}
}
left := max - len(progress) - len(queued)
if max < 1 || left > 0 {
rf.WithNeedTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool {
rf.WithNeedTruncated(protocol.LocalDeviceID, func(f files.FileIntf) bool {
left--
ft := f.(protocol.FileInfoTruncated)
if !seen[ft.Name] {
@ -970,7 +970,7 @@ func sendIndexTo(initial bool, minLocalVer uint64, conn protocol.Connection, fol
maxLocalVer := uint64(0)
var err error
fs.WithHave(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
fs.WithHave(protocol.LocalDeviceID, func(fi files.FileIntf) bool {
f := fi.(protocol.FileInfo)
if f.LocalVersion <= minLocalVer {
return true
@ -1169,7 +1169,7 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
batch = batch[:0]
// TODO: We should limit the Have scanning to start at sub
seenPrefix := false
fs.WithHaveTruncated(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
fs.WithHaveTruncated(protocol.LocalDeviceID, func(fi files.FileIntf) bool {
f := fi.(protocol.FileInfoTruncated)
if !strings.HasPrefix(f.Name, sub) {
// Return true so that we keep iterating, until we get to the part
@ -1310,7 +1310,7 @@ func (m *Model) Override(folder string) {
m.setState(folder, FolderScanning)
batch := make([]protocol.FileInfo, 0, indexBatchSize)
fs.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
fs.WithNeed(protocol.LocalDeviceID, func(fi files.FileIntf) bool {
need := fi.(protocol.FileInfo)
if len(batch) == indexBatchSize {
fs.Update(protocol.LocalDeviceID, batch)

5
internal/model/puller.go
View File

@ -29,6 +29,7 @@ import (
"github.com/syncthing/syncthing/internal/config"
"github.com/syncthing/syncthing/internal/events"
"github.com/syncthing/syncthing/internal/files"
"github.com/syncthing/syncthing/internal/ignore"
"github.com/syncthing/syncthing/internal/osutil"
"github.com/syncthing/syncthing/internal/protocol"
@ -288,7 +289,7 @@ func (p *Puller) pullerIteration(ignores *ignore.Matcher) int {
}
p.model.fmut.RLock()
files := p.model.folderFiles[p.folder]
folderFiles := p.model.folderFiles[p.folder]
p.model.fmut.RUnlock()
// !!!
@ -301,7 +302,7 @@ func (p *Puller) pullerIteration(ignores *ignore.Matcher) int {
var deletions []protocol.FileInfo
files.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
folderFiles.WithNeed(protocol.LocalDeviceID, func(intf files.FileIntf) bool {
// Needed items are delivered sorted lexicographically. This isn't
// really optimal from a performance point of view - it would be

9
internal/protocol/message.go
View File

@ -132,15 +132,6 @@ func (f FileInfoTruncated) HasPermissionBits() bool {
return f.Flags&FlagNoPermBits == 0
}
type FileIntf interface {
Size() int64
IsDeleted() bool
IsInvalid() bool
IsDirectory() bool
IsSymlink() bool
HasPermissionBits() bool
}
type BlockInfo struct {
Offset int64 // noencode (cache only)
Size uint32