Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

cmd/syncthing, lib/db, lib/model: Track more detailed file/dirs/links/deleted counts

This commit is contained in:
Jakob Borg 2016-10-17 14:10:17 +02:00
parent b8a90b7eaa
commit 4e8c8d7e2c
8 changed files with 143 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
cmd/syncthing/gui.go
View File

@ -72,7 +72,7 @@ type modelIntf interface {
Completion(device protocol.DeviceID, folder string) model.FolderCompletion
Override(folder string)
NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated, int)
NeedSize(folder string) (nfiles, ndeletes int, bytes int64)
NeedSize(folder string) db.Counts
ConnectionStats() map[string]interface{}
DeviceStatistics() map[string]stats.DeviceStatistics
FolderStatistics() map[string]stats.FolderStatistics
@ -90,8 +90,8 @@ type modelIntf interface {
ScanFolderSubdirs(folder string, subs []string) error
BringToFront(folder, file string)
ConnectedTo(deviceID protocol.DeviceID) bool
GlobalSize(folder string) (nfiles, deleted int, bytes int64)
LocalSize(folder string) (nfiles, deleted int, bytes int64)
GlobalSize(folder string) db.Counts
LocalSize(folder string) db.Counts
CurrentSequence(folder string) (int64, bool)
RemoteSequence(folder string) (int64, bool)
State(folder string) (string, time.Time, error)
@ -634,16 +634,16 @@ func folderSummary(cfg configIntf, m modelIntf, folder string) map[string]interf
res["invalid"] = "" // Deprecated, retains external API for now
globalFiles, globalDeleted, globalBytes := m.GlobalSize(folder)
res["globalFiles"], res["globalDeleted"], res["globalBytes"] = globalFiles, globalDeleted, globalBytes
global := m.GlobalSize(folder)
res["globalFiles"], res["globalDirectories"], res["globalSymlinks"], res["globalDeleted"], res["globalBytes"] = global.Files, global.Directories, global.Symlinks, global.Deleted, global.Bytes
localFiles, localDeleted, localBytes := m.LocalSize(folder)
res["localFiles"], res["localDeleted"], res["localBytes"] = localFiles, localDeleted, localBytes
local := m.LocalSize(folder)
res["localFiles"], res["localDirectories"], res["localSymlinks"], res["localDeleted"], res["localBytes"] = local.Files, local.Directories, local.Symlinks, local.Deleted, local.Bytes
needFiles, needDeletes, needBytes := m.NeedSize(folder)
res["needFiles"], res["needDeletes"], res["needBytes"] = needFiles, needDeletes, needBytes
need := m.NeedSize(folder)
res["needFiles"], res["needDirectories"], res["needSymlinks"], res["needDeletes"], res["needBytes"] = need.Files, need.Directories, need.Symlinks, need.Deleted, need.Bytes
res["inSyncFiles"], res["inSyncBytes"] = globalFiles-needFiles, globalBytes-needBytes
res["inSyncFiles"], res["inSyncBytes"] = global.Files-need.Files, global.Bytes-need.Bytes
var err error
res["state"], res["stateChanged"], err = m.State(folder)

12
cmd/syncthing/mocked_model_test.go
View File

@ -31,8 +31,8 @@ func (m *mockedModel) NeedFolderFiles(folder string, page, perpage int) ([]db.Fi
return nil, nil, nil, 0
}
func (m *mockedModel) NeedSize(folder string) (nfiles, ndeletes int, bytes int64) {
return 0, 0, 0
func (m *mockedModel) NeedSize(folder string) db.Counts {
return db.Counts{}
}
func (m *mockedModel) ConnectionStats() map[string]interface{} {
@ -95,12 +95,12 @@ func (m *mockedModel) ConnectedTo(deviceID protocol.DeviceID) bool {
return false
}
func (m *mockedModel) GlobalSize(folder string) (nfiles, deleted int, bytes int64) {
return 0, 0, 0
func (m *mockedModel) GlobalSize(folder string) db.Counts {
return db.Counts{}
}
func (m *mockedModel) LocalSize(folder string) (nfiles, deleted int, bytes int64) {
return 0, 0, 0
func (m *mockedModel) LocalSize(folder string) db.Counts {
return db.Counts{}
}
func (m *mockedModel) CurrentSequence(folder string) (int64, bool) {

14
cmd/syncthing/usage_report.go
View File

@ -93,14 +93,14 @@ func reportData(cfg configIntf, m modelIntf) map[string]interface{} {
var totFiles, maxFiles int
var totBytes, maxBytes int64
for folderID := range cfg.Folders() {
files, _, bytes := m.GlobalSize(folderID)
totFiles += files
totBytes += bytes
if files > maxFiles {
maxFiles = files
global := m.GlobalSize(folderID)
totFiles += global.Files
totBytes += global.Bytes
if global.Files > maxFiles {
maxFiles = global.Files
}
if bytes > maxBytes {
maxBytes = bytes
if global.Bytes > maxBytes {
maxBytes = global.Bytes
}
}

14
gui/default/index.html
View File

@ -310,16 +310,24 @@
</tr>
<tr>
<th><span class="fa fa-fw fa-globe"></span>&nbsp;<span translate>Global State</span></th>
<td class="text-right">{{model[folder.id].globalFiles | alwaysNumber}} <span translate>items</span>, ~{{model[folder.id].globalBytes | binary}}B</td>
<td class="text-right">
{{model[folder.id].globalFiles | alwaysNumber}} <span translate>files</span>,
{{model[folder.id].globalDirectories | alwaysNumber}} <span translate>directories</span>,
~{{model[folder.id].globalBytes | binary}}B
</td>
</tr>
<tr>
<th><span class="fa fa-fw fa-home"></span>&nbsp;<span translate>Local State</span></th>
<td class="text-right">{{model[folder.id].localFiles | alwaysNumber}} <span translate>items</span>, ~{{model[folder.id].localBytes | binary}}B</td>
<td class="text-right">
{{model[folder.id].localFiles | alwaysNumber}} <span translate>files</span>,
{{model[folder.id].localDirectories | alwaysNumber}} <span translate>directories</span>,
~{{model[folder.id].localBytes | binary}}B
</td>
</tr>
<tr ng-if="model[folder.id].needFiles > 0">
<th><span class="fa fa-fw fa-cloud-download"></span>&nbsp;<span translate>Out of Sync Items</span></th>
<td class="text-right">
<a href="" ng-click="showNeed(folder.id)">{{model[folder.id].needFiles | alwaysNumber}} <span translate>items</span>, ~{{model[folder.id].needBytes | binary}}B</a>
<a href="" ng-click="showNeed(folder.id)">{{model[folder.id].needFiles+model[folder.id].needDirectories+model[folder.id].needSymlinks+model[folder.id].needDeletes | alwaysNumber}} <span translate>items</span>, ~{{model[folder.id].needBytes | binary}}B</a>
</td>
</tr>
<tr ng-if="folderStatus(folder) === 'scanning' && scanRate(folder.id) > 0">

54
lib/db/set.go
View File

@ -51,11 +51,17 @@ type FileIntf interface {
// continue iteration, false to stop.
type Iterator func(f FileIntf) bool
type Counts struct {
Files int
Directories int
Symlinks int
Deleted int
Bytes int64
}
type sizeTracker struct {
files int
deleted int
bytes int64
mut stdsync.Mutex
Counts
mut stdsync.Mutex
}
func (s *sizeTracker) addFile(f FileIntf) {
@ -64,12 +70,17 @@ func (s *sizeTracker) addFile(f FileIntf) {
}
s.mut.Lock()
if f.IsDeleted() {
s.deleted++
} else {
s.files++
switch {
case f.IsDeleted():
s.Deleted++
case f.IsDirectory():
s.Directories++
case f.IsSymlink():
s.Symlinks++
default:
s.Files++
}
s.bytes += f.FileSize()
s.Bytes += f.FileSize()
s.mut.Unlock()
}
@ -79,22 +90,27 @@ func (s *sizeTracker) removeFile(f FileIntf) {
}
s.mut.Lock()
if f.IsDeleted() {
s.deleted--
} else {
s.files--
switch {
case f.IsDeleted():
s.Deleted--
case f.IsDirectory():
s.Directories--
case f.IsSymlink():
s.Symlinks--
default:
s.Files--
}
s.bytes -= f.FileSize()
if s.deleted < 0 || s.files < 0 {
s.Bytes -= f.FileSize()
if s.Deleted < 0 || s.Files < 0 || s.Directories < 0 || s.Symlinks < 0 {
panic("bug: removed more than added")
}
s.mut.Unlock()
}
func (s *sizeTracker) Size() (files, deleted int, bytes int64) {
func (s *sizeTracker) Size() Counts {
s.mut.Lock()
defer s.mut.Unlock()
return s.files, s.deleted, s.bytes
return s.Counts
}
func NewFileSet(folder string, db *Instance) *FileSet {
@ -259,11 +275,11 @@ func (s *FileSet) Sequence(device protocol.DeviceID) int64 {
return s.remoteSequence[device]
}
func (s *FileSet) LocalSize() (files, deleted int, bytes int64) {
func (s *FileSet) LocalSize() Counts {
return s.localSize.Size()
}
func (s *FileSet) GlobalSize() (files, deleted int, bytes int64) {
func (s *FileSet) GlobalSize() Counts {
return s.globalSize.Size()
}

52
lib/db/set_test.go
View File

@ -168,27 +168,33 @@ func TestGlobalSet(t *testing.T) {
t.Errorf("Global incorrect;\n A: %v !=\n E: %v", g, expectedGlobal)
}
globalFiles, globalDeleted, globalBytes := 0, 0, int64(0)
globalFiles, globalDirectories, globalDeleted, globalBytes := 0, 0, 0, int64(0)
for _, f := range g {
if f.IsInvalid() {
continue
}
if f.IsDeleted() {
switch {
case f.IsDeleted():
globalDeleted++
} else {
case f.IsDirectory():
globalDirectories++
default:
globalFiles++
}
globalBytes += f.FileSize()
}
gsFiles, gsDeleted, gsBytes := m.GlobalSize()
if gsFiles != globalFiles {
t.Errorf("Incorrect GlobalSize files; %d != %d", gsFiles, globalFiles)
gs := m.GlobalSize()
if gs.Files != globalFiles {
t.Errorf("Incorrect GlobalSize files; %d != %d", gs.Files, globalFiles)
}
if gsDeleted != globalDeleted {
t.Errorf("Incorrect GlobalSize deleted; %d != %d", gsDeleted, globalDeleted)
if gs.Directories != globalDirectories {
t.Errorf("Incorrect GlobalSize directories; %d != %d", gs.Directories, globalDirectories)
}
if gsBytes != globalBytes {
t.Errorf("Incorrect GlobalSize bytes; %d != %d", gsBytes, globalBytes)
if gs.Deleted != globalDeleted {
t.Errorf("Incorrect GlobalSize deleted; %d != %d", gs.Deleted, globalDeleted)
}
if gs.Bytes != globalBytes {
t.Errorf("Incorrect GlobalSize bytes; %d != %d", gs.Bytes, globalBytes)
}
h := fileList(haveList(m, protocol.LocalDeviceID))
@ -198,27 +204,33 @@ func TestGlobalSet(t *testing.T) {
t.Errorf("Have incorrect;\n A: %v !=\n E: %v", h, localTot)
}
haveFiles, haveDeleted, haveBytes := 0, 0, int64(0)
haveFiles, haveDirectories, haveDeleted, haveBytes := 0, 0, 0, int64(0)
for _, f := range h {
if f.IsInvalid() {
continue
}
if f.IsDeleted() {
switch {
case f.IsDeleted():
haveDeleted++
} else {
case f.IsDirectory():
haveDirectories++
default:
haveFiles++
}
haveBytes += f.FileSize()
}
lsFiles, lsDeleted, lsBytes := m.LocalSize()
if lsFiles != haveFiles {
t.Errorf("Incorrect LocalSize files; %d != %d", lsFiles, haveFiles)
ls := m.LocalSize()
if ls.Files != haveFiles {
t.Errorf("Incorrect LocalSize files; %d != %d", ls.Files, haveFiles)
}
if lsDeleted != haveDeleted {
t.Errorf("Incorrect LocalSize deleted; %d != %d", lsDeleted, haveDeleted)
if ls.Directories != haveDirectories {
t.Errorf("Incorrect LocalSize directories; %d != %d", ls.Directories, haveDirectories)
}
if lsBytes != haveBytes {
t.Errorf("Incorrect LocalSize bytes; %d != %d", lsBytes, haveBytes)
if ls.Deleted != haveDeleted {
t.Errorf("Incorrect LocalSize deleted; %d != %d", ls.Deleted, haveDeleted)
}
if ls.Bytes != haveBytes {
t.Errorf("Incorrect LocalSize bytes; %d != %d", ls.Bytes, haveBytes)
}
h = fileList(haveList(m, remoteDevice0))

46
lib/model/model.go
View File

@ -476,7 +476,7 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) FolderComple
return FolderCompletion{} // Folder doesn't exist, so we hardly have any of it
}
_, _, tot := rf.GlobalSize()
tot := rf.GlobalSize().Bytes
if tot == 0 {
// Folder is empty, so we have all of it
return FolderCompletion{
@ -531,42 +531,49 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) FolderComple
}
}
func sizeOfFile(f db.FileIntf) (files, deleted int, bytes int64) {
if !f.IsDeleted() {
files++
} else {
deleted++
func addSizeOfFile(s *db.Counts, f db.FileIntf) {
switch {
case f.IsDeleted():
s.Deleted++
case f.IsDirectory():
s.Directories++
case f.IsSymlink():
s.Symlinks++
default:
s.Files++
}
bytes += f.FileSize()
s.Bytes += f.FileSize()
return
}
// GlobalSize returns the number of files, deleted files and total bytes for all
// files in the global model.
func (m *Model) GlobalSize(folder string) (nfiles, deleted int, bytes int64) {
func (m *Model) GlobalSize(folder string) db.Counts {
m.fmut.RLock()
defer m.fmut.RUnlock()
if rf, ok := m.folderFiles[folder]; ok {
nfiles, deleted, bytes = rf.GlobalSize()
return rf.GlobalSize()
}
return
return db.Counts{}
}
// LocalSize returns the number of files, deleted files and total bytes for all
// files in the local folder.
func (m *Model) LocalSize(folder string) (nfiles, deleted int, bytes int64) {
func (m *Model) LocalSize(folder string) db.Counts {
m.fmut.RLock()
defer m.fmut.RUnlock()
if rf, ok := m.folderFiles[folder]; ok {
nfiles, deleted, bytes = rf.LocalSize()
return rf.LocalSize()
}
return
return db.Counts{}
}
// NeedSize returns the number and total size of currently needed files.
func (m *Model) NeedSize(folder string) (nfiles, ndeletes int, bytes int64) {
func (m *Model) NeedSize(folder string) db.Counts {
m.fmut.RLock()
defer m.fmut.RUnlock()
var result db.Counts
if rf, ok := m.folderFiles[folder]; ok {
ignores := m.folderIgnores[folder]
cfg := m.folderCfgs[folder]
@ -575,16 +582,13 @@ func (m *Model) NeedSize(folder string) (nfiles, ndeletes int, bytes int64) {
return true
}
fs, de, by := sizeOfFile(f)
nfiles += fs
ndeletes += de
bytes += by
addSizeOfFile(&result, f)
return true
})
}
bytes -= m.progressEmitter.BytesCompleted(folder)
l.Debugf("%v NeedSize(%q): %d %d", m, folder, nfiles, bytes)
return
result.Bytes -= m.progressEmitter.BytesCompleted(folder)
l.Debugf("%v NeedSize(%q): %v", m, folder, result)
return result
}
// NeedFolderFiles returns paginated list of currently needed files in

34
lib/model/model_test.go
View File

@ -1324,8 +1324,8 @@ func TestIssue3028(t *testing.T) {
// Get a count of how many files are there now
locorigfiles, _, _ := m.LocalSize("default")
globorigfiles, _, _ := m.GlobalSize("default")
locorigfiles := m.LocalSize("default").Files
globorigfiles := m.GlobalSize("default").Files
// Delete and rescan specifically these two
@ -1336,19 +1336,19 @@ func TestIssue3028(t *testing.T) {
// Verify that the number of files decreased by two and the number of
// deleted files increases by two
locnowfiles, locdelfiles, _ := m.LocalSize("default")
globnowfiles, globdelfiles, _ := m.GlobalSize("default")
if locnowfiles != locorigfiles-2 {
t.Errorf("Incorrect local accounting; got %d current files, expected %d", locnowfiles, locorigfiles-2)
loc := m.LocalSize("default")
glob := m.GlobalSize("default")
if loc.Files != locorigfiles-2 {
t.Errorf("Incorrect local accounting; got %d current files, expected %d", loc.Files, locorigfiles-2)
}
if globnowfiles != globorigfiles-2 {
t.Errorf("Incorrect global accounting; got %d current files, expected %d", globnowfiles, globorigfiles-2)
if glob.Files != globorigfiles-2 {
t.Errorf("Incorrect global accounting; got %d current files, expected %d", glob.Files, globorigfiles-2)
}
if locdelfiles != 2 {
t.Errorf("Incorrect local accounting; got %d deleted files, expected 2", locdelfiles)
if loc.Deleted != 2 {
t.Errorf("Incorrect local accounting; got %d deleted files, expected 2", loc.Deleted)
}
if globdelfiles != 2 {
t.Errorf("Incorrect global accounting; got %d deleted files, expected 2", globdelfiles)
if glob.Deleted != 2 {
t.Errorf("Incorrect global accounting; got %d deleted files, expected 2", glob.Deleted)
}
}
@ -1722,14 +1722,14 @@ func TestIssue3496(t *testing.T) {
t.Log(comp)
// Check that NeedSize does the correct thing
files, deletes, bytes := m.NeedSize("default")
if files != 1 || bytes != 1234 {
need := m.NeedSize("default")
if need.Files != 1 || need.Bytes != 1234 {
// The one we added synthetically above
t.Errorf("Incorrect need size; %d, %d != 1, 1234", files, bytes)
t.Errorf("Incorrect need size; %d, %d != 1, 1234", need.Files, need.Bytes)
}
if deletes != len(localFiles)-1 {
if need.Deleted != len(localFiles)-1 {
// The rest
t.Errorf("Incorrect need deletes; %d != %d", deletes, len(localFiles)-1)
t.Errorf("Incorrect need deletes; %d != %d", need.Deleted, len(localFiles)-1)
}
}