Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Only buffer file names, not full &FileInfo

This commit is contained in:
Jakob Borg 2014-12-30 09:31:34 +01:00
parent 34deb82aea
commit 2496185629
5 changed files with 68 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
internal/model/model.go
View File

@ -79,7 +79,7 @@ const (
type service interface {
Serve()
Stop()
Jobs() ([]protocol.FileInfoTruncated, []protocol.FileInfoTruncated) // In progress, Queued
Jobs() ([]string, []string) // In progress, Queued
Bump(string)
}
@ -427,20 +427,25 @@ func (m *Model) NeedFolderFiles(folder string, max int) ([]protocol.FileInfoTrun
m.fmut.RLock()
defer m.fmut.RUnlock()
if rf, ok := m.folderFiles[folder]; ok {
progress := []protocol.FileInfoTruncated{}
queued := []protocol.FileInfoTruncated{}
rest := []protocol.FileInfoTruncated{}
seen := map[string]struct{}{}
var progress, queued, rest []protocol.FileInfoTruncated
var seen map[string]bool
runner, ok := m.folderRunners[folder]
if ok {
progress, queued = runner.Jobs()
seen = make(map[string]struct{}, len(progress)+len(queued))
for _, file := range progress {
seen[file.Name] = struct{}{}
progressNames, queuedNames := runner.Jobs()
progress = make([]protocol.FileInfoTruncated, len(progressNames))
queued = make([]protocol.FileInfoTruncated, len(queuedNames))
seen = make(map[string]bool, len(progressNames)+len(queuedNames))
for i, name := range progressNames {
progress[i] = rf.GetGlobal(name).ToTruncated() /// XXX: Should implement GetGlobalTruncated directly
seen[name] = true
}
for _, file := range queued {
seen[file.Name] = struct{}{}
for i, name := range queuedNames {
queued[i] = rf.GetGlobal(name).ToTruncated() /// XXX: Should implement GetGlobalTruncated directly
seen[name] = true
}
}
left := max - len(progress) - len(queued)
@ -448,8 +453,7 @@ func (m *Model) NeedFolderFiles(folder string, max int) ([]protocol.FileInfoTrun
rf.WithNeedTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool {
left--
ft := f.(protocol.FileInfoTruncated)
_, ok := seen[ft.Name]
if !ok {
if !seen[ft.Name] {
rest = append(rest, ft)
}
return max < 1 || left > 0

15
internal/model/puller.go
View File

@ -340,7 +340,7 @@ func (p *Puller) pullerIteration(checksum bool, ignores *ignore.Matcher) int {
default:
// A new or changed file or symlink. This is the only case where we
// do stuff concurrently in the background
p.queue.Push(&file)
p.queue.Push(file.Name)
}
changed++
@ -348,11 +348,12 @@ func (p *Puller) pullerIteration(checksum bool, ignores *ignore.Matcher) int {
})
for {
f := p.queue.Pop()
if f == nil {
fileName, ok := p.queue.Pop()
if !ok {
break
}
p.handleFile(*f, copyChan, finisherChan)
f := p.model.CurrentGlobalFile(p.folder, fileName)
p.handleFile(f, copyChan, finisherChan)
}
// Signal copy and puller routines that we are done with the in data for
@ -492,7 +493,7 @@ func (p *Puller) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocksSt
if debug {
l.Debugln(p, "taking shortcut on", file.Name)
}
p.queue.Done(&file)
p.queue.Done(file.Name)
if file.IsSymlink() {
p.shortcutSymlink(curFile, file)
} else {
@ -860,7 +861,7 @@ func (p *Puller) finisherRoutine(in <-chan *sharedPullerState) {
continue
}
p.queue.Done(&state.file)
p.queue.Done(state.file.Name)
p.performFinish(state)
p.model.receivedFile(p.folder, state.file.Name)
if p.progressEmitter != nil {
@ -875,7 +876,7 @@ func (p *Puller) Bump(filename string) {
p.queue.Bump(filename)
}
func (p *Puller) Jobs() ([]protocol.FileInfoTruncated, []protocol.FileInfoTruncated) {
func (p *Puller) Jobs() ([]string, []string) {
return p.queue.Jobs()
}

43
internal/model/queue.go
View File

@ -15,15 +15,11 @@
package model
import (
"sync"
"github.com/syncthing/syncthing/internal/protocol"
)
import "sync"
type JobQueue struct {
progress []*protocol.FileInfo
queued []*protocol.FileInfo
progress []string
queued []string
mut sync.Mutex
}
@ -31,26 +27,26 @@ func NewJobQueue() *JobQueue {
return &JobQueue{}
}
func (q *JobQueue) Push(file *protocol.FileInfo) {
func (q *JobQueue) Push(file string) {
q.mut.Lock()
q.queued = append(q.queued, file)
q.mut.Unlock()
}
func (q *JobQueue) Pop() *protocol.FileInfo {
func (q *JobQueue) Pop() (string, bool) {
q.mut.Lock()
defer q.mut.Unlock()
if len(q.queued) == 0 {
return nil
return "", false
}
var f *protocol.FileInfo
f, q.queued[0] = q.queued[0], nil
var f string
f = q.queued[0]
q.queued = q.queued[1:]
q.progress = append(q.progress, f)
return f
return f, true
}
func (q *JobQueue) Bump(filename string) {
@ -58,40 +54,35 @@ func (q *JobQueue) Bump(filename string) {
defer q.mut.Unlock()
for i := range q.queued {
if q.queued[i].Name == filename {
if q.queued[i] == filename {
q.queued[0], q.queued[i] = q.queued[i], q.queued[0]
return
}
}
}
func (q *JobQueue) Done(file *protocol.FileInfo) {
func (q *JobQueue) Done(file string) {
q.mut.Lock()
defer q.mut.Unlock()
for i := range q.progress {
if q.progress[i].Name == file.Name {
if q.progress[i] == file {
copy(q.progress[i:], q.progress[i+1:])
q.progress[len(q.progress)-1] = nil
q.progress = q.progress[:len(q.progress)-1]
return
}
}
}
func (q *JobQueue) Jobs() ([]protocol.FileInfoTruncated, []protocol.FileInfoTruncated) {
func (q *JobQueue) Jobs() ([]string, []string) {
q.mut.Lock()
defer q.mut.Unlock()
progress := make([]protocol.FileInfoTruncated, len(q.progress))
for i := range q.progress {
progress[i] = q.progress[i].ToTruncated()
}
progress := make([]string, len(q.progress))
copy(progress, q.progress)
queued := make([]protocol.FileInfoTruncated, len(q.queued))
for i := range q.queued {
queued[i] = q.queued[i].ToTruncated()
}
queued := make([]string, len(q.queued))
copy(queued, q.queued)
return progress, queued
}

58
internal/model/queue_test.go
View File

@ -18,25 +18,15 @@ package model
import (
"fmt"
"testing"
"github.com/syncthing/syncthing/internal/protocol"
)
var (
f1 = &protocol.FileInfo{Name: "f1"}
f2 = &protocol.FileInfo{Name: "f2"}
f3 = &protocol.FileInfo{Name: "f3"}
f4 = &protocol.FileInfo{Name: "f4"}
f5 = &protocol.FileInfo{Name: "f5"}
)
func TestJobQueue(t *testing.T) {
// Some random actions
q := NewJobQueue()
q.Push(f1)
q.Push(f2)
q.Push(f3)
q.Push(f4)
q.Push("f1")
q.Push("f2")
q.Push("f3")
q.Push("f4")
progress, queued := q.Jobs()
if len(progress) != 0 || len(queued) != 4 {
@ -44,8 +34,8 @@ func TestJobQueue(t *testing.T) {
}
for i := 1; i < 5; i++ {
n := q.Pop()
if n == nil || n.Name != fmt.Sprintf("f%d", i) {
n, ok := q.Pop()
if !ok || n != fmt.Sprintf("f%d", i) {
t.Fatal("Wrong element")
}
progress, queued = q.Jobs()
@ -65,7 +55,7 @@ func TestJobQueue(t *testing.T) {
t.Fatal("Wrong length")
}
q.Done(f5) // Does not exist
q.Done("f5") // Does not exist
progress, queued = q.Jobs()
if len(progress) != 0 || len(queued) != 4 {
t.Fatal("Wrong length")
@ -90,8 +80,8 @@ func TestJobQueue(t *testing.T) {
t.Fatal("Wrong length")
}
n := q.Pop()
if n == nil || n.Name != s {
n, ok := q.Pop()
if !ok || n != s {
t.Fatal("Wrong element")
}
progress, queued = q.Jobs()
@ -99,24 +89,26 @@ func TestJobQueue(t *testing.T) {
t.Fatal("Wrong length")
}
q.Done(f5) // Does not exist
q.Done("f5") // Does not exist
progress, queued = q.Jobs()
if len(progress) != 5-i || len(queued) != i-1 {
t.Fatal("Wrong length")
}
}
if len(q.progress) != 4 || q.Pop() != nil {
_, ok := q.Pop()
if len(q.progress) != 4 || ok {
t.Fatal("Wrong length")
}
q.Done(f1)
q.Done(f2)
q.Done(f3)
q.Done(f4)
q.Done(f5) // Does not exist
q.Done("f1")
q.Done("f2")
q.Done("f3")
q.Done("f4")
q.Done("f5") // Does not exist
if len(q.progress) != 0 || q.Pop() != nil {
_, ok = q.Pop()
if len(q.progress) != 0 || ok {
t.Fatal("Wrong length")
}
@ -125,7 +117,7 @@ func TestJobQueue(t *testing.T) {
t.Fatal("Wrong length")
}
q.Bump("")
q.Done(f5) // Does not exist
q.Done("f5") // Does not exist
progress, queued = q.Jobs()
if len(progress) != 0 || len(queued) != 0 {
t.Fatal("Wrong length")
@ -178,8 +170,8 @@ func BenchmarkJobQueueBump(b *testing.B) {
files := genFiles(b.N)
q := NewJobQueue()
for j := range files {
q.Push(&files[j])
for _, f := range files {
q.Push(f.Name)
}
b.ResetTimer()
@ -194,11 +186,11 @@ func BenchmarkJobQueuePushPopDone10k(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
q := NewJobQueue()
for j := range files {
q.Push(&files[j])
for _, f := range files {
q.Push(f.Name)
}
for range files {
n := q.Pop()
n, _ := q.Pop()
q.Done(n)
}
}

4
internal/model/scanner.go
View File

@ -18,8 +18,6 @@ package model
import (
"fmt"
"time"
"github.com/syncthing/syncthing/internal/protocol"
)
type Scanner struct {
@ -80,6 +78,6 @@ func (s *Scanner) String() string {
func (s *Scanner) Bump(string) {}
func (s *Scanner) Jobs() ([]protocol.FileInfoTruncated, []protocol.FileInfoTruncated) {
func (s *Scanner) Jobs() ([]string, []string) {
return nil, nil
}