Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/config, lib/model: Limit concurrent pulls (fixes #5914) (#6290) 

Adds a new folder state "Waiting to Sync" in the same vein as the
existing "Waiting to Scan". This vastly improves performances in the
rare cases when there are lots and lots of folders operating.
This commit is contained in:
Jakob Borg 2020-01-27 17:31:17 +01:00 committed by GitHub
parent 84920bff63
commit d91c4b010b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
34 changed files with 118 additions and 362 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
gui/default/index.html
View File

@ -313,7 +313,7 @@
<span ng-switch-when="paused"><span class="hidden-xs" translate>Paused</span><span class="visible-xs" aria-label="{{'Paused' | translate}}"><i class="fas fa-fw fa-pause"></i></span></span>
<span ng-switch-when="unknown"><span class="hidden-xs" translate>Unknown</span><span class="visible-xs" aria-label="{{'Unknown' | translate}}"><i class="fas fa-fw fa-question-circle"></i></span></span>
<span ng-switch-when="unshared"><span class="hidden-xs" translate>Unshared</span><span class="visible-xs" aria-label="{{'Unshared' | translate}}"><i class="fas fa-fw fa-unlink"></i></span></span>
<span ng-switch-when="scan-waiting"><span class="hidden-xs" translate>Waiting to scan</span><span class="visible-xs" aria-label="{{'Waiting to scan' | translate}}"><i class="fas fa-fw fa-hourglass-half"></i></span></span>
<span ng-switch-when="scan-waiting"><span class="hidden-xs" translate>Waiting to Scan</span><span class="visible-xs" aria-label="{{'Waiting to Scan' | translate}}"><i class="fas fa-fw fa-hourglass-half"></i></span></span>
<span ng-switch-when="stopped"><span class="hidden-xs" translate>Stopped</span><span class="visible-xs" aria-label="{{'Stopped' | translate}}"><i class="fas fa-fw fa-stop"></i></span></span>
<span ng-switch-when="scanning">
<span class="hidden-xs" translate>Scanning</span>
@ -324,6 +324,10 @@
</span>
<span ng-switch-when="idle"><span class="hidden-xs" translate>Up to Date</span><span class="visible-xs" aria-label="{{'Up to Date' | translate}}"><i class="fas fa-fw fa-check"></i></span></span>
<span ng-switch-when="localadditions"><span class="hidden-xs" translate>Local Additions</span><span class="visible-xs" aria-label="{{'Local Additions' | translate}}"><i class="fas fa-fw fa-check"></i></span></span>
<span ng-switch-when="sync-waiting">
<span class="hidden-xs" translate>Waiting to Sync</span>
<span class="visible-xs" aria-label="{{'Waiting to Sync' | translate}}"><i class="fas fa-fw fa-hourglass-half"></i></span>
</span>
<span ng-switch-when="sync-preparing">
<span class="hidden-xs" translate>Preparing to Sync</span>
<span class="visible-xs" aria-label="{{'Preparing to Sync' | translate}}"><i class="fas fa-fw fa-hourglass-half"></i></span>

2
gui/default/syncthing/core/syncthingController.js
View File

@ -837,7 +837,7 @@ angular.module('syncthing.core')
if (status === 'stopped' || status === 'outofsync' || status === 'error' || status === 'faileditems') {
return 'danger';
}
if (status === 'unshared' || status === 'scan-waiting') {
if (status === 'unshared' || status === 'scan-waiting' || status === 'sync-waiting') {
return 'warning';
}

2
lib/config/config.go
View File

@ -31,7 +31,7 @@ import (
const (
OldestHandledVersion = 10
CurrentVersion = 29
CurrentVersion = 30
MaxRescanIntervalS = 365 * 24 * 60 * 60
)

28
lib/config/config_test.go
View File

@ -86,8 +86,13 @@ func TestDefaultValues(t *testing.T) {
func TestDeviceConfig(t *testing.T) {
for i := OldestHandledVersion; i <= CurrentVersion; i++ {
cfgFile := fmt.Sprintf("testdata/v%d.xml", i)
if _, err := os.Stat(cfgFile); os.IsNotExist(err) {
continue
}
os.RemoveAll(filepath.Join("testdata", DefaultMarkerName))
wr, err := load(fmt.Sprintf("testdata/v%d.xml", i), device1)
wr, err := load(cfgFile, device1)
if err != nil {
t.Fatal(err)
}
@ -1127,6 +1132,27 @@ func TestRemoveDeviceWithEmptyID(t *testing.T) {
}
}
func TestMaxConcurrentFolders(t *testing.T) {
cases := []struct {
input int
output int
}{
{input: -42, output: 0},
{input: -1, output: 0},
{input: 0, output: runtime.GOMAXPROCS(-1)},
{input: 1, output: 1},
{input: 42, output: 42},
}
for _, tc := range cases {
opts := OptionsConfiguration{RawMaxFolderConcurrency: tc.input}
res := opts.MaxFolderConcurrency()
if res != tc.output {
t.Errorf("Wrong MaxFolderConcurrency, %d => %d, expected %d", tc.input, res, tc.output)
}
}
}
// defaultConfigAsMap returns a valid default config as a JSON-decoded
// map[string]interface{}. This is useful to override random elements and
// re-encode into JSON.

8
lib/config/migrations.go
View File

@ -25,6 +25,7 @@ import (
// update the config version. The order of migrations doesn't matter here,
// put the newest on top for readability.
var migrations = migrationSet{
{30, migrateToConfigV30},
{29, migrateToConfigV29},
{28, migrateToConfigV28},
{27, migrateToConfigV27},
@ -84,6 +85,13 @@ func (m migration) apply(cfg *Configuration) {
cfg.Version = m.targetVersion
}
func migrateToConfigV30(cfg *Configuration) {
// The "max concurrent scans" option is now spelled "max folder concurrency"
// to be more general.
cfg.Options.RawMaxFolderConcurrency = cfg.Options.DeprecatedMaxConcurrentScans
cfg.Options.DeprecatedMaxConcurrentScans = 0
}
func migrateToConfigV29(cfg *Configuration) {
// The new crash reporting option should follow the state of global
// discovery / usage reporting, and we should display an appropriate

25
lib/config/optionsconfiguration.go
View File

@ -8,6 +8,7 @@ package config
import (
"fmt"
"runtime"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/util"
@ -52,7 +53,7 @@ type OptionsConfiguration struct {
TrafficClass int `xml:"trafficClass" json:"trafficClass"`
DefaultFolderPath string `xml:"defaultFolderPath" json:"defaultFolderPath" default:"~"`
SetLowPriority bool `xml:"setLowPriority" json:"setLowPriority" default:"true"`
MaxConcurrentScans int `xml:"maxConcurrentScans" json:"maxConcurrentScans"`
RawMaxFolderConcurrency int `xml:"maxFolderConcurrency" json:"maxFolderConcurrency"`
CRURL string `xml:"crashReportingURL" json:"crURL" default:"https://crash.syncthing.net/newcrash"` // crash reporting URL
CREnabled bool `xml:"crashReportingEnabled" json:"crashReportingEnabled" default:"true" restart:"true"`
StunKeepaliveStartS int `xml:"stunKeepaliveStartS" json:"stunKeepaliveStartS" default:"180"` // 0 for off
@ -66,6 +67,7 @@ type OptionsConfiguration struct {
DeprecatedUPnPTimeoutS int `xml:"upnpTimeoutSeconds,omitempty" json:"-"`
DeprecatedRelayServers []string `xml:"relayServer,omitempty" json:"-"`
DeprecatedMinHomeDiskFreePct float64 `xml:"minHomeDiskFreePct,omitempty" json:"-"`
DeprecatedMaxConcurrentScans int `xml:"maxConcurrentScans,omitempty" json:"-"`
}
func (opts OptionsConfiguration) Copy() OptionsConfiguration {
@ -152,3 +154,24 @@ func (opts OptionsConfiguration) GlobalDiscoveryServers() []string {
}
return util.UniqueTrimmedStrings(servers)
}
func (opts OptionsConfiguration) MaxFolderConcurrency() int {
// If a value is set, trust that.
if opts.RawMaxFolderConcurrency > 0 {
return opts.RawMaxFolderConcurrency
}
if opts.RawMaxFolderConcurrency < 0 {
// -1 etc means unlimited, which in the implementation means zero
return 0
}
// Otherwise default to the number of CPU cores in the system as a rough
// approximation of system powerfullness.
if n := runtime.GOMAXPROCS(-1); n > 0 {
return n
}
// We should never get here to begin with, but since we're here let's
// use some sort of reasonable compromise between the old "no limit" and
// getting nothing done... (Median number of folders out there at time
// of writing is two, 95-percentile at 12 folders.)
return 4 // https://xkcd.com/221/
}

12
lib/config/testdata/v10.xml vendored
View File

@ -1,12 +0,0 @@
<configuration version="10">
<folder id="test" path="testdata" ro="true" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>b</address>
</device>
</configuration>

13
lib/config/testdata/v11.xml vendored
View File

@ -1,13 +0,0 @@
<configuration version="11">
<folder id="test" path="testdata" ro="true" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFreePct>1</minDiskFreePct>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>b</address>
</device>
</configuration>

14
lib/config/testdata/v12.xml vendored
View File

@ -1,14 +0,0 @@
<configuration version="12">
<folder id="test" path="testdata" ro="true" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFreePct>1</minDiskFreePct>
<maxConflicts>-1</maxConflicts>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

14
lib/config/testdata/v13.xml vendored
View File

@ -1,14 +0,0 @@
<configuration version="13">
<folder id="test" path="testdata" ro="true" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFreePct>1</minDiskFreePct>
<maxConflicts>-1</maxConflicts>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

14
lib/config/testdata/v14.xml vendored
View File

@ -1,14 +0,0 @@
<configuration version="14">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFreePct>1</minDiskFreePct>
<maxConflicts>-1</maxConflicts>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

14
lib/config/testdata/v15.xml vendored
View File

@ -1,14 +0,0 @@
<configuration version="15">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFreePct>1</minDiskFreePct>
<maxConflicts>-1</maxConflicts>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

14
lib/config/testdata/v16.xml vendored
View File

@ -1,14 +0,0 @@
<configuration version="16">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFreePct>1</minDiskFreePct>
<maxConflicts>-1</maxConflicts>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

15
lib/config/testdata/v17.xml vendored
View File

@ -1,15 +0,0 @@
<configuration version="17">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFreePct>1</minDiskFreePct>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

15
lib/config/testdata/v18.xml vendored
View File

@ -1,15 +0,0 @@
<configuration version="18">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFreePct>1</minDiskFreePct>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

15
lib/config/testdata/v19.xml vendored
View File

@ -1,15 +0,0 @@
<configuration version="19">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFreePct>1</minDiskFreePct>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

15
lib/config/testdata/v20.xml vendored
View File

@ -1,15 +0,0 @@
<configuration version="20">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFree unit="%">1</minDiskFree>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

15
lib/config/testdata/v21.xml vendored
View File

@ -1,15 +0,0 @@
<configuration version="21">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFree unit="%">1</minDiskFree>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

16
lib/config/testdata/v23.xml vendored
View File

@ -1,16 +0,0 @@
<configuration version="22">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<filesystemType>basic</filesystemType>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFree unit="%">1</minDiskFree>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

16
lib/config/testdata/v24.xml vendored
View File

@ -1,16 +0,0 @@
<configuration version="22">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<filesystemType>basic</filesystemType>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFree unit="%">1</minDiskFree>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

16
lib/config/testdata/v25.xml vendored
View File

@ -1,16 +0,0 @@
<configuration version="25">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" fsWatcherEnabled="false" fsWatcherDelayS="10" autoNormalize="true">
<filesystemType>basic</filesystemType>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFree unit="%">1</minDiskFree>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

16
lib/config/testdata/v26.xml vendored
View File

@ -1,16 +0,0 @@
<configuration version="26">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" fsWatcherEnabled="false" fsWatcherDelayS="10" autoNormalize="true">
<filesystemType>basic</filesystemType>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFree unit="%">1</minDiskFree>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

16
lib/config/testdata/v27.xml vendored
View File

@ -1,16 +0,0 @@
<configuration version="27">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" fsWatcherEnabled="false" fsWatcherDelayS="10" autoNormalize="true">
<filesystemType>basic</filesystemType>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFree unit="%">1</minDiskFree>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

16
lib/config/testdata/v28.xml vendored
View File

@ -1,16 +0,0 @@
<configuration version="28">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" fsWatcherEnabled="false" fsWatcherDelayS="10" autoNormalize="true">
<filesystemType>basic</filesystemType>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFree unit="%">1</minDiskFree>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

16
lib/config/testdata/v29.xml vendored
View File

@ -1,16 +0,0 @@
<configuration version="28">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" fsWatcherEnabled="false" fsWatcherDelayS="10" autoNormalize="true">
<filesystemType>basic</filesystemType>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFree unit="%">1</minDiskFree>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

12
lib/config/testdata/v6.xml vendored
View File

@ -1,12 +0,0 @@
<configuration version="6">
<folder id="test" path="testdata" ro="true" ignorePerms="false" rescanIntervalS="600">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="true">
<address>a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="true">
<address>b</address>
</device>
</configuration>

12
lib/config/testdata/v7.xml vendored
View File

@ -1,12 +0,0 @@
<configuration version="7">
<folder id="test" path="testdata" ro="true" ignorePerms="false" rescanIntervalS="600">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="true">
<address>a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="true">
<address>b</address>
</device>
</configuration>

12
lib/config/testdata/v8.xml vendored
View File

@ -1,12 +0,0 @@
<configuration version="8">
<folder id="test" path="testdata" ro="true" ignorePerms="false" rescanIntervalS="600">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="true">
<address>a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="true">
<address>b</address>
</device>
</configuration>

12
lib/config/testdata/v9.xml vendored
View File

@ -1,12 +0,0 @@
<configuration version="9">
<folder id="test" path="testdata" ro="true" ignorePerms="false" rescanIntervalS="600">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>b</address>
</device>
</configuration>

42
lib/model/folder.go
View File

@ -33,8 +33,9 @@ import (
"github.com/thejerf/suture"
)
// scanLimiter limits the number of concurrent scans. A limit of zero means no limit.
var scanLimiter = newByteSemaphore(0)
// folderIOLimiter limits the number of concurrent I/O heavy operations,
// such as scans and pulls. A limit of zero means no limit.
var folderIOLimiter = newByteSemaphore(0)
type folder struct {
suture.Service
@ -130,7 +131,7 @@ func (f *folder) serve(ctx context.Context) {
pull := func() {
startTime := time.Now()
if f.puller.pull() {
if f.pull() {
// We're good. Don't schedule another pull and reset
// the pause interval.
pause = f.basePause()
@ -164,7 +165,7 @@ func (f *folder) serve(ctx context.Context) {
case <-initialCompleted:
// Initial scan has completed, we should do a pull
initialCompleted = nil // never hit this case again
if !f.puller.pull() {
if !f.pull() {
// Pulling failed, try again later.
pullFailTimer.Reset(pause)
}
@ -279,6 +280,35 @@ func (f *folder) getHealthError() error {
return nil
}
func (f *folder) pull() bool {
select {
case <-f.initialScanFinished:
default:
// Once the initial scan finished, a pull will be scheduled
return true
}
// If there is nothing to do, don't even enter sync-waiting state.
abort := true
snap := f.fset.Snapshot()
snap.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
abort = false
return false
})
snap.Release()
if abort {
return true
}
f.setState(FolderSyncWaiting)
defer f.setState(FolderIdle)
folderIOLimiter.take(1)
defer folderIOLimiter.give(1)
return f.puller.pull()
}
func (f *folder) scanSubdirs(subDirs []string) error {
if err := f.getHealthError(); err != nil {
// If there is a health error we set it as the folder error. We do not
@ -312,8 +342,8 @@ func (f *folder) scanSubdirs(subDirs []string) error {
f.setError(nil)
f.setState(FolderScanWaiting)
scanLimiter.take(1)
defer scanLimiter.give(1)
folderIOLimiter.take(1)
defer folderIOLimiter.give(1)
for i := range subDirs {
sub := osutil.NativeFilename(subDirs[i])

19
lib/model/folder_sendrecv.go
View File

@ -140,25 +140,6 @@ func newSendReceiveFolder(model *model, fset *db.FileSet, ignores *ignore.Matche
// pull returns true if it manages to get all needed items from peers, i.e. get
// the device in sync with the global state.
func (f *sendReceiveFolder) pull() bool {
select {
case <-f.initialScanFinished:
default:
// Once the initial scan finished, a pull will be scheduled
return true
}
// If there is nothing to do, don't even enter pulling state.
abort := true
snap := f.fset.Snapshot()
snap.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
abort = false
return false
})
snap.Release()
if abort {
return true
}
if err := f.CheckHealth(); err != nil {
l.Debugln("Skipping pull of", f.Description(), "due to folder error:", err)
return false

3
lib/model/folderstate.go
View File

@ -19,6 +19,7 @@ const (
FolderIdle folderState = iota
FolderScanning
FolderScanWaiting
FolderSyncWaiting
FolderSyncPreparing
FolderSyncing
FolderError
@ -32,6 +33,8 @@ func (s folderState) String() string {
return "scanning"
case FolderScanWaiting:
return "scan-waiting"
case FolderSyncWaiting:
return "sync-waiting"
case FolderSyncPreparing:
return "sync-preparing"
case FolderSyncing:

4
lib/model/model.go
View File

@ -208,7 +208,7 @@ func NewModel(cfg config.Wrapper, id protocol.DeviceID, clientName, clientVersio
m.deviceStatRefs[devID] = stats.NewDeviceStatisticsReference(m.db, devID.String())
}
m.Add(m.progressEmitter)
scanLimiter.setCapacity(cfg.Options().MaxConcurrentScans)
folderIOLimiter.setCapacity(cfg.Options().MaxFolderConcurrency())
return m
}
@ -2483,7 +2483,7 @@ func (m *model) CommitConfiguration(from, to config.Configuration) bool {
}
m.fmut.Unlock()
scanLimiter.setCapacity(to.Options.MaxConcurrentScans)
folderIOLimiter.setCapacity(to.Options.MaxFolderConcurrency())
// Some options don't require restart as those components handle it fine
// by themselves. Compare the options structs containing only the

11
test/folders.sh Executable file
View File

@ -0,0 +1,11 @@
#!/bin/bash
for ((id=0;id<200;id++)); do
cat <<EOT
<folder id="$id" label="" path="$id?maxsize=1000&amp;seed=$id" type="sendreceive" rescanIntervalS="3600" fsWatcherEnabled="false" fsWatcherDelayS="10" ignorePerms="false" autoNormalize="true">
<filesystemType>fake</filesystemType>
<device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU" introducedBy=""></device>
<device id="MRIW7OK-NETT3M4-N6SBWME-N25O76W-YJKVXPH-FUMQJ3S-P57B74J-GBITBAC" introducedBy=""></device>
</folder>
EOT
done