Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/config, lib/model: Limit concurrent pulls (fixes #5914) (#6290) 

Adds a new folder state "Waiting to Sync" in the same vein as the
existing "Waiting to Scan". This vastly improves performances in the
rare cases when there are lots and lots of folders operating.
This commit is contained in:
Jakob Borg 2020-01-27 17:31:17 +01:00 committed by GitHub
parent 84920bff63
commit d91c4b010b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
34 changed files with 118 additions and 362 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
gui/default/index.html
View File

@ -313,7 +313,7 @@
<span ng-switch-when="paused"><span class="hidden-xs" translate>Paused</span><span class="visible-xs" aria-label="{{'Paused' | translate}}"><i class="fas fa-fw fa-pause"></i></span></span> <span ng-switch-when="paused"><span class="hidden-xs" translate>Paused</span><span class="visible-xs" aria-label="{{'Paused' | translate}}"><i class="fas fa-fw fa-pause"></i></span></span>
<span ng-switch-when="unknown"><span class="hidden-xs" translate>Unknown</span><span class="visible-xs" aria-label="{{'Unknown' | translate}}"><i class="fas fa-fw fa-question-circle"></i></span></span> <span ng-switch-when="unknown"><span class="hidden-xs" translate>Unknown</span><span class="visible-xs" aria-label="{{'Unknown' | translate}}"><i class="fas fa-fw fa-question-circle"></i></span></span>
<span ng-switch-when="unshared"><span class="hidden-xs" translate>Unshared</span><span class="visible-xs" aria-label="{{'Unshared' | translate}}"><i class="fas fa-fw fa-unlink"></i></span></span> <span ng-switch-when="unshared"><span class="hidden-xs" translate>Unshared</span><span class="visible-xs" aria-label="{{'Unshared' | translate}}"><i class="fas fa-fw fa-unlink"></i></span></span>
<span ng-switch-when="scan-waiting"><span class="hidden-xs" translate>Waiting to scan</span><span class="visible-xs" aria-label="{{'Waiting to scan' | translate}}"><i class="fas fa-fw fa-hourglass-half"></i></span></span> <span ng-switch-when="scan-waiting"><span class="hidden-xs" translate>Waiting to Scan</span><span class="visible-xs" aria-label="{{'Waiting to Scan' | translate}}"><i class="fas fa-fw fa-hourglass-half"></i></span></span>
<span ng-switch-when="stopped"><span class="hidden-xs" translate>Stopped</span><span class="visible-xs" aria-label="{{'Stopped' | translate}}"><i class="fas fa-fw fa-stop"></i></span></span> <span ng-switch-when="stopped"><span class="hidden-xs" translate>Stopped</span><span class="visible-xs" aria-label="{{'Stopped' | translate}}"><i class="fas fa-fw fa-stop"></i></span></span>
<span ng-switch-when="scanning"> <span ng-switch-when="scanning">
<span class="hidden-xs" translate>Scanning</span> <span class="hidden-xs" translate>Scanning</span>
@ -324,6 +324,10 @@
</span> </span>
<span ng-switch-when="idle"><span class="hidden-xs" translate>Up to Date</span><span class="visible-xs" aria-label="{{'Up to Date' | translate}}"><i class="fas fa-fw fa-check"></i></span></span> <span ng-switch-when="idle"><span class="hidden-xs" translate>Up to Date</span><span class="visible-xs" aria-label="{{'Up to Date' | translate}}"><i class="fas fa-fw fa-check"></i></span></span>
<span ng-switch-when="localadditions"><span class="hidden-xs" translate>Local Additions</span><span class="visible-xs" aria-label="{{'Local Additions' | translate}}"><i class="fas fa-fw fa-check"></i></span></span> <span ng-switch-when="localadditions"><span class="hidden-xs" translate>Local Additions</span><span class="visible-xs" aria-label="{{'Local Additions' | translate}}"><i class="fas fa-fw fa-check"></i></span></span>
<span ng-switch-when="sync-waiting">
<span class="hidden-xs" translate>Waiting to Sync</span>
<span class="visible-xs" aria-label="{{'Waiting to Sync' | translate}}"><i class="fas fa-fw fa-hourglass-half"></i></span>
</span>
<span ng-switch-when="sync-preparing"> <span ng-switch-when="sync-preparing">
<span class="hidden-xs" translate>Preparing to Sync</span> <span class="hidden-xs" translate>Preparing to Sync</span>
<span class="visible-xs" aria-label="{{'Preparing to Sync' | translate}}"><i class="fas fa-fw fa-hourglass-half"></i></span> <span class="visible-xs" aria-label="{{'Preparing to Sync' | translate}}"><i class="fas fa-fw fa-hourglass-half"></i></span>

2
gui/default/syncthing/core/syncthingController.js
View File

@ -837,7 +837,7 @@ angular.module('syncthing.core')
if (status === 'stopped' || status === 'outofsync' || status === 'error' || status === 'faileditems') { if (status === 'stopped' || status === 'outofsync' || status === 'error' || status === 'faileditems') {
return 'danger'; return 'danger';
} }
if (status === 'unshared' || status === 'scan-waiting') { if (status === 'unshared' || status === 'scan-waiting' || status === 'sync-waiting') {
return 'warning'; return 'warning';
} }

2
lib/config/config.go
View File

@ -31,7 +31,7 @@ import (
const ( const (
OldestHandledVersion = 10 OldestHandledVersion = 10
CurrentVersion = 29 CurrentVersion = 30
MaxRescanIntervalS = 365 * 24 * 60 * 60 MaxRescanIntervalS = 365 * 24 * 60 * 60
) )

28
lib/config/config_test.go
View File

@ -86,8 +86,13 @@ func TestDefaultValues(t *testing.T) {
func TestDeviceConfig(t *testing.T) { func TestDeviceConfig(t *testing.T) {
for i := OldestHandledVersion; i <= CurrentVersion; i++ { for i := OldestHandledVersion; i <= CurrentVersion; i++ {
cfgFile := fmt.Sprintf("testdata/v%d.xml", i)
if _, err := os.Stat(cfgFile); os.IsNotExist(err) {
continue
}
os.RemoveAll(filepath.Join("testdata", DefaultMarkerName)) os.RemoveAll(filepath.Join("testdata", DefaultMarkerName))
wr, err := load(fmt.Sprintf("testdata/v%d.xml", i), device1) wr, err := load(cfgFile, device1)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -1127,6 +1132,27 @@ func TestRemoveDeviceWithEmptyID(t *testing.T) {
} }
} }
func TestMaxConcurrentFolders(t *testing.T) {
cases := []struct {
input int
output int
}{
{input: -42, output: 0},
{input: -1, output: 0},
{input: 0, output: runtime.GOMAXPROCS(-1)},
{input: 1, output: 1},
{input: 42, output: 42},
}
for _, tc := range cases {
opts := OptionsConfiguration{RawMaxFolderConcurrency: tc.input}
res := opts.MaxFolderConcurrency()
if res != tc.output {
t.Errorf("Wrong MaxFolderConcurrency, %d => %d, expected %d", tc.input, res, tc.output)
}
}
}
// defaultConfigAsMap returns a valid default config as a JSON-decoded // defaultConfigAsMap returns a valid default config as a JSON-decoded
// map[string]interface{}. This is useful to override random elements and // map[string]interface{}. This is useful to override random elements and
// re-encode into JSON. // re-encode into JSON.

8
lib/config/migrations.go
View File

@ -25,6 +25,7 @@ import (
// update the config version. The order of migrations doesn't matter here, // update the config version. The order of migrations doesn't matter here,
// put the newest on top for readability. // put the newest on top for readability.
var migrations = migrationSet{ var migrations = migrationSet{
{30, migrateToConfigV30},
{29, migrateToConfigV29}, {29, migrateToConfigV29},
{28, migrateToConfigV28}, {28, migrateToConfigV28},
{27, migrateToConfigV27}, {27, migrateToConfigV27},
@ -84,6 +85,13 @@ func (m migration) apply(cfg *Configuration) {
cfg.Version = m.targetVersion cfg.Version = m.targetVersion
} }
func migrateToConfigV30(cfg *Configuration) {
// The "max concurrent scans" option is now spelled "max folder concurrency"
// to be more general.
cfg.Options.RawMaxFolderConcurrency = cfg.Options.DeprecatedMaxConcurrentScans
cfg.Options.DeprecatedMaxConcurrentScans = 0
}
func migrateToConfigV29(cfg *Configuration) { func migrateToConfigV29(cfg *Configuration) {
// The new crash reporting option should follow the state of global // The new crash reporting option should follow the state of global
// discovery / usage reporting, and we should display an appropriate // discovery / usage reporting, and we should display an appropriate

25
lib/config/optionsconfiguration.go
View File

@ -8,6 +8,7 @@ package config
import ( import (
"fmt" "fmt"
"runtime"
"github.com/syncthing/syncthing/lib/rand" "github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/util" "github.com/syncthing/syncthing/lib/util"
@ -52,7 +53,7 @@ type OptionsConfiguration struct {
TrafficClass int `xml:"trafficClass" json:"trafficClass"` TrafficClass int `xml:"trafficClass" json:"trafficClass"`
DefaultFolderPath string `xml:"defaultFolderPath" json:"defaultFolderPath" default:"~"` DefaultFolderPath string `xml:"defaultFolderPath" json:"defaultFolderPath" default:"~"`
SetLowPriority bool `xml:"setLowPriority" json:"setLowPriority" default:"true"` SetLowPriority bool `xml:"setLowPriority" json:"setLowPriority" default:"true"`
MaxConcurrentScans int `xml:"maxConcurrentScans" json:"maxConcurrentScans"` RawMaxFolderConcurrency int `xml:"maxFolderConcurrency" json:"maxFolderConcurrency"`
CRURL string `xml:"crashReportingURL" json:"crURL" default:"https://crash.syncthing.net/newcrash"` // crash reporting URL CRURL string `xml:"crashReportingURL" json:"crURL" default:"https://crash.syncthing.net/newcrash"` // crash reporting URL
CREnabled bool `xml:"crashReportingEnabled" json:"crashReportingEnabled" default:"true" restart:"true"` CREnabled bool `xml:"crashReportingEnabled" json:"crashReportingEnabled" default:"true" restart:"true"`
StunKeepaliveStartS int `xml:"stunKeepaliveStartS" json:"stunKeepaliveStartS" default:"180"` // 0 for off StunKeepaliveStartS int `xml:"stunKeepaliveStartS" json:"stunKeepaliveStartS" default:"180"` // 0 for off
@ -66,6 +67,7 @@ type OptionsConfiguration struct {
DeprecatedUPnPTimeoutS int `xml:"upnpTimeoutSeconds,omitempty" json:"-"` DeprecatedUPnPTimeoutS int `xml:"upnpTimeoutSeconds,omitempty" json:"-"`
DeprecatedRelayServers []string `xml:"relayServer,omitempty" json:"-"` DeprecatedRelayServers []string `xml:"relayServer,omitempty" json:"-"`
DeprecatedMinHomeDiskFreePct float64 `xml:"minHomeDiskFreePct,omitempty" json:"-"` DeprecatedMinHomeDiskFreePct float64 `xml:"minHomeDiskFreePct,omitempty" json:"-"`
DeprecatedMaxConcurrentScans int `xml:"maxConcurrentScans,omitempty" json:"-"`
} }
func (opts OptionsConfiguration) Copy() OptionsConfiguration { func (opts OptionsConfiguration) Copy() OptionsConfiguration {
@ -152,3 +154,24 @@ func (opts OptionsConfiguration) GlobalDiscoveryServers() []string {
} }
return util.UniqueTrimmedStrings(servers) return util.UniqueTrimmedStrings(servers)
} }
func (opts OptionsConfiguration) MaxFolderConcurrency() int {
// If a value is set, trust that.
if opts.RawMaxFolderConcurrency > 0 {
return opts.RawMaxFolderConcurrency
}
if opts.RawMaxFolderConcurrency < 0 {
// -1 etc means unlimited, which in the implementation means zero
return 0
}
// Otherwise default to the number of CPU cores in the system as a rough
// approximation of system powerfullness.
if n := runtime.GOMAXPROCS(-1); n > 0 {
return n
}
// We should never get here to begin with, but since we're here let's
// use some sort of reasonable compromise between the old "no limit" and
// getting nothing done... (Median number of folders out there at time
// of writing is two, 95-percentile at 12 folders.)
return 4 // https://xkcd.com/221/
}

12
lib/config/testdata/v10.xml vendored
View File

@ -1,12 +0,0 @@
<configuration version="10">
<folder id="test" path="testdata" ro="true" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>b</address>
</device>
</configuration>

13
lib/config/testdata/v11.xml vendored
View File

@ -1,13 +0,0 @@
<configuration version="11">
<folder id="test" path="testdata" ro="true" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFreePct>1</minDiskFreePct>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>b</address>
</device>
</configuration>

14
lib/config/testdata/v12.xml vendored
View File

@ -1,14 +0,0 @@
<configuration version="12">
<folder id="test" path="testdata" ro="true" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFreePct>1</minDiskFreePct>
<maxConflicts>-1</maxConflicts>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

14
lib/config/testdata/v13.xml vendored
View File

@ -1,14 +0,0 @@
<configuration version="13">
<folder id="test" path="testdata" ro="true" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFreePct>1</minDiskFreePct>
<maxConflicts>-1</maxConflicts>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

14
lib/config/testdata/v14.xml vendored
View File

@ -1,14 +0,0 @@
<configuration version="14">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFreePct>1</minDiskFreePct>
<maxConflicts>-1</maxConflicts>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

14
lib/config/testdata/v15.xml vendored
View File

@ -1,14 +0,0 @@
<configuration version="15">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFreePct>1</minDiskFreePct>
<maxConflicts>-1</maxConflicts>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

14
lib/config/testdata/v16.xml vendored
View File

@ -1,14 +0,0 @@
<configuration version="16">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFreePct>1</minDiskFreePct>
<maxConflicts>-1</maxConflicts>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

15
lib/config/testdata/v17.xml vendored
View File

@ -1,15 +0,0 @@
<configuration version="17">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFreePct>1</minDiskFreePct>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

15
lib/config/testdata/v18.xml vendored
View File

@ -1,15 +0,0 @@
<configuration version="18">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFreePct>1</minDiskFreePct>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

15
lib/config/testdata/v19.xml vendored
View File

@ -1,15 +0,0 @@
<configuration version="19">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFreePct>1</minDiskFreePct>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

15
lib/config/testdata/v20.xml vendored
View File

@ -1,15 +0,0 @@
<configuration version="20">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFree unit="%">1</minDiskFree>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

15
lib/config/testdata/v21.xml vendored
View File

@ -1,15 +0,0 @@
<configuration version="21">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFree unit="%">1</minDiskFree>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

16
lib/config/testdata/v23.xml vendored
View File

@ -1,16 +0,0 @@
<configuration version="22">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<filesystemType>basic</filesystemType>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFree unit="%">1</minDiskFree>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

16
lib/config/testdata/v24.xml vendored
View File

@ -1,16 +0,0 @@
<configuration version="22">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<filesystemType>basic</filesystemType>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFree unit="%">1</minDiskFree>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

16
lib/config/testdata/v25.xml vendored
View File

@ -1,16 +0,0 @@
<configuration version="25">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" fsWatcherEnabled="false" fsWatcherDelayS="10" autoNormalize="true">
<filesystemType>basic</filesystemType>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFree unit="%">1</minDiskFree>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

16
lib/config/testdata/v26.xml vendored
View File

@ -1,16 +0,0 @@
<configuration version="26">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" fsWatcherEnabled="false" fsWatcherDelayS="10" autoNormalize="true">
<filesystemType>basic</filesystemType>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFree unit="%">1</minDiskFree>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

16
lib/config/testdata/v27.xml vendored
View File

@ -1,16 +0,0 @@
<configuration version="27">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" fsWatcherEnabled="false" fsWatcherDelayS="10" autoNormalize="true">
<filesystemType>basic</filesystemType>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFree unit="%">1</minDiskFree>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

16
lib/config/testdata/v28.xml vendored
View File

@ -1,16 +0,0 @@
<configuration version="28">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" fsWatcherEnabled="false" fsWatcherDelayS="10" autoNormalize="true">
<filesystemType>basic</filesystemType>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFree unit="%">1</minDiskFree>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

16
lib/config/testdata/v29.xml vendored
View File

@ -1,16 +0,0 @@
<configuration version="28">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" fsWatcherEnabled="false" fsWatcherDelayS="10" autoNormalize="true">
<filesystemType>basic</filesystemType>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFree unit="%">1</minDiskFree>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

12
lib/config/testdata/v6.xml vendored
View File

@ -1,12 +0,0 @@
<configuration version="6">
<folder id="test" path="testdata" ro="true" ignorePerms="false" rescanIntervalS="600">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="true">
<address>a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="true">
<address>b</address>
</device>
</configuration>

12
lib/config/testdata/v7.xml vendored
View File

@ -1,12 +0,0 @@
<configuration version="7">
<folder id="test" path="testdata" ro="true" ignorePerms="false" rescanIntervalS="600">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="true">
<address>a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="true">
<address>b</address>
</device>
</configuration>

12
lib/config/testdata/v8.xml vendored
View File

@ -1,12 +0,0 @@
<configuration version="8">
<folder id="test" path="testdata" ro="true" ignorePerms="false" rescanIntervalS="600">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="true">
<address>a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="true">
<address>b</address>
</device>
</configuration>

12
lib/config/testdata/v9.xml vendored
View File

@ -1,12 +0,0 @@
<configuration version="9">
<folder id="test" path="testdata" ro="true" ignorePerms="false" rescanIntervalS="600">
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>b</address>
</device>
</configuration>

42
lib/model/folder.go
View File

@ -33,8 +33,9 @@ import (
"github.com/thejerf/suture" "github.com/thejerf/suture"
) )
// scanLimiter limits the number of concurrent scans. A limit of zero means no limit. // folderIOLimiter limits the number of concurrent I/O heavy operations,
var scanLimiter = newByteSemaphore(0) // such as scans and pulls. A limit of zero means no limit.
var folderIOLimiter = newByteSemaphore(0)
type folder struct { type folder struct {
suture.Service suture.Service
@ -130,7 +131,7 @@ func (f *folder) serve(ctx context.Context) {
pull := func() { pull := func() {
startTime := time.Now() startTime := time.Now()
if f.puller.pull() { if f.pull() {
// We're good. Don't schedule another pull and reset // We're good. Don't schedule another pull and reset
// the pause interval. // the pause interval.
pause = f.basePause() pause = f.basePause()
@ -164,7 +165,7 @@ func (f *folder) serve(ctx context.Context) {
case <-initialCompleted: case <-initialCompleted:
// Initial scan has completed, we should do a pull // Initial scan has completed, we should do a pull
initialCompleted = nil // never hit this case again initialCompleted = nil // never hit this case again
if !f.puller.pull() { if !f.pull() {
// Pulling failed, try again later. // Pulling failed, try again later.
pullFailTimer.Reset(pause) pullFailTimer.Reset(pause)
} }
@ -279,6 +280,35 @@ func (f *folder) getHealthError() error {
return nil return nil
} }
func (f *folder) pull() bool {
select {
case <-f.initialScanFinished:
default:
// Once the initial scan finished, a pull will be scheduled
return true
}
// If there is nothing to do, don't even enter sync-waiting state.
abort := true
snap := f.fset.Snapshot()
snap.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
abort = false
return false
})
snap.Release()
if abort {
return true
}
f.setState(FolderSyncWaiting)
defer f.setState(FolderIdle)
folderIOLimiter.take(1)
defer folderIOLimiter.give(1)
return f.puller.pull()
}
func (f *folder) scanSubdirs(subDirs []string) error { func (f *folder) scanSubdirs(subDirs []string) error {
if err := f.getHealthError(); err != nil { if err := f.getHealthError(); err != nil {
// If there is a health error we set it as the folder error. We do not // If there is a health error we set it as the folder error. We do not
@ -312,8 +342,8 @@ func (f *folder) scanSubdirs(subDirs []string) error {
f.setError(nil) f.setError(nil)
f.setState(FolderScanWaiting) f.setState(FolderScanWaiting)
scanLimiter.take(1) folderIOLimiter.take(1)
defer scanLimiter.give(1) defer folderIOLimiter.give(1)
for i := range subDirs { for i := range subDirs {
sub := osutil.NativeFilename(subDirs[i]) sub := osutil.NativeFilename(subDirs[i])

19
lib/model/folder_sendrecv.go
View File

@ -140,25 +140,6 @@ func newSendReceiveFolder(model *model, fset *db.FileSet, ignores *ignore.Matche
// pull returns true if it manages to get all needed items from peers, i.e. get // pull returns true if it manages to get all needed items from peers, i.e. get
// the device in sync with the global state. // the device in sync with the global state.
func (f *sendReceiveFolder) pull() bool { func (f *sendReceiveFolder) pull() bool {
select {
case <-f.initialScanFinished:
default:
// Once the initial scan finished, a pull will be scheduled
return true
}
// If there is nothing to do, don't even enter pulling state.
abort := true
snap := f.fset.Snapshot()
snap.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
abort = false
return false
})
snap.Release()
if abort {
return true
}
if err := f.CheckHealth(); err != nil { if err := f.CheckHealth(); err != nil {
l.Debugln("Skipping pull of", f.Description(), "due to folder error:", err) l.Debugln("Skipping pull of", f.Description(), "due to folder error:", err)
return false return false

3
lib/model/folderstate.go
View File

@ -19,6 +19,7 @@ const (
FolderIdle folderState = iota FolderIdle folderState = iota
FolderScanning FolderScanning
FolderScanWaiting FolderScanWaiting
FolderSyncWaiting
FolderSyncPreparing FolderSyncPreparing
FolderSyncing FolderSyncing
FolderError FolderError
@ -32,6 +33,8 @@ func (s folderState) String() string {
return "scanning" return "scanning"
case FolderScanWaiting: case FolderScanWaiting:
return "scan-waiting" return "scan-waiting"
case FolderSyncWaiting:
return "sync-waiting"
case FolderSyncPreparing: case FolderSyncPreparing:
return "sync-preparing" return "sync-preparing"
case FolderSyncing: case FolderSyncing:

4
lib/model/model.go
View File

@ -208,7 +208,7 @@ func NewModel(cfg config.Wrapper, id protocol.DeviceID, clientName, clientVersio
m.deviceStatRefs[devID] = stats.NewDeviceStatisticsReference(m.db, devID.String()) m.deviceStatRefs[devID] = stats.NewDeviceStatisticsReference(m.db, devID.String())
} }
m.Add(m.progressEmitter) m.Add(m.progressEmitter)
scanLimiter.setCapacity(cfg.Options().MaxConcurrentScans) folderIOLimiter.setCapacity(cfg.Options().MaxFolderConcurrency())
return m return m
} }
@ -2483,7 +2483,7 @@ func (m *model) CommitConfiguration(from, to config.Configuration) bool {
} }
m.fmut.Unlock() m.fmut.Unlock()
scanLimiter.setCapacity(to.Options.MaxConcurrentScans) folderIOLimiter.setCapacity(to.Options.MaxFolderConcurrency())
// Some options don't require restart as those components handle it fine // Some options don't require restart as those components handle it fine
// by themselves. Compare the options structs containing only the // by themselves. Compare the options structs containing only the

11
test/folders.sh Executable file
View File

@ -0,0 +1,11 @@
#!/bin/bash
for ((id=0;id<200;id++)); do
cat <<EOT
<folder id="$id" label="" path="$id?maxsize=1000&amp;seed=$id" type="sendreceive" rescanIntervalS="3600" fsWatcherEnabled="false" fsWatcherDelayS="10" ignorePerms="false" autoNormalize="true">
<filesystemType>fake</filesystemType>
<device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU" introducedBy=""></device>
<device id="MRIW7OK-NETT3M4-N6SBWME-N25O76W-YJKVXPH-FUMQJ3S-P57B74J-GBITBAC" introducedBy=""></device>
</folder>
EOT
done