all: Display fs watcher status and retry starting it (ref #4552) (#4713)

2018-02-04 22:46:24 +01:00 · 2018-02-04 22:46:24 +01:00 · f87f13081b
parent 649d4cf7b0
commit f87f13081b
6 changed files with 122 additions and 25 deletions
--- a/cmd/syncthing/gui.go
+++ b/cmd/syncthing/gui.go
@ -112,6 +112,7 @@ type modelIntf interface {
 	State(folder string) (string, time.Time, error)
 	UsageReportingStats(version int, preview bool) map[string]interface{}
 	PullErrors(folder string) ([]model.FileError, error)
+	WatchError(folder string) error
 }

 type configIntf interface {
@ -733,6 +734,11 @@ func folderSummary(cfg configIntf, m modelIntf, folder string) (map[string]inter
 		}
 	}

+	err = m.WatchError(folder)
+	if err != nil {
+		res["watchError"] = err.Error()
+	}
+
 	return res, nil
 }

--- a/cmd/syncthing/mocked_model_test.go
+++ b/cmd/syncthing/mocked_model_test.go
@ -136,3 +136,7 @@ func (m *mockedModel) UsageReportingStats(version int, preview bool) map[string]
 func (m *mockedModel) PullErrors(folder string) ([]model.FileError, error) {
 	return nil, nil
 }
+
+func (m *mockedModel) WatchError(folder string) error {
+	return nil
+}
--- a/gui/default/assets/lang/lang-en.json
+++ b/gui/default/assets/lang/lang-en.json
@ -65,6 +65,7 @@
   "Device that last modified the item": "Device that last modified the item",
   "Devices": "Devices",
   "Disabled": "Disabled",
+   "Disabled periodic scanning": "Disabled periodic scanning",
   "Disconnected": "Disconnected",
   "Discovered": "Discovered",
   "Discovery": "Discovery",
@ -89,6 +90,7 @@
   "Error": "Error",
   "External File Versioning": "External File Versioning",
   "Failed Items": "Failed Items",
+   "Failed to setup, retrying": "Failed to setup, retrying",
   "Failure to connect to IPv6 servers is expected if there is no IPv6 connectivity.": "Failure to connect to IPv6 servers is expected if there is no IPv6 connectivity.",
   "File Pull Order": "File Pull Order",
   "File Versioning": "File Versioning",
@ -183,6 +185,7 @@
   "Pause": "Pause",
   "Pause All": "Pause All",
   "Paused": "Paused",
+   "Periodic scan every": "Periodic scan every",
   "Please consult the release notes before performing a major upgrade.": "Please consult the release notes before performing a major upgrade.",
   "Please set a GUI Authentication User and Password in the Settings dialog.": "Please set a GUI Authentication User and Password in the Settings dialog.",
   "Please wait": "Please wait",
@ -205,6 +208,7 @@
   "Rescan": "Rescan",
   "Rescan All": "Rescan All",
   "Rescan Interval": "Rescan Interval",
+   "Rescans": "Rescans",
   "Restart": "Restart",
   "Restart Needed": "Restart Needed",
   "Restarting": "Restarting",
@ -213,6 +217,7 @@
   "Resume": "Resume",
   "Resume All": "Resume All",
   "Reused": "Reused",
+   "Running": "Running",
   "Save": "Save",
   "Scan Time Remaining": "Scan Time Remaining",
   "Scanning": "Scanning",
--- a/gui/default/index.html
+++ b/gui/default/index.html
@ -371,16 +371,35 @@
                        <span translate>Yes</span>
                      </td>
                    </tr>
-                    <tr ng-if="folder.fsNotifications">
-                      <th><span class="fa fa-fw fa-bolt"></span>&nbsp;<span translate>Filesystem Notifications</span></th>
+                    <tr>
+                      <th><span class="fa fa-fw fa-refresh"></span>&nbsp;<span translate>Rescans</span></th>
                      <td class="text-right">
-                        <span translate>Yes</span>
+                        <span ng-if="folder.rescanIntervalS > 0 && !folder.fsWatcherEnabled" tooltip data-original-title="{{'Periodic scan every' | translate}} {{folder.rescanIntervalS | duration}} {{'and disabled watching for changes' | translate}}">
+                          <span class="fa fa-clock-o"></span>&nbsp;{{folder.rescanIntervalS | duration}}&ensp;
+                          <span class="fa fa-eye-slash"></span><span translate>Disabled</span>
+                        </span>
+                        <span ng-if="folder.rescanIntervalS > 0 && folder.fsWatcherEnabled && !model[folder.id].watchError" tooltip data-original-title="{{'Periodic scan every' | translate}} {{folder.rescanIntervalS | duration}} {{'and watching for changes' | translate}}">
+                          <span class="fa fa-clock-o"></span>{{folder.rescanIntervalS | duration}}&ensp;
+                          <span class="fa fa-eye"></span>&nbsp;<span translate>Running</span>
+                        </span>
+                        <span ng-if="folder.rescanIntervalS > 0 && folder.fsWatcherEnabled && model[folder.id].watchError" tooltip data-original-title="{{'Periodic scan every' | translate}} {{folder.rescanIntervalS | duration}} {{'and failed setting up watching for changes, retrying every 1m:' | translate}}<br/>{{model[folder.id].watchError}}">
+                          <span class="fa fa-clock-o"></span>{{folder.rescanIntervalS | duration}}&ensp;
+                          <span class="fa fa-eye-slash"></span>&nbsp;<span translate>Failed to setup, retrying</span>
+                        </span>
+                        <span ng-if="folder.rescanIntervalS <= 0 && !folder.fsWatcherEnabled" tooltip data-original-title="{{'Disabled periodic scanning' | translate}} {{'and disabled watching for changes' | translate}}">
+                          <span class="fa fa-clock-o"></span>&nbsp;<span translate>Disabled</span>&ensp;
+                          <span class="fa fa-eye-slash"></span>&nbsp;<span translate>Disabled</span>
+                        </span>
+                        <span ng-if="folder.rescanIntervalS <= 0 && folder.fsWatcherEnabled && !model[folder.id].watchError" tooltip data-original-title="{{'Disabled periodic scanning' | translate}} {{'and watching for changes' | translate}}">
+                          <span class="fa fa-clock-o"></span>&nbsp;<span translate>Disabled</span>&ensp;
+                          <span class="fa fa-eye"></span>&nbsp;<span translate>Running</span>
+                        </span>
+                        <span ng-if="folder.rescanIntervalS <= 0 && folder.fsWatcherEnabled && model[folder.id].watchError" tooltip data-original-title="{{'Disabled periodic scanning' | translate}} {{'and failed setting up watching for changes, retrying every 1m:' | translate}}<br/>{{model[folder.id].watchError}}">
+                          <span class="fa fa-clock-o"></span>&nbsp;<span translate>Disabled</span>&ensp;
+                          <span class="fa fa-eye-slash"></span>&nbsp;<span translate>Failed to setup, retrying</span>
+                        </span>
                      </td>
                    </tr>
-                    <tr ng-if="(folder.rescanIntervalS != 60 && !folder.fsNotifications) || (folder.rescanIntervalS != 3600 && folder.fsNotifications)">
-                      <th><span class="fa fa-fw fa-refresh"></span>&nbsp;<span translate>Rescan Interval</span></th>
-                      <td class="text-right">{{folder.rescanIntervalS}} s</td>
-                    </tr>
                    <tr ng-if="folder.order != 'random'">
                      <th><span class="fa fa-fw fa-sort"></span>&nbsp;<span translate>File Pull Order</span></th>
                      <td class="text-right" ng-switch="folder.order">
--- a/lib/model/folder.go
+++ b/lib/model/folder.go
@ -8,12 +8,17 @@ package model

 import (
 	"context"
+	"errors"
 	"time"

 	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/ignore"
+	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syncthing/syncthing/lib/watchaggregator"
 )

+var errWatchNotStarted error = errors.New("not started")
+
 type folder struct {
 	stateTracker
 	config.FolderConfiguration
@ -26,6 +31,8 @@ type folder struct {
 	watchCancel         context.CancelFunc
 	watchChan           chan []string
 	restartWatchChan    chan struct{}
+	watchErr            error
+	watchErrMut         sync.Mutex
 }

 func newFolder(model *Model, cfg config.FolderConfiguration) folder {
@ -41,6 +48,8 @@ func newFolder(model *Model, cfg config.FolderConfiguration) folder {
 		model:               model,
 		initialScanFinished: make(chan struct{}),
 		watchCancel:         func() {},
+		watchErr:            errWatchNotStarted,
+		watchErrMut:         sync.NewMutex(),
 	}
 }

@ -127,28 +136,22 @@ func (f *folder) scanTimerFired() {
 	f.scan.Reschedule()
 }

-func (f *folder) startWatch() {
-	ctx, cancel := context.WithCancel(f.ctx)
-	f.model.fmut.RLock()
-	ignores := f.model.folderIgnores[f.folderID]
-	f.model.fmut.RUnlock()
-	eventChan, err := f.Filesystem().Watch(".", ignores, ctx, f.IgnorePerms)
-	if err != nil {
-		l.Warnf("Failed to start filesystem watcher for folder %s: %v", f.Description(), err)
-	} else {
-		f.watchChan = make(chan []string)
-		f.watchCancel = cancel
-		watchaggregator.Aggregate(eventChan, f.watchChan, f.FolderConfiguration, f.model.cfg, ctx)
-		l.Infoln("Started filesystem watcher for folder", f.Description())
-	}
+func (f *folder) WatchError() error {
+	f.watchErrMut.Lock()
+	defer f.watchErrMut.Unlock()
+	return f.watchErr
 }

-func (f *folder) restartWatch() {
+// stopWatch immediately aborts watching and may be called asynchronously
+func (f *folder) stopWatch() {
 	f.watchCancel()
-	f.startWatch()
-	f.Scan(nil)
+	f.watchErrMut.Lock()
+	f.watchErr = errWatchNotStarted
+	f.watchErrMut.Unlock()
 }

+// scheduleWatchRestart makes sure watching is restarted from the main for loop
+// in a folder's Serve and thus may be called asynchronously (e.g. when ignores change).
 func (f *folder) scheduleWatchRestart() {
 	select {
 	case f.restartWatchChan <- struct{}{}:
@ -159,6 +162,56 @@ func (f *folder) scheduleWatchRestart() {
 	}
 }

+// restartWatch should only ever be called synchronously. If you want to use
+// this asynchronously, you should probably use scheduleWatchRestart instead.
+func (f *folder) restartWatch() {
+	f.stopWatch()
+	f.startWatch()
+	f.Scan(nil)
+}
+
+// startWatch should only ever be called synchronously. If you want to use
+// this asynchronously, you should probably use scheduleWatchRestart instead.
+func (f *folder) startWatch() {
+	ctx, cancel := context.WithCancel(f.ctx)
+	f.model.fmut.RLock()
+	ignores := f.model.folderIgnores[f.folderID]
+	f.model.fmut.RUnlock()
+	f.watchChan = make(chan []string)
+	f.watchCancel = cancel
+	go f.startWatchAsync(ctx, ignores)
+}
+
+// startWatchAsync tries to start the filesystem watching and retries every minute on failure.
+// It is a convenience function that should not be used except in startWatch.
+func (f *folder) startWatchAsync(ctx context.Context, ignores *ignore.Matcher) {
+	timer := time.NewTimer(0)
+	for {
+		select {
+		case <-timer.C:
+			eventChan, err := f.Filesystem().Watch(".", ignores, ctx, f.IgnorePerms)
+			f.watchErrMut.Lock()
+			prevErr := f.watchErr
+			f.watchErr = err
+			f.watchErrMut.Unlock()
+			if err != nil {
+				if prevErr == errWatchNotStarted {
+					l.Warnf("Failed to start filesystem watcher for folder %s: %v", f.Description(), err)
+				} else {
+					l.Debugf("Failed to start filesystem watcher for folder %s again: %v", f.Description(), err)
+				}
+				timer.Reset(time.Minute)
+				continue
+			}
+			watchaggregator.Aggregate(eventChan, f.watchChan, f.FolderConfiguration, f.model.cfg, ctx)
+			l.Debugln("Started filesystem watcher for folder", f.Description())
+			return
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
 func (f *folder) setError(err error) {
 	_, _, oldErr := f.getState()
 	if (err != nil && oldErr != nil && oldErr.Error() == err.Error()) || (err == nil && oldErr == nil) {
@ -177,7 +230,7 @@ func (f *folder) setError(err error) {

 	if f.FSWatcherEnabled {
 		if err != nil {
-			f.watchCancel()
+			f.stopWatch()
 		} else {
 			f.scheduleWatchRestart()
 		}
--- a/lib/model/model.go
+++ b/lib/model/model.go
@ -65,6 +65,7 @@ type service interface {
 	Stop()
 	CheckHealth() error
 	PullErrors() []FileError
+	WatchError() error

 	getState() (folderState, time.Time, error)
 	setState(state folderState)
@ -2213,6 +2214,15 @@ func (m *Model) PullErrors(folder string) ([]FileError, error) {
 	return m.folderRunners[folder].PullErrors(), nil
 }

+func (m *Model) WatchError(folder string) error {
+	m.fmut.RLock()
+	defer m.fmut.RUnlock()
+	if err := m.checkFolderRunningLocked(folder); err != nil {
+		return err
+	}
+	return m.folderRunners[folder].WatchError()
+}
+
 func (m *Model) Override(folder string) {
 	m.fmut.RLock()
 	fs, ok := m.folderFiles[folder]