Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

cmd/syncthing: Add more stats to usage reports (ref #3628) 

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4347
This commit is contained in:
Audrius Butkevicius 2017-10-12 06:16:46 +00:00 committed by Jakob Borg
parent 813e6ddf83
commit 2760d032ca
23 changed files with 358 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
cmd/syncthing/gui.go
View File

@ -100,6 +100,7 @@ type modelIntf interface {
CurrentSequence(folder string) (int64, bool)
RemoteSequence(folder string) (int64, bool)
State(folder string) (string, time.Time, error)
UsageReportingStats(version int) map[string]interface{}
}
type configIntf interface {
@ -119,6 +120,7 @@ type configIntf interface {
type connectionsIntf interface {
Status() map[string]interface{}
NATType() string
}
type rater interface {
@ -800,18 +802,6 @@ func (s *apiService) postSystemConfig(w http.ResponseWriter, r *http.Request) {
}
}
// Fixup usage reporting settings
if curAcc := s.cfg.Options().URAccepted; to.Options.URAccepted > curAcc {
// UR was enabled
to.Options.URAccepted = usageReportVersion
to.Options.URUniqueID = rand.String(8)
} else if to.Options.URAccepted < curAcc {
// UR was disabled
to.Options.URAccepted = -1
to.Options.URUniqueID = ""
}
// Activate and save
if err := s.cfg.Replace(to); err != nil {
@ -903,6 +893,7 @@ func (s *apiService) getSystemStatus(w http.ResponseWriter, r *http.Request) {
// gives us percent
res["cpuPercent"] = s.cpu.Rate() / 10 / float64(runtime.NumCPU())
res["pathSeparator"] = string(filepath.Separator)
res["urVersionMax"] = usageReportVersion
res["uptime"] = int(time.Since(startTime).Seconds())
res["startTime"] = startTime
@ -981,7 +972,11 @@ func (s *apiService) getSystemDiscovery(w http.ResponseWriter, r *http.Request)
}
func (s *apiService) getReport(w http.ResponseWriter, r *http.Request) {
sendJSON(w, reportData(s.cfg, s.model))
version := usageReportVersion
if val, _ := strconv.Atoi(r.URL.Query().Get("version")); val > 0 {
version = val
}
sendJSON(w, reportData(s.cfg, s.model, s.connectionsService, version))
}
func (s *apiService) getRandomString(w http.ResponseWriter, r *http.Request) {

16
cmd/syncthing/main.go
View File

@ -882,24 +882,14 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
// Unique ID will be set and config saved below if necessary.
}
if opts.URAccepted > 0 && opts.URAccepted < usageReportVersion {
l.Infoln("Anonymous usage report has changed; revoking acceptance")
opts.URAccepted = 0
opts.URUniqueID = ""
cfg.SetOptions(opts)
}
if opts.URAccepted >= usageReportVersion && opts.URUniqueID == "" {
// Generate and save a new unique ID if it is missing.
if opts.URUniqueID == "" {
opts.URUniqueID = rand.String(8)
cfg.SetOptions(opts)
cfg.Save()
}
// The usageReportingManager registers itself to listen to configuration
// changes, and there's nothing more we need to tell it from the outside.
// Hence we don't keep the returned pointer.
newUsageReportingManager(cfg, m)
usageReportingSvc := newUsageReportingService(cfg, m, connectionsService)
mainService.Add(usageReportingSvc)
if opts.RestartOnWakeup {
go standbyMonitor()

4
cmd/syncthing/mocked_connections_test.go
View File

@ -11,3 +11,7 @@ type mockedConnections struct{}
func (m *mockedConnections) Status() map[string]interface{} {
return nil
}
func (m *mockedConnections) NATType() string {
return ""
}

4
cmd/syncthing/mocked_model_test.go
View File

@ -114,3 +114,7 @@ func (m *mockedModel) RemoteSequence(folder string) (int64, bool) {
func (m *mockedModel) State(folder string) (string, time.Time, error) {
return "", time.Time{}, nil
}
func (m *mockedModel) UsageReportingStats(version int) map[string]interface{} {
return nil
}

122
cmd/syncthing/usage_report.go
View File

@ -12,7 +12,6 @@ import (
"crypto/rand"
"crypto/tls"
"encoding/json"
"fmt"
"net/http"
"runtime"
"sort"
@ -20,71 +19,25 @@ import (
"time"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/connections"
"github.com/syncthing/syncthing/lib/dialer"
"github.com/syncthing/syncthing/lib/model"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/scanner"
"github.com/syncthing/syncthing/lib/upgrade"
"github.com/thejerf/suture"
)
// Current version number of the usage report, for acceptance purposes. If
// fields are added or changed this integer must be incremented so that users
// are prompted for acceptance of the new report.
const usageReportVersion = 2
type usageReportingManager struct {
cfg *config.Wrapper
model *model.Model
sup *suture.Supervisor
}
func newUsageReportingManager(cfg *config.Wrapper, m *model.Model) *usageReportingManager {
mgr := &usageReportingManager{
cfg: cfg,
model: m,
}
// Start UR if it's enabled.
mgr.CommitConfiguration(config.Configuration{}, cfg.RawCopy())
// Listen to future config changes so that we can start and stop as
// appropriate.
cfg.Subscribe(mgr)
return mgr
}
func (m *usageReportingManager) VerifyConfiguration(from, to config.Configuration) error {
return nil
}
func (m *usageReportingManager) CommitConfiguration(from, to config.Configuration) bool {
if to.Options.URAccepted >= usageReportVersion && m.sup == nil {
// Usage reporting was turned on; lets start it.
service := newUsageReportingService(m.cfg, m.model)
m.sup = suture.NewSimple("usageReporting")
m.sup.Add(service)
m.sup.ServeBackground()
} else if to.Options.URAccepted < usageReportVersion && m.sup != nil {
// Usage reporting was turned off
m.sup.Stop()
m.sup = nil
}
return true
}
func (m *usageReportingManager) String() string {
return fmt.Sprintf("usageReportingManager@%p", m)
}
const usageReportVersion = 3
// reportData returns the data to be sent in a usage report. It's used in
// various places, so not part of the usageReportingManager object.
func reportData(cfg configIntf, m modelIntf) map[string]interface{} {
func reportData(cfg configIntf, m modelIntf, connectionsService connectionsIntf, version int) map[string]interface{} {
opts := cfg.Options()
res := make(map[string]interface{})
res["urVersion"] = usageReportVersion
res["urVersion"] = version
res["uniqueID"] = opts.URUniqueID
res["version"] = Version
res["longVersion"] = LongVersion
@ -227,25 +180,40 @@ func reportData(cfg configIntf, m modelIntf) map[string]interface{} {
res["upgradeAllowedAuto"] = !(upgrade.DisabledByCompilation || noUpgradeFromEnv) && opts.AutoUpgradeIntervalH > 0
res["upgradeAllowedPre"] = !(upgrade.DisabledByCompilation || noUpgradeFromEnv) && opts.AutoUpgradeIntervalH > 0 && opts.UpgradeToPreReleases
if version >= 3 {
res["uptime"] = time.Now().Sub(startTime).Seconds()
res["natType"] = connectionsService.NATType()
}
for key, value := range m.UsageReportingStats(version) {
res[key] = value
}
return res
}
type usageReportingService struct {
cfg *config.Wrapper
model *model.Model
stop chan struct{}
cfg *config.Wrapper
model *model.Model
connectionsService *connections.Service
forceRun chan struct{}
stop chan struct{}
}
func newUsageReportingService(cfg *config.Wrapper, model *model.Model) *usageReportingService {
return &usageReportingService{
cfg: cfg,
model: model,
stop: make(chan struct{}),
func newUsageReportingService(cfg *config.Wrapper, model *model.Model, connectionsService *connections.Service) *usageReportingService {
svc := &usageReportingService{
cfg: cfg,
model: model,
connectionsService: connectionsService,
forceRun: make(chan struct{}),
stop: make(chan struct{}),
}
cfg.Subscribe(svc)
return svc
}
func (s *usageReportingService) sendUsageReport() error {
d := reportData(s.cfg, s.model)
d := reportData(s.cfg, s.model, s.connectionsService, s.cfg.Options().URAccepted)
var b bytes.Buffer
json.NewEncoder(&b).Encode(d)
@ -264,27 +232,45 @@ func (s *usageReportingService) sendUsageReport() error {
func (s *usageReportingService) Serve() {
s.stop = make(chan struct{})
l.Infoln("Starting usage reporting")
defer l.Infoln("Stopping usage reporting")
t := time.NewTimer(time.Duration(s.cfg.Options().URInitialDelayS) * time.Second) // time to initial report at start
t := time.NewTimer(time.Duration(s.cfg.Options().URInitialDelayS) * time.Second)
for {
select {
case <-s.stop:
return
case <-s.forceRun:
t.Reset(0)
case <-t.C:
err := s.sendUsageReport()
if err != nil {
l.Infoln("Usage report:", err)
if s.cfg.Options().URAccepted >= 2 {
err := s.sendUsageReport()
if err != nil {
l.Infoln("Usage report:", err)
} else {
l.Infof("Sent usage report (version %d)", s.cfg.Options().URAccepted)
}
}
t.Reset(24 * time.Hour) // next report tomorrow
}
}
}
func (s *usageReportingService) VerifyConfiguration(from, to config.Configuration) error {
return nil
}
func (s *usageReportingService) CommitConfiguration(from, to config.Configuration) bool {
if from.Options.URAccepted != to.Options.URAccepted || from.Options.URUniqueID != to.Options.URUniqueID || from.Options.URURL != to.Options.URURL {
s.forceRun <- struct{}{}
}
return true
}
func (s *usageReportingService) Stop() {
close(s.stop)
close(s.forceRun)
}
func (usageReportingService) String() string {
return "usageReportingService"
}
// cpuBench returns CPU performance as a measure of single threaded SHA-256 MiB/s

2
gui/default/syncthing/core/modalDirective.js
View File

@ -1,6 +1,8 @@
angular.module('syncthing.core')
.directive('modal', function () {
return {
// If you ever change any of the petroglyphs below, please search for $parent.$parent,
// as some templates rely on the way scope is composed in this case.
restrict: 'E',
templateUrl: 'modal.html',
replace: true,

51
gui/default/syncthing/core/syncthingController.js
View File

@ -33,6 +33,8 @@ angular.module('syncthing.core')
$scope.folderRejections = {};
$scope.protocolChanged = false;
$scope.reportData = {};
$scope.reportDataPreview = {};
$scope.reportDataPreviewVersion = '';
$scope.reportPreview = false;
$scope.folders = {};
$scope.seenError = '';
@ -133,7 +135,11 @@ angular.module('syncthing.core')
}).error($scope.emitHTTPError);
$http.get(urlbase + '/svc/report').success(function (data) {
$scope.reportData = data;
$scope.reportDataPreview = $scope.reportData = data;
if ($scope.system && $scope.config.options.urSeen < $scope.system.urVersionMax) {
// Usage reporting format has changed, prompt the user to re-accept.
$('#ur').modal();
}
}).error($scope.emitHTTPError);
$http.get(urlbase + '/system/upgrade').success(function (data) {
@ -376,6 +382,7 @@ angular.module('syncthing.core')
$scope.config = config;
$scope.config.options._listenAddressesStr = $scope.config.options.listenAddresses.join(', ');
$scope.config.options._globalAnnounceServersStr = $scope.config.options.globalAnnounceServers.join(', ');
$scope.config.options._urAcceptedStr = "" + $scope.config.options.urAccepted;
$scope.devices = $scope.config.devices;
$scope.devices.forEach(function (deviceCfg) {
@ -412,6 +419,10 @@ angular.module('syncthing.core')
$scope.myID = data.myID;
$scope.system = data;
if ($scope.reportDataPreviewVersion === '') {
$scope.reportDataPreviewVersion = $scope.system.urVersionMax;
}
var listenersFailed = [];
for (var address in data.connectionServiceStatus) {
if (data.connectionServiceStatus[address].error) {
@ -1058,7 +1069,6 @@ angular.module('syncthing.core')
$scope.editSettings = function () {
// Make a working copy
$scope.tmpOptions = angular.copy($scope.config.options);
$scope.tmpOptions.urEnabled = ($scope.tmpOptions.urAccepted > 0);
$scope.tmpOptions.deviceName = $scope.thisDevice().name;
$scope.tmpOptions.upgrades = "none";
if ($scope.tmpOptions.autoUpgradeIntervalH > 0) {
@ -1088,18 +1098,31 @@ angular.module('syncthing.core')
}).error($scope.emitHTTPError);
};
$scope.urVersions = function() {
var result = [];
if ($scope.system) {
for (var i = $scope.system.urVersionMax; i >= 2; i--) {
result.push("" + i);
}
}
return result;
};
$scope.saveSettings = function () {
// Make sure something changed
var changed = !angular.equals($scope.config.options, $scope.tmpOptions) || !angular.equals($scope.config.gui, $scope.tmpGUI);
var themeChanged = $scope.config.gui.theme !== $scope.tmpGUI.theme;
if (changed) {
// Angular has issues with selects with numeric values, so we handle strings here.
$scope.tmpOptions.urAccepted = parseInt($scope.tmpOptions._urAcceptedStr);
// Check if auto-upgrade has been enabled or disabled. This
// also has an effect on usage reporting, so do the check
// for that later.
if ($scope.tmpOptions.upgrades == "candidate") {
$scope.tmpOptions.autoUpgradeIntervalH = $scope.tmpOptions.autoUpgradeIntervalH || 12;
$scope.tmpOptions.upgradeToPreReleases = true;
$scope.tmpOptions.urEnabled = true;
$scope.tmpOptions.urAccepted = $scope.system.urVersionMax;
$scope.tmpOptions.urSeen = $scope.system.urVersionMax;
} else if ($scope.tmpOptions.upgrades == "stable") {
$scope.tmpOptions.autoUpgradeIntervalH = $scope.tmpOptions.autoUpgradeIntervalH || 12;
$scope.tmpOptions.upgradeToPreReleases = false;
@ -1107,13 +1130,6 @@ angular.module('syncthing.core')
$scope.tmpOptions.autoUpgradeIntervalH = 0;
}
// Check if usage reporting has been enabled or disabled
if ($scope.tmpOptions.urEnabled && $scope.tmpOptions.urAccepted <= 0) {
$scope.tmpOptions.urAccepted = 1000;
} else if (!$scope.tmpOptions.urEnabled && $scope.tmpOptions.urAccepted > 0) {
$scope.tmpOptions.urAccepted = -1;
}
// Check if protocol will need to be changed on restart
if ($scope.config.gui.useTLS !== $scope.tmpGUI.useTLS) {
$scope.protocolChanged = true;
@ -1691,13 +1707,17 @@ angular.module('syncthing.core')
};
$scope.acceptUR = function () {
$scope.config.options.urAccepted = 1000; // Larger than the largest existing report version
$scope.config.options.urAccepted = $scope.system.urVersionMax;
$scope.config.options.urSeen = $scope.system.urVersionMax;
$scope.saveConfig();
$('#ur').modal('hide');
};
$scope.declineUR = function () {
$scope.config.options.urAccepted = -1;
if ($scope.config.options.urAccepted === 0) {
$scope.config.options.urAccepted = -1;
}
$scope.config.options.urSeen = $scope.system.urVersionMax;
$scope.saveConfig();
$('#ur').modal('hide');
};
@ -1747,6 +1767,13 @@ angular.module('syncthing.core')
$scope.reportPreview = true;
};
$scope.refreshReportDataPreview = function () {
$scope.reportDataPreview = '';
$http.get(urlbase + '/svc/report?version=' + $scope.reportDataPreviewVersion).success(function (data) {
$scope.reportDataPreview = data;
}).error($scope.emitHTTPError);
};
$scope.rescanAllFolders = function () {
$http.post(urlbase + "/db/scan");
};

12
gui/default/syncthing/settings/settingsModalView.html
View File

@ -139,10 +139,14 @@
</div>
<div class="form-group">
<div class="checkbox" ng-if="tmpOptions.upgrades != 'candidate'">
<label>
<input id="UREnabled" type="checkbox" ng-model="tmpOptions.urEnabled"/> <span translate>Anonymous Usage Reporting</span> (<a href="" translate data-toggle="modal" data-target="#urPreview">Preview</a>)
</label>
<div ng-if="tmpOptions.upgrades != 'candidate'">
<label translate for="urVersion">Anonymous Usage Reporting</label> (<a href="" translate data-toggle="modal" data-target="#urPreview">Preview</a>)
<select class="form-control" id="urVersion" ng-model="tmpOptions._urAcceptedStr">
<option ng-repeat="n in urVersions()" value="{{n}}">Version {{n}}</option>
<!-- 1 does not exist, as we did not support incremental formats back then. -->
<option value="0" translate>Undecided (will prompt)</option>
<option value="-1" translate>Disabled</option>
</select>
</div>
<p class="help-block" ng-if="tmpOptions.upgrades == 'candidate'">
<span translate>Usage reporting is always enabled for candidate releases.</span> (<a href="" translate data-toggle="modal" data-target="#urPreview">Preview</a>)

11
gui/default/syncthing/usagereport/usageReportModalView.html
View File

@ -1,8 +1,13 @@
<modal id="ur" status="info" icon="bar-chart" heading="{{'Allow Anonymous Usage Reporting?' | translate}}" large="yes" closeable="no">
<div class="modal-body">
<p translate>The encrypted usage report is sent daily. It is used to track common platforms, folder sizes and app versions. If the reported data set is changed you will be prompted with this dialog again.</p>
<p translate>The aggregated statistics are publicly available at the URL below.</p>
<p><a href="https://data.syncthing.net/" target="_blank">https://data.syncthing.net/</a></p>
<div ng-if="config.options.urAccepted > 0 && config.options.urAccepted < system.urVersionMax">
<p>Anonymous Usage report format has changed. Would you like to move to the new format?</p>
</div>
<div ng-if="!(config.options.urAccepted > 0 && config.options.urAccepted < system.urVersionMax)">
<p translate>The encrypted usage report is sent daily. It is used to track common platforms, folder sizes and app versions. If the reported data set is changed you will be prompted with this dialog again.</p>
<p translate>The aggregated statistics are publicly available at the URL below.</p>
<p><a href="https://data.syncthing.net/" target="_blank">https://data.syncthing.net/</a></p>
</div>
<button type="button" class="btn btn-default btn-sm" ng-click="showReportPreview()" ng-show="!reportPreview">
<span class="fa fa-file-text-o"></span>&nbsp;<span translate>Preview Usage Report</span>
</button>

7
gui/default/syncthing/usagereport/usageReportPreviewModalView.html
View File

@ -5,8 +5,13 @@
</p>
<p translate>The aggregated statistics are publicly available at the URL below.</p>
<p><a href="https://data.syncthing.net/" target="_blank">https://data.syncthing.net/</a></p>
<label translate>Version</label>
<select id="urPreviewVersion" class="form-control" ng-model="$parent.$parent.reportDataPreviewVersion" ng-change="refreshReportDataPreview()" >
<option ng-repeat="n in urVersions()" value="{{n}}">Version {{n}}</option>
</select>
<hr>
<form>
<textarea class="form-control" rows="20">{{reportData | json}}</textarea>
<textarea class="form-control" rows="20" ng-if="reportDataPreview">{{reportDataPreview | json}}</textarea>
</form>
</div>
<div class="modal-footer">

11
lib/config/config.go
View File

@ -32,7 +32,7 @@ import (
const (
OldestHandledVersion = 10
CurrentVersion = 23
CurrentVersion = 24
MaxRescanIntervalS = 365 * 24 * 60 * 60
)
@ -326,6 +326,9 @@ func (cfg *Configuration) clean() error {
if cfg.Version == 22 {
convertV22V23(cfg)
}
if cfg.Version == 23 {
convertV23V24(cfg)
}
// Build a list of available devices
existingDevices := make(map[protocol.DeviceID]bool)
@ -375,6 +378,12 @@ func (cfg *Configuration) clean() error {
return nil
}
func convertV23V24(cfg *Configuration) {
cfg.Options.URSeen = 2
cfg.Version = 24
}
func convertV22V23(cfg *Configuration) {
permBits := fs.FileMode(0777)
if runtime.GOOS == "windows" {

1
lib/config/config_test.go
View File

@ -200,6 +200,7 @@ func TestOverriddenValues(t *testing.T) {
ProgressUpdateIntervalS: 10,
LimitBandwidthInLan: true,
MinHomeDiskFree: Size{5.2, "%"},
URSeen: 2,
URURL: "https://localhost/newdata",
URInitialDelayS: 800,
URPostInsecurely: true,

1
lib/config/optionsconfiguration.go
View File

@ -112,6 +112,7 @@ type OptionsConfiguration struct {
NATRenewalM int `xml:"natRenewalMinutes" json:"natRenewalMinutes" default:"30"`
NATTimeoutS int `xml:"natTimeoutSeconds" json:"natTimeoutSeconds" default:"10"`
URAccepted int `xml:"urAccepted" json:"urAccepted"` // Accepted usage reporting version; 0 for off (undecided), -1 for off (permanently)
URSeen int `xml:"urSeen" json:"urSeen"` // Report which the user has been prompted for.
URUniqueID string `xml:"urUniqueID" json:"urUniqueId"` // Unique ID for reporting purposes, regenerated when UR is turned on.
URURL string `xml:"urURL" json:"urURL" default:"https://data.syncthing.net/newdata"`
URPostInsecurely bool `xml:"urPostInsecurely" json:"urPostInsecurely" default:"false"` // For testing

16
lib/config/testdata/v24.xml vendored Normal file
View File

@ -0,0 +1,16 @@
<configuration version="22">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<filesystemType>basic</filesystemType>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFree unit="%">1</minDiskFree>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

19
lib/connections/kcp_listen.go
View File

@ -12,14 +12,16 @@ import (
"net/url"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/AudriusButkevicius/kcp-go"
"github.com/AudriusButkevicius/pfilter"
"github.com/ccding/go-stun/stun"
"github.com/xtaci/smux"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/nat"
"github.com/xtaci/smux"
)
func init() {
@ -38,6 +40,7 @@ type kcpListener struct {
stop chan struct{}
conns chan internalConn
factory listenerFactory
nat atomic.Value
address *url.URL
err error
@ -183,6 +186,14 @@ func (t *kcpListener) Factory() listenerFactory {
return t.factory
}
func (t *kcpListener) NATType() string {
v := t.nat.Load().(stun.NATType)
if v == stun.NATUnknown || v == stun.NATError {
return "unknown"
}
return v.String()
}
func (t *kcpListener) stunRenewal(listener net.PacketConn) {
client := stun.NewClientWithConnection(listener)
client.SetSoftwareName("syncthing")
@ -199,6 +210,7 @@ func (t *kcpListener) stunRenewal(listener net.PacketConn) {
if t.cfg.Options().StunKeepaliveS < 1 {
time.Sleep(time.Second)
oldType = stun.NATUnknown
t.nat.Store(stun.NATUnknown)
t.mut.Lock()
t.address = nil
t.mut.Unlock()
@ -222,6 +234,7 @@ func (t *kcpListener) stunRenewal(listener net.PacketConn) {
if oldType != natType {
l.Infof("%s detected NAT type: %s", t.uri, natType)
t.nat.Store(natType)
}
for {
@ -273,7 +286,7 @@ func (t *kcpListener) stunRenewal(listener net.PacketConn) {
type kcpListenerFactory struct{}
func (f *kcpListenerFactory) New(uri *url.URL, cfg *config.Wrapper, tlsCfg *tls.Config, conns chan internalConn, natService *nat.Service) genericListener {
return &kcpListener{
l := &kcpListener{
uri: fixupPort(uri, config.DefaultKCPPort),
cfg: cfg,
tlsCfg: tlsCfg,
@ -281,6 +294,8 @@ func (f *kcpListenerFactory) New(uri *url.URL, cfg *config.Wrapper, tlsCfg *tls.
stop: make(chan struct{}),
factory: f,
}
l.nat.Store(stun.NATUnknown)
return l
}
func (kcpListenerFactory) Enabled(cfg config.Configuration) bool {

4
lib/connections/relay_listen.go
View File

@ -171,6 +171,10 @@ func (t *relayListener) String() string {
return t.uri.String()
}
func (t *relayListener) NATType() string {
return "unknown"
}
type relayListenerFactory struct{}
func (f *relayListenerFactory) New(uri *url.URL, cfg *config.Wrapper, tlsCfg *tls.Config, conns chan internalConn, natService *nat.Service) genericListener {

12
lib/connections/service.go
View File

@ -574,6 +574,18 @@ func (s *Service) Status() map[string]interface{} {
return result
}
func (s *Service) NATType() string {
s.listenersMut.RLock()
defer s.listenersMut.RUnlock()
for _, listener := range s.listeners {
natType := listener.NATType()
if natType != "unknown" {
return natType
}
}
return "unknown"
}
func (s *Service) getDialerFactory(cfg config.Configuration, uri *url.URL) (dialerFactory, error) {
dialerFactory, ok := dialers[uri.Scheme]
if !ok {

19
lib/connections/structs.go
View File

@ -25,6 +25,7 @@ type Connection interface {
protocol.Connection
io.Closer
Type() string
Transport() string
RemoteAddr() net.Addr
}
@ -74,10 +75,27 @@ func (t connType) String() string {
}
}
func (t connType) Transport() string {
switch t {
case connTypeRelayClient, connTypeRelayServer:
return "relay"
case connTypeTCPClient, connTypeTCPServer:
return "tcp"
case connTypeKCPClient, connTypeKCPServer:
return "kcp"
default:
return "unknown"
}
}
func (c internalConn) Type() string {
return c.connType.String()
}
func (c internalConn) Transport() string {
return c.connType.Transport()
}
func (c internalConn) String() string {
return fmt.Sprintf("%s-%s/%s", c.LocalAddr(), c.RemoteAddr(), c.connType.String())
}
@ -116,6 +134,7 @@ type genericListener interface {
OnAddressesChanged(func(genericListener))
String() string
Factory() listenerFactory
NATType() string
}
type Model interface {

4
lib/connections/tcp_listen.go
View File

@ -176,6 +176,10 @@ func (t *tcpListener) Factory() listenerFactory {
return t.factory
}
func (t *tcpListener) NATType() string {
return "unknown"
}
type tcpListenerFactory struct{}
func (f *tcpListenerFactory) New(uri *url.URL, cfg *config.Wrapper, tlsCfg *tls.Config, conns chan internalConn, natService *nat.Service) genericListener {

4
lib/model/folder.go
View File

@ -60,6 +60,10 @@ func (f *folder) Jobs() ([]string, []string) {
func (f *folder) BringToFront(string) {}
func (f *folder) BlockStats() map[string]int {
return nil
}
func (f *folder) scanSubdirs(subDirs []string) error {
if err := f.model.internalScanFolderSubdirs(f.ctx, f.folderID, subDirs); err != nil {
// Potentially sets the error twice, once in the scanner just

101
lib/model/model.go
View File

@ -52,6 +52,7 @@ type service interface {
Scan(subs []string) error
Serve()
Stop()
BlockStats() map[string]int
getState() (folderState, time.Time, error)
setState(state folderState)
@ -405,6 +406,105 @@ func (m *Model) RestartFolder(cfg config.FolderConfiguration) {
m.fmut.Unlock()
}
func (m *Model) UsageReportingStats(version int) map[string]interface{} {
stats := make(map[string]interface{})
if version >= 3 {
// Block stats
m.fmut.Lock()
blockStats := make(map[string]int)
for _, folder := range m.folderRunners {
for k, v := range folder.BlockStats() {
blockStats[k] += v
}
}
m.fmut.Unlock()
stats["blockStats"] = blockStats
// Transport stats
m.pmut.Lock()
transportStats := make(map[string]int)
for _, conn := range m.conn {
transportStats[conn.Transport()]++
}
m.pmut.Unlock()
stats["transportStats"] = transportStats
// Ignore stats
ignoreStats := map[string]int{
"lines": 0,
"inverts": 0,
"folded": 0,
"deletable": 0,
"rooted": 0,
"includes": 0,
"escapedIncludes": 0,
"doubleStars": 0,
"stars": 0,
}
var seenPrefix [3]bool
for folder := range m.cfg.Folders() {
lines, _, err := m.GetIgnores(folder)
if err != nil {
continue
}
ignoreStats["lines"] += len(lines)
for _, line := range lines {
// Allow prefixes to be specified in any order, but only once.
for {
if strings.HasPrefix(line, "!") && !seenPrefix[0] {
seenPrefix[0] = true
line = line[1:]
ignoreStats["inverts"] += 1
} else if strings.HasPrefix(line, "(?i)") && !seenPrefix[1] {
seenPrefix[1] = true
line = line[4:]
ignoreStats["folded"] += 1
} else if strings.HasPrefix(line, "(?d)") && !seenPrefix[2] {
seenPrefix[2] = true
line = line[4:]
ignoreStats["deletable"] += 1
} else {
seenPrefix[0] = false
seenPrefix[1] = false
seenPrefix[2] = false
break
}
}
// Noops, remove
if strings.HasSuffix(line, "**") {
line = line[:len(line)-2]
}
if strings.HasPrefix(line, "**/") {
line = line[3:]
}
if strings.HasPrefix(line, "/") {
ignoreStats["rooted"] += 1
} else if strings.HasPrefix(line, "#include ") {
ignoreStats["includes"] += 1
if strings.Contains(line, "..") {
ignoreStats["escapedIncludes"] += 1
}
}
if strings.Contains(line, "**") {
ignoreStats["doubleStars"] += 1
// Remove not to trip up star checks.
strings.Replace(line, "**", "", -1)
}
if strings.Contains(line, "*") {
ignoreStats["stars"] += 1
}
}
}
stats["ignoreStats"] = ignoreStats
}
return stats
}
type ConnectionInfo struct {
protocol.Statistics
Connected bool
@ -2449,6 +2549,7 @@ func (m *Model) CommitConfiguration(from, to config.Configuration) bool {
// Some options don't require restart as those components handle it fine
// by themselves.
from.Options.URAccepted = to.Options.URAccepted
from.Options.URSeen = to.Options.URSeen
from.Options.URUniqueID = to.Options.URUniqueID
from.Options.ListenAddresses = to.Options.ListenAddresses
from.Options.RelaysEnabled = to.Options.RelaysEnabled

3
lib/model/model_test.go
View File

@ -308,6 +308,9 @@ func (f *fakeConnection) RemoteAddr() net.Addr {
func (f *fakeConnection) Type() string {
return "fake"
}
func (f *fakeConnection) Transport() string {
return "fake"
}
func (f *fakeConnection) DownloadProgress(folder string, updates []protocol.FileDownloadProgressUpdate) {
f.downloadProgressMessages = append(f.downloadProgressMessages, downloadProgressMessage{

30
lib/model/rwfolder.go
View File

@ -93,6 +93,9 @@ type sendReceiveFolder struct {
errors map[string]string // path -> error string
errorsMut sync.Mutex
blockStats map[string]int
blockStatsMut sync.Mutex
}
func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner, fs fs.Filesystem) service {
@ -107,6 +110,9 @@ func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver vers
remoteIndex: make(chan struct{}, 1), // This needs to be 1-buffered so that we queue a notification if we're busy doing a pull when it comes.
errorsMut: sync.NewMutex(),
blockStats: make(map[string]int),
blockStatsMut: sync.NewMutex(),
}
f.configureCopiersAndPullers()
@ -875,6 +881,11 @@ func (f *sendReceiveFolder) renameFile(source, target protocol.FileInfo) {
}
if err == nil {
f.blockStatsMut.Lock()
f.blockStats["total"] += len(target.Blocks)
f.blockStats["renamed"] += len(target.Blocks)
f.blockStatsMut.Unlock()
// The file was renamed, so we have handled both the necessary delete
// of the source and the creation of the target. Fix-up the metadata,
// and update the local index of the target file.
@ -1443,6 +1454,15 @@ func (f *sendReceiveFolder) finisherRoutine(in <-chan *sharedPullerState) {
if err != nil {
l.Debugln("Puller: final:", err)
f.newError(state.file.Name, err)
} else {
f.blockStatsMut.Lock()
f.blockStats["total"] += state.reused + state.copyTotal + state.pullTotal
f.blockStats["reused"] += state.reused
f.blockStats["pulled"] += state.pullTotal
f.blockStats["copyOrigin"] += state.copyOrigin
f.blockStats["copyOriginShifted"] += state.copyOriginShifted
f.blockStats["copyElsewhere"] += state.copyTotal - state.copyOrigin
f.blockStatsMut.Unlock()
}
events.Default.Log(events.ItemFinished, map[string]interface{}{
"folder": f.folderID,
@ -1459,6 +1479,16 @@ func (f *sendReceiveFolder) finisherRoutine(in <-chan *sharedPullerState) {
}
}
func (f *sendReceiveFolder) BlockStats() map[string]int {
f.blockStatsMut.Lock()
stats := make(map[string]int)
for k, v := range f.blockStats {
stats[k] = v
}
f.blockStatsMut.Unlock()
return stats
}
// Moves the given filename to the front of the job queue
func (f *sendReceiveFolder) BringToFront(filename string) {
f.queue.BringToFront(filename)