Add support for multiple announce servers (fixes #677)

Somebody owes me a beer.
This commit is contained in:
Audrius Butkevicius 2014-11-18 22:57:21 +00:00 committed by Jakob Borg
parent 78981862be
commit fd2d2c035e
9 changed files with 162 additions and 80 deletions

View File

@ -826,7 +826,7 @@ func renewUPnP(port int) {
if forwardedPort != 0 {
externalPort = forwardedPort
discoverer.StopGlobal()
discoverer.StartGlobal(opts.GlobalAnnServer, uint16(forwardedPort))
discoverer.StartGlobal(opts.GlobalAnnServers, uint16(forwardedPort))
if debugNet {
l.Debugf("Updated UPnP port mapping for external port %d on device %s.", forwardedPort, igd.FriendlyIdentifier())
}
@ -1098,7 +1098,7 @@ func discovery(extPort int) *discover.Discoverer {
if opts.GlobalAnnEnabled {
l.Infoln("Starting global discovery announcements")
disc.StartGlobal(opts.GlobalAnnServer, uint16(extPort))
disc.StartGlobal(opts.GlobalAnnServers, uint16(extPort))
}
return disc

View File

@ -203,11 +203,17 @@
<th><span class="glyphicon glyphicon-dashboard"></span>&emsp;<span translate>CPU Utilization</span></th>
<td class="text-right">{{system.cpuPercent | alwaysNumber | natural:1}}%</td>
</tr>
<tr ng-if="system.extAnnounceOK != undefined">
<th><span class="glyphicon glyphicon-bullhorn"></span>&emsp;<span translate>Global Discovery Server</span></th>
<tr ng-if="system.extAnnounceOK != undefined && announceServersTotal > 0">
<th><span class="glyphicon glyphicon-bullhorn"></span>&emsp;<span translate>Global Discovery Servers</span></th>
<td class="text-right">
<span class="data text-success" ng-if="system.extAnnounceOK"><span translate>Online</span></span>
<span class="data text-danger" ng-if="!system.extAnnounceOK"><span translate>Offline</span></span>
<span ng-if="announceServersFailed.length == 0" class="data text-success">
<span>OK</span>
</span>
<span ng-if="announceServersFailed.length != 0" class="data text-danger">
<span popover data-trigger="hover" data-placement="bottom" data-content="{{announceServersFailed.join('\n')}}">
{{announceServersTotal-announceServersFailed.length}}/{{announceServersTotal}}
</span>
</span>
</td>
</tr>
<tr>
@ -609,8 +615,8 @@
<input id="DeviceName" class="form-control" type="text" ng-model="tmpOptions.DeviceName">
</div>
<div class="form-group">
<label translate for="ListenStr">Sync Protocol Listen Addresses</label>
<input id="ListenStr" class="form-control" type="text" ng-model="tmpOptions.ListenStr">
<label translate for="ListenAddressStr">Sync Protocol Listen Addresses</label>
<input id="ListenAddressStr" class="form-control" type="text" ng-model="tmpOptions.ListenAddressStr">
</div>
<div class="form-group">
<label translate for="MaxRecvKbps">Incoming Rate Limit (KiB/s)</label>
@ -653,8 +659,8 @@
</div>
</div>
<div class="form-group">
<label translate for="GlobalAnnServer">Global Discovery Server</label>
<input ng-disabled="!tmpOptions.GlobalAnnEnabled" id="GlobalAnnServer" class="form-control" type="text" ng-model="tmpOptions.GlobalAnnServer">
<label translate for="GlobalAnnServersStr">Global Discovery Server</label>
<input ng-disabled="!tmpOptions.GlobalAnnEnabled" id="GlobalAnnServersStr" class="form-control" type="text" ng-model="tmpOptions.GlobalAnnServersStr">
</div>
</div>
@ -865,6 +871,7 @@
<script src="scripts/syncthing/core/directives/modalDirective.js"></script>
<script src="scripts/syncthing/core/directives/uniqueFolderDirective.js"></script>
<script src="scripts/syncthing/core/directives/validDeviceidDirective.js"></script>
<script src="scripts/syncthing/core/directives/popoverDirective.js"></script>
<script src="scripts/syncthing/core/filters/alwaysNumberFilter.js"></script>
<script src="scripts/syncthing/core/filters/basenameFilter.js"></script>
<script src="scripts/syncthing/core/filters/binaryFilter.js"></script>

View File

@ -245,7 +245,8 @@ angular.module('syncthing.core')
var hasConfig = !isEmptyObject($scope.config);
$scope.config = config;
$scope.config.Options.ListenStr = $scope.config.Options.ListenAddress.join(', ');
$scope.config.Options.ListenAddressStr = $scope.config.Options.ListenAddress.join(', ');
$scope.config.Options.GlobalAnnServersStr = $scope.config.Options.GlobalAnnServers.join(', ');
$scope.devices = $scope.config.Devices;
$scope.devices.forEach(function (deviceCfg) {
@ -272,6 +273,14 @@ angular.module('syncthing.core')
$http.get(urlbase + '/system').success(function (data) {
$scope.myID = data.myID;
$scope.system = data;
$scope.announceServersTotal = Object.keys(data.extAnnounceOK).length;
var failed = [];
for (var server in data.extAnnounceOK) {
if (!data.extAnnounceOK[server]) {
failed.push(server);
}
}
$scope.announceServersFailed = failed;
console.log("refreshSystem", data);
});
}
@ -599,8 +608,11 @@ angular.module('syncthing.core')
$scope.thisDevice().Name = $scope.tmpOptions.DeviceName;
$scope.config.Options = angular.copy($scope.tmpOptions);
$scope.config.GUI = angular.copy($scope.tmpGUI);
$scope.config.Options.ListenAddress = $scope.config.Options.ListenStr.split(',').map(function (x) {
return x.trim();
['ListenAddress', 'GlobalAnnServers'].forEach(function (key) {
$scope.config.Options[key] = $scope.config.Options[key + "Str"].split(/[ ,]+/).map(function (x) {
return x.trim();
});
});
$scope.saveConfig();

View File

@ -0,0 +1,9 @@
angular.module('syncthing.core')
.directive('popover', function () {
return {
restrict: 'A',
link: function (scope, element, attributes) {
$(element).popover();
}
};
});

File diff suppressed because one or more lines are too long

View File

@ -160,7 +160,7 @@ type FolderDeviceConfiguration struct {
type OptionsConfiguration struct {
ListenAddress []string `xml:"listenAddress" default:"0.0.0.0:22000"`
GlobalAnnServer string `xml:"globalAnnounceServer" default:"announce.syncthing.net:22026"`
GlobalAnnServers []string `xml:"globalAnnounceServer" default:"announce.syncthing.net:22026"`
GlobalAnnEnabled bool `xml:"globalAnnounceEnabled" default:"true"`
LocalAnnEnabled bool `xml:"localAnnounceEnabled" default:"true"`
LocalAnnPort int `xml:"localAnnouncePort" default:"21025"`
@ -239,8 +239,6 @@ func (cfg *Configuration) WriteXML(w io.Writer) error {
func (cfg *Configuration) prepare(myID protocol.DeviceID) {
fillNilSlices(&cfg.Options)
cfg.Options.ListenAddress = uniqueStrings(cfg.Options.ListenAddress)
// Initialize an empty slice for folders if the config has none
if cfg.Folders == nil {
cfg.Folders = []FolderConfiguration{}
@ -362,6 +360,9 @@ func (cfg *Configuration) prepare(myID protocol.DeviceID) {
n.Addresses = []string{"dynamic"}
}
}
cfg.Options.ListenAddress = uniqueStrings(cfg.Options.ListenAddress)
cfg.Options.GlobalAnnServers = uniqueStrings(cfg.Options.GlobalAnnServers)
}
// ChangeRequiresRestart returns true if updating the configuration requires a
@ -469,8 +470,8 @@ func convertV2V3(cfg *Configuration) {
// The global discovery format and port number changed in v0.9. Having the
// default announce server but old port number is guaranteed to be legacy.
if cfg.Options.GlobalAnnServer == "announce.syncthing.net:22025" {
cfg.Options.GlobalAnnServer = "announce.syncthing.net:22026"
if len(cfg.Options.GlobalAnnServers) == 1 && cfg.Options.GlobalAnnServers[0] == "announce.syncthing.net:22025" {
cfg.Options.GlobalAnnServers = []string{"announce.syncthing.net:22026"}
}
cfg.Version = 3

View File

@ -36,7 +36,7 @@ func init() {
func TestDefaultValues(t *testing.T) {
expected := OptionsConfiguration{
ListenAddress: []string{"0.0.0.0:22000"},
GlobalAnnServer: "announce.syncthing.net:22026",
GlobalAnnServers: []string{"announce.syncthing.net:22026"},
GlobalAnnEnabled: true,
LocalAnnEnabled: true,
LocalAnnPort: 21025,
@ -138,7 +138,7 @@ func TestNoListenAddress(t *testing.T) {
func TestOverriddenValues(t *testing.T) {
expected := OptionsConfiguration{
ListenAddress: []string{":23000"},
GlobalAnnServer: "syncthing.nym.se:22026",
GlobalAnnServers: []string{"syncthing.nym.se:22026"},
GlobalAnnEnabled: false,
LocalAnnEnabled: false,
LocalAnnPort: 42123,
@ -163,7 +163,7 @@ func TestOverriddenValues(t *testing.T) {
}
if !reflect.DeepEqual(cfg.Options(), expected) {
t.Errorf("Overridden config differs;\n E: %#v\n A: %#v", expected, cfg.Options)
t.Errorf("Overridden config differs;\n E: %#v\n A: %#v", expected, cfg.Options())
}
}

View File

@ -42,13 +42,13 @@ type Discoverer struct {
multicastBeacon beacon.Interface
registry map[protocol.DeviceID][]CacheEntry
registryLock sync.RWMutex
extServer string
extServers []string
extPort uint16
localBcastTick <-chan time.Time
stopGlobal chan struct{}
globalWG sync.WaitGroup
forcedBcastTick chan time.Time
extAnnounceOK bool
extAnnounceOK map[string]bool
extAnnounceOKmut sync.Mutex
}
@ -70,6 +70,7 @@ func NewDiscoverer(id protocol.DeviceID, addresses []string) *Discoverer {
errorRetryIntv: 60 * time.Second,
cacheLifetime: 5 * time.Minute,
registry: make(map[protocol.DeviceID][]CacheEntry),
extAnnounceOK: make(map[string]bool),
}
}
@ -110,14 +111,26 @@ func (d *Discoverer) StartLocal(localPort int, localMCAddr string) {
}
}
func (d *Discoverer) StartGlobal(server string, extPort uint16) {
func (d *Discoverer) StartGlobal(servers []string, extPort uint16) {
// Wait for any previous announcer to stop before starting a new one.
d.globalWG.Wait()
d.extServer = server
d.extServers = servers
d.extPort = extPort
d.stopGlobal = make(chan struct{})
d.globalWG.Add(1)
go d.sendExternalAnnouncements()
go func() {
defer d.globalWG.Done()
buf := d.announcementPkt()
for _, extServer := range d.extServers {
d.globalWG.Add(1)
go func(server string) {
d.sendExternalAnnouncements(server, buf)
d.globalWG.Done()
}(extServer)
}
}()
}
func (d *Discoverer) StopGlobal() {
@ -127,7 +140,7 @@ func (d *Discoverer) StopGlobal() {
}
}
func (d *Discoverer) ExtAnnounceOK() bool {
func (d *Discoverer) ExtAnnounceOK() map[string]bool {
d.extAnnounceOKmut.Lock()
defer d.extAnnounceOKmut.Unlock()
return d.extAnnounceOK
@ -144,7 +157,7 @@ func (d *Discoverer) Lookup(device protocol.DeviceID) []string {
addrs[i] = cached[i].Address
}
return addrs
} else if len(d.extServer) != 0 && time.Since(d.localBcastStart) > d.localBcastIntv {
} else if len(d.extServers) != 0 && time.Since(d.localBcastStart) > d.localBcastIntv {
// Only perform external lookups if we have at least one external
// server and one local announcement interval has passed. This is to
// avoid finding local peers on their remote address at startup.
@ -188,20 +201,24 @@ func (d *Discoverer) All() map[protocol.DeviceID][]CacheEntry {
func (d *Discoverer) announcementPkt() []byte {
var addrs []Address
for _, astr := range d.listenAddrs {
addr, err := net.ResolveTCPAddr("tcp", astr)
if err != nil {
l.Warnln("%v: not announcing %s", err, astr)
continue
} else if debug {
l.Debugf("discover: announcing %s: %#v", astr, addr)
}
if len(addr.IP) == 0 || addr.IP.IsUnspecified() {
addrs = append(addrs, Address{Port: uint16(addr.Port)})
} else if bs := addr.IP.To4(); bs != nil {
addrs = append(addrs, Address{IP: bs, Port: uint16(addr.Port)})
} else if bs := addr.IP.To16(); bs != nil {
addrs = append(addrs, Address{IP: bs, Port: uint16(addr.Port)})
if d.extPort != 0 {
addrs = []Address{{Port: d.extPort}}
} else {
for _, astr := range d.listenAddrs {
addr, err := net.ResolveTCPAddr("tcp", astr)
if err != nil {
l.Warnln("%v: not announcing %s", err, astr)
continue
} else if debug {
l.Debugf("discover: announcing %s: %#v", astr, addr)
}
if len(addr.IP) == 0 || addr.IP.IsUnspecified() {
addrs = append(addrs, Address{Port: uint16(addr.Port)})
} else if bs := addr.IP.To4(); bs != nil {
addrs = append(addrs, Address{IP: bs, Port: uint16(addr.Port)})
} else if bs := addr.IP.To16(); bs != nil {
addrs = append(addrs, Address{IP: bs, Port: uint16(addr.Port)})
}
}
}
var pkt = Announce{
@ -235,44 +252,43 @@ func (d *Discoverer) sendLocalAnnouncements() {
}
}
func (d *Discoverer) sendExternalAnnouncements() {
defer d.globalWG.Done()
remote, err := net.ResolveUDPAddr("udp", d.extServer)
for err != nil {
l.Warnf("Global discovery: %v; trying again in %v", err, d.errorRetryIntv)
time.Sleep(d.errorRetryIntv)
remote, err = net.ResolveUDPAddr("udp", d.extServer)
}
func (d *Discoverer) sendExternalAnnouncements(extServer string, buf []byte) {
timer := time.NewTimer(0)
conn, err := net.ListenUDP("udp", nil)
for err != nil {
timer.Reset(d.errorRetryIntv)
l.Warnf("Global discovery: %v; trying again in %v", err, d.errorRetryIntv)
time.Sleep(d.errorRetryIntv)
select {
case <-d.stopGlobal:
return
case <-timer.C:
}
conn, err = net.ListenUDP("udp", nil)
}
var buf []byte
if d.extPort != 0 {
var pkt = Announce{
Magic: AnnouncementMagic,
This: Device{d.myID[:], []Address{{Port: d.extPort}}},
remote, err := net.ResolveUDPAddr("udp", extServer)
for err != nil {
timer.Reset(d.errorRetryIntv)
l.Warnf("Global discovery: %s: %v; trying again in %v", extServer, err, d.errorRetryIntv)
select {
case <-d.stopGlobal:
return
case <-timer.C:
}
buf = pkt.MustMarshalXDR()
} else {
buf = d.announcementPkt()
remote, err = net.ResolveUDPAddr("udp", extServer)
}
// Delay the first announcement until after a full local announcement
// cycle, to increase the chance of other peers finding us locally first.
nextAnnouncement := time.NewTimer(d.localBcastIntv)
timer.Reset(d.localBcastIntv)
for {
select {
case <-d.stopGlobal:
return
case <-nextAnnouncement.C:
case <-timer.C:
var ok bool
if debug {
@ -282,28 +298,29 @@ func (d *Discoverer) sendExternalAnnouncements() {
_, err := conn.WriteTo(buf, remote)
if err != nil {
if debug {
l.Debugln("discover: warning:", err)
l.Debugln("discover: %s: warning:", extServer, err)
}
ok = false
} else {
// Verify that the announce server responds positively for our device ID
time.Sleep(1 * time.Second)
res := d.externalLookup(d.myID)
res := d.externalLookupOnServer(extServer, d.myID)
if debug {
l.Debugln("discover: external lookup check:", res)
l.Debugln("discover:", extServer, "external lookup check:", res)
}
ok = len(res) > 0
}
d.extAnnounceOKmut.Lock()
d.extAnnounceOK = ok
d.extAnnounceOK[extServer] = ok
d.extAnnounceOKmut.Unlock()
if ok {
nextAnnouncement.Reset(d.globalBcastIntv)
timer.Reset(d.globalBcastIntv)
} else {
nextAnnouncement.Reset(d.errorRetryIntv)
timer.Reset(d.errorRetryIntv)
}
}
}
@ -390,10 +407,39 @@ func (d *Discoverer) registerDevice(addr net.Addr, device Device) bool {
}
func (d *Discoverer) externalLookup(device protocol.DeviceID) []string {
extIP, err := net.ResolveUDPAddr("udp", d.extServer)
// Buffer up to as many answers as we have servers to query.
results := make(chan []string, len(d.extServers))
// Query all servers.
wg := sync.WaitGroup{}
for _, extServer := range d.extServers {
wg.Add(1)
go func(server string) {
result := d.externalLookupOnServer(server, device)
if debug {
l.Debugln("discover:", result, "from", server, "for", device)
}
results <- result
wg.Done()
}(extServer)
}
wg.Wait()
close(results)
addrs := []string{}
for result := range results {
addrs = append(addrs, result...)
}
return addrs
}
func (d *Discoverer) externalLookupOnServer(extServer string, device protocol.DeviceID) []string {
extIP, err := net.ResolveUDPAddr("udp", extServer)
if err != nil {
if debug {
l.Debugf("discover: %v; no external lookup", err)
l.Debugf("discover: %s: %v; no external lookup", extServer, err)
}
return nil
}
@ -401,7 +447,7 @@ func (d *Discoverer) externalLookup(device protocol.DeviceID) []string {
conn, err := net.DialUDP("udp", nil, extIP)
if err != nil {
if debug {
l.Debugf("discover: %v; no external lookup", err)
l.Debugf("discover: %s: %v; no external lookup", extServer, err)
}
return nil
}
@ -410,7 +456,7 @@ func (d *Discoverer) externalLookup(device protocol.DeviceID) []string {
err = conn.SetDeadline(time.Now().Add(5 * time.Second))
if err != nil {
if debug {
l.Debugf("discover: %v; no external lookup", err)
l.Debugf("discover: %s: %v; no external lookup", extServer, err)
}
return nil
}
@ -419,7 +465,7 @@ func (d *Discoverer) externalLookup(device protocol.DeviceID) []string {
_, err = conn.Write(buf)
if err != nil {
if debug {
l.Debugf("discover: %v; no external lookup", err)
l.Debugf("discover: %s: %v; no external lookup", extServer, err)
}
return nil
}
@ -432,20 +478,20 @@ func (d *Discoverer) externalLookup(device protocol.DeviceID) []string {
return nil
}
if debug {
l.Debugf("discover: %v; no external lookup", err)
l.Debugf("discover: %s: %v; no external lookup", extServer, err)
}
return nil
}
if debug {
l.Debugf("discover: read external:\n%s", hex.Dump(buf[:n]))
l.Debugf("discover: %s: read external:\n%s", extServer, hex.Dump(buf[:n]))
}
var pkt Announce
err = pkt.UnmarshalXDR(buf[:n])
if err != nil && err != io.EOF {
if debug {
l.Debugln("discover:", err)
l.Debugln("discover:", extServer, err)
}
return nil
}

View File

@ -1125,7 +1125,9 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
if (ignores != nil && ignores.Match(f.Name)) || symlinkInvalid(f.IsSymlink()) {
// File has been ignored or an unsupported symlink. Set invalid bit.
l.Debugln("setting invalid bit on ignored", f)
if debug {
l.Debugln("setting invalid bit on ignored", f)
}
nf := protocol.FileInfo{
Name: f.Name,
Flags: f.Flags | protocol.FlagInvalid,