Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

New global discovery protocol over HTTPS (fixes #628, fixes #1907)

This commit is contained in:
Jakob Borg 2015-09-20 15:30:25 +02:00
parent a7169a6348
commit b0cd7be39b
41 changed files with 1937 additions and 1504 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Godeps/Godeps.json generated
View File

@ -43,11 +43,11 @@
},
{
"ImportPath": "github.com/syncthing/relaysrv/client",
"Rev": "7fe1fdd8c751df165ea825bc8d3e895f118bb236"
"Rev": "6e126fb97e2ff566d35f8d8824e86793d22b2147"
},
{
"ImportPath": "github.com/syncthing/relaysrv/protocol",
"Rev": "7fe1fdd8c751df165ea825bc8d3e895f118bb236"
"Rev": "6e126fb97e2ff566d35f8d8824e86793d22b2147"
},
{
"ImportPath": "github.com/syndtr/goleveldb/leveldb",
@ -55,7 +55,7 @@
},
{
"ImportPath": "github.com/thejerf/suture",
"Rev": "fc7aaeabdc43fe41c5328efa1479ffea0b820978"
"Rev": "860b44045335c64a6d54ac7eed22a3aedfc687c9"
},
{
"ImportPath": "github.com/vitrun/qart/coding",

20
Godeps/_workspace/src/github.com/syncthing/relaysrv/client/client.go generated vendored
View File

@ -32,6 +32,7 @@ type ProtocolClient struct {
mut sync.RWMutex
connected bool
latency time.Duration
}
func NewProtocolClient(uri *url.URL, certs []tls.Certificate, invitations chan protocol.SessionInvitation) *ProtocolClient {
@ -168,6 +169,13 @@ func (c *ProtocolClient) StatusOK() bool {
return con
}
func (c *ProtocolClient) Latency() time.Duration {
c.mut.RLock()
lat := c.latency
c.mut.RUnlock()
return lat
}
func (c *ProtocolClient) String() string {
return fmt.Sprintf("ProtocolClient@%p", c)
}
@ -177,11 +185,21 @@ func (c *ProtocolClient) connect() error {
return fmt.Errorf("Unsupported relay schema:", c.URI.Scheme)
}
conn, err := tls.Dial("tcp", c.URI.Host, c.config)
t0 := time.Now()
tcpConn, err := net.Dial("tcp", c.URI.Host)
if err != nil {
return err
}
c.mut.Lock()
c.latency = time.Since(t0)
c.mut.Unlock()
conn := tls.Client(tcpConn, c.config)
if err = conn.Handshake(); err != nil {
return err
}
if err := conn.SetDeadline(time.Now().Add(10 * time.Second)); err != nil {
conn.Close()
return err

26
Godeps/_workspace/src/github.com/syncthing/relaysrv/client/methods.go generated vendored
View File

@ -8,6 +8,7 @@ import (
"net"
"net/url"
"strconv"
"strings"
"time"
syncthingprotocol "github.com/syncthing/protocol"
@ -20,10 +21,10 @@ func GetInvitationFromRelay(uri *url.URL, id syncthingprotocol.DeviceID, certs [
}
conn, err := tls.Dial("tcp", uri.Host, configForCerts(certs))
conn.SetDeadline(time.Now().Add(10 * time.Second))
if err != nil {
return protocol.SessionInvitation{}, err
}
conn.SetDeadline(time.Now().Add(10 * time.Second))
if err := performHandshakeAndValidation(conn, uri); err != nil {
return protocol.SessionInvitation{}, err
@ -97,6 +98,29 @@ func JoinSession(invitation protocol.SessionInvitation) (net.Conn, error) {
}
}
func TestRelay(uri *url.URL, certs []tls.Certificate, sleep time.Duration, times int) bool {
id := syncthingprotocol.NewDeviceID(certs[0].Certificate[0])
invs := make(chan protocol.SessionInvitation, 1)
c := NewProtocolClient(uri, certs, invs)
go c.Serve()
defer func() {
close(invs)
c.Stop()
}()
for i := 0; i < times; i++ {
_, err := GetInvitationFromRelay(uri, id, certs)
if err == nil {
return true
}
if !strings.Contains(err.Error(), "Incorrect response code") {
return false
}
time.Sleep(sleep)
}
return false
}
func configForCerts(certs []tls.Certificate) *tls.Config {
return &tls.Config{
Certificates: certs,

3
Godeps/_workspace/src/github.com/syncthing/relaysrv/protocol/packets.go generated vendored
View File

@ -7,8 +7,9 @@ package protocol
import (
"fmt"
syncthingprotocol "github.com/syncthing/protocol"
"net"
syncthingprotocol "github.com/syncthing/protocol"
)
const (

10
Godeps/_workspace/src/github.com/thejerf/suture/README.md generated vendored
View File

@ -6,10 +6,8 @@ Suture
Suture provides Erlang-ish supervisor trees for Go. "Supervisor trees" ->
"sutree" -> "suture" -> holds your code together when it's trying to die.
This is intended to be a production-quality library going into code that I
will be very early on the phone tree to support when it goes down. However,
it has not been deployed into something quite that serious yet. (I will
update this statement when that changes.)
This library has hit maturity, and isn't expected to be changed
radically. This can also be imported via gopkg.in/thejerf/suture.v1 .
It is intended to deal gracefully with the real failure cases that can
occur with supervision trees (such as burning all your CPU time endlessly
@ -24,10 +22,6 @@ This module is fully covered with [godoc](http://godoc.org/github.com/thejerf/su
including an example, usage, and everything else you might expect from a
README.md on GitHub. (DRY.)
This is not currently tagged with particular git tags for Go as this is
currently considered to be alpha code. As I move this into production and
feel more confident about it, I'll give it relevant tags.
Code Signing
------------

96
cmd/stfinddevice/main.go
View File

@ -7,41 +7,105 @@
package main
import (
"crypto/tls"
"errors"
"flag"
"log"
"fmt"
"net/url"
"os"
"time"
"github.com/syncthing/protocol"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/discover"
)
func main() {
log.SetFlags(0)
log.SetOutput(os.Stdout)
var timeout = 5 * time.Second
func main() {
var server string
flag.StringVar(&server, "server", "udp4://announce.syncthing.net:22027", "Announce server")
flag.StringVar(&server, "server", "", "Announce server (blank for default set)")
flag.DurationVar(&timeout, "timeout", timeout, "Query timeout")
flag.Usage = usage
flag.Parse()
if len(flag.Args()) != 1 || server == "" {
log.Printf("Usage: %s [-server=\"udp4://announce.syncthing.net:22027\"] <device>", os.Args[0])
if flag.NArg() != 1 {
flag.Usage()
os.Exit(64)
}
id, err := protocol.DeviceIDFromString(flag.Args()[0])
if err != nil {
log.Println(err)
fmt.Println(err)
os.Exit(1)
}
discoverer := discover.NewDiscoverer(protocol.LocalDeviceID, nil, nil)
discoverer.StartGlobal([]string{server}, nil)
addresses, relays := discoverer.Lookup(id)
for _, addr := range addresses {
log.Println("address:", addr)
}
for _, addr := range relays {
log.Println("relay:", addr)
if server != "" {
checkServers(id, server)
} else {
checkServers(id, config.DefaultDiscoveryServers...)
}
}
type checkResult struct {
server string
direct []string
relays []discover.Relay
error
}
func checkServers(deviceID protocol.DeviceID, servers ...string) {
t0 := time.Now()
resc := make(chan checkResult)
for _, srv := range servers {
srv := srv
go func() {
res := checkServer(deviceID, srv)
res.server = srv
resc <- res
}()
}
for _ = range servers {
res := <-resc
u, _ := url.Parse(res.server)
fmt.Printf("%s (%v):\n", u.Host, time.Since(t0))
if res.error != nil {
fmt.Println(" " + res.error.Error())
}
for _, addr := range res.direct {
fmt.Println(" address:", addr)
}
for _, rel := range res.relays {
fmt.Printf(" relay: %s (%d ms)\n", rel.URL, rel.Latency)
}
}
}
func checkServer(deviceID protocol.DeviceID, server string) checkResult {
disco, err := discover.NewGlobal(server, tls.Certificate{}, nil, nil)
if err != nil {
return checkResult{error: err}
}
res := make(chan checkResult, 1)
time.AfterFunc(timeout, func() {
res <- checkResult{error: errors.New("timeout")}
})
go func() {
direct, relays, err := disco.Lookup(deviceID)
res <- checkResult{direct: direct, relays: relays, error: err}
}()
return <-res
}
func usage() {
fmt.Printf("Usage:\n\t%s [options] <device ID>\n\nOptions:\n", os.Args[0])
flag.PrintDefaults()
}

16
cmd/syncthing/externaladdr.go → cmd/syncthing/addresslister.go
View File

@ -32,6 +32,17 @@ func newAddressLister(upnpSvc *upnpSvc, cfg *config.Wrapper) *addressLister {
// port number - this means that the outside address of a NAT gateway should
// be substituted.
func (e *addressLister) ExternalAddresses() []string {
return e.addresses(false)
}
// AllAddresses returns a list of addresses that are our best guess for where
// we are reachable from the local network. Same conditions as
// ExternalAddresses, but private IPv4 addresses are included.
func (e *addressLister) AllAddresses() []string {
return e.addresses(true)
}
func (e *addressLister) addresses(includePrivateIPV4 bool) []string {
var addrs []string
// Grab our listen addresses from the config. Unspecified ones are passed
@ -56,6 +67,9 @@ func (e *addressLister) ExternalAddresses() []string {
} else if isPublicIPv4(addr.IP) || isPublicIPv6(addr.IP) {
// A public address; include as is.
addrs = append(addrs, "tcp://"+addr.String())
} else if includePrivateIPV4 && addr.IP.To4().IsGlobalUnicast() {
// A private IPv4 address.
addrs = append(addrs, "tcp://"+addr.String())
}
}
@ -67,8 +81,6 @@ func (e *addressLister) ExternalAddresses() []string {
}
}
l.Infoln("External addresses:", addrs)
return addrs
}

15
cmd/syncthing/connections.go
View File

@ -43,7 +43,7 @@ type connectionSvc struct {
myID protocol.DeviceID
model *model.Model
tlsCfg *tls.Config
discoverer *discover.Discoverer
discoverer discover.Finder
conns chan model.IntermediateConnection
relaySvc *relay.Svc
@ -54,7 +54,7 @@ type connectionSvc struct {
relaysEnabled bool
}
func newConnectionSvc(cfg *config.Wrapper, myID protocol.DeviceID, mdl *model.Model, tlsCfg *tls.Config, discoverer *discover.Discoverer, relaySvc *relay.Svc) *connectionSvc {
func newConnectionSvc(cfg *config.Wrapper, myID protocol.DeviceID, mdl *model.Model, tlsCfg *tls.Config, discoverer discover.Finder, relaySvc *relay.Svc) *connectionSvc {
svc := &connectionSvc{
Supervisor: suture.NewSimple("connectionSvc"),
cfg: cfg,
@ -264,13 +264,14 @@ func (s *connectionSvc) connect() {
}
var addrs []string
var relays []string
var relays []discover.Relay
for _, addr := range deviceCfg.Addresses {
if addr == "dynamic" {
if s.discoverer != nil {
t, r := s.discoverer.Lookup(deviceID)
addrs = append(addrs, t...)
relays = append(relays, r...)
if t, r, err := s.discoverer.Lookup(deviceID); err == nil {
addrs = append(addrs, t...)
relays = append(relays, r...)
}
}
} else {
addrs = append(addrs, addr)
@ -333,7 +334,7 @@ func (s *connectionSvc) connect() {
s.lastRelayCheck[deviceID] = time.Now()
for _, addr := range relays {
uri, err := url.Parse(addr)
uri, err := url.Parse(addr.URL)
if err != nil {
l.Infoln("Failed to parse relay connection url:", addr, err)
continue

50
cmd/syncthing/gui.go
View File

@ -33,6 +33,7 @@ import (
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/model"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/relay"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/tlsutil"
"github.com/syncthing/syncthing/lib/upgrade"
@ -58,14 +59,15 @@ type apiSvc struct {
assetDir string
model *model.Model
eventSub *events.BufferedSubscription
discoverer *discover.Discoverer
discoverer *discover.CachingMux
relaySvc *relay.Svc
listener net.Listener
fss *folderSummarySvc
stop chan struct{}
systemConfigMut sync.Mutex
}
func newAPISvc(id protocol.DeviceID, cfg config.GUIConfiguration, assetDir string, m *model.Model, eventSub *events.BufferedSubscription, discoverer *discover.Discoverer) (*apiSvc, error) {
func newAPISvc(id protocol.DeviceID, cfg config.GUIConfiguration, assetDir string, m *model.Model, eventSub *events.BufferedSubscription, discoverer *discover.CachingMux, relaySvc *relay.Svc) (*apiSvc, error) {
svc := &apiSvc{
id: id,
cfg: cfg,
@ -73,6 +75,7 @@ func newAPISvc(id protocol.DeviceID, cfg config.GUIConfiguration, assetDir strin
model: m,
eventSub: eventSub,
discoverer: discoverer,
relaySvc: relaySvc,
systemConfigMut: sync.NewMutex(),
}
@ -164,7 +167,6 @@ func (s *apiSvc) Serve() {
postRestMux.HandleFunc("/rest/db/override", s.postDBOverride) // folder
postRestMux.HandleFunc("/rest/db/scan", s.postDBScan) // folder [sub...] [delay]
postRestMux.HandleFunc("/rest/system/config", s.postSystemConfig) // <body>
postRestMux.HandleFunc("/rest/system/discovery", s.postSystemDiscovery) // device addr
postRestMux.HandleFunc("/rest/system/error", s.postSystemError) // <body>
postRestMux.HandleFunc("/rest/system/error/clear", s.postSystemErrorClear) // -
postRestMux.HandleFunc("/rest/system/ping", s.restPing) // -
@ -630,11 +632,30 @@ func (s *apiSvc) getSystemStatus(w http.ResponseWriter, r *http.Request) {
res["alloc"] = m.Alloc
res["sys"] = m.Sys - m.HeapReleased
res["tilde"] = tilde
if cfg.Options().GlobalAnnEnabled && s.discoverer != nil {
res["extAnnounceOK"] = s.discoverer.ExtAnnounceOK()
if cfg.Options().LocalAnnEnabled || cfg.Options().GlobalAnnEnabled {
res["discoveryEnabled"] = true
discoErrors := make(map[string]string)
discoMethods := 0
for disco, err := range s.discoverer.ChildErrors() {
discoMethods++
if err != nil {
discoErrors[disco] = err.Error()
}
}
res["discoveryMethods"] = discoMethods
res["discoveryErrors"] = discoErrors
}
if relaySvc != nil {
res["relayClientStatus"] = relaySvc.ClientStatus()
if s.relaySvc != nil {
res["relaysEnabled"] = true
relayClientStatus := make(map[string]bool)
relayClientLatency := make(map[string]int)
for _, relay := range s.relaySvc.Relays() {
latency, ok := s.relaySvc.RelayStatus(relay)
relayClientStatus[relay] = ok
relayClientLatency[relay] = int(latency / time.Millisecond)
}
res["relayClientStatus"] = relayClientStatus
res["relayClientLatency"] = relayClientLatency
}
cpuUsageLock.RLock()
var cpusum float64
@ -679,25 +700,16 @@ func (s *apiSvc) showGuiError(l logger.LogLevel, err string) {
guiErrorsMut.Unlock()
}
func (s *apiSvc) postSystemDiscovery(w http.ResponseWriter, r *http.Request) {
var qs = r.URL.Query()
var device = qs.Get("device")
var addr = qs.Get("addr")
if len(device) != 0 && len(addr) != 0 && s.discoverer != nil {
s.discoverer.Hint(device, []string{addr})
}
}
func (s *apiSvc) getSystemDiscovery(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
devices := map[string][]discover.CacheEntry{}
devices := make(map[string]discover.CacheEntry)
if s.discoverer != nil {
// Device ids can't be marshalled as keys so we need to manually
// rebuild this map using strings. Discoverer may be nil if discovery
// has not started yet.
for device, entries := range s.discoverer.All() {
devices[device.String()] = entries
for device, entry := range s.discoverer.Cache() {
devices[device.String()] = entry
}
}

82
cmd/syncthing/main.go
View File

@ -114,7 +114,6 @@ var (
writeRateLimit *ratelimit.Bucket
readRateLimit *ratelimit.Bucket
stop = make(chan int)
relaySvc *relay.Svc
cert tls.Certificate
lans []*net.IPNet
)
@ -689,8 +688,7 @@ func syncthingMain() {
var addrList *addressLister
// Start UPnP. The UPnP service will restart global discovery if the
// external port changes.
// Start UPnP
if opts.UPnPEnabled {
upnpSvc := newUPnPSvc(cfg, addr.Port)
@ -703,14 +701,6 @@ func syncthingMain() {
addrList = newAddressLister(nil, cfg)
}
// Start discovery
discoverer := discovery(addrList, relaySvc)
// GUI
setupGUI(mainSvc, cfg, m, apiSub, discoverer)
// Start relay management
var relaySvc *relay.Svc
@ -719,9 +709,51 @@ func syncthingMain() {
mainSvc.Add(relaySvc)
}
// Start discovery
cachedDiscovery := discover.NewCachingMux()
mainSvc.Add(cachedDiscovery)
if cfg.Options().GlobalAnnEnabled {
for _, srv := range cfg.GlobalDiscoveryServers() {
l.Infoln("Using discovery server", srv)
gd, err := discover.NewGlobal(srv, cert, addrList, relaySvc)
if err != nil {
l.Warnln("Global discovery:", err)
continue
}
// Each global discovery server gets its results cached for five
// minutes, and is not asked again for a minute when it's returned
// unsuccessfully.
cachedDiscovery.Add(gd, 5*time.Minute, time.Minute)
}
}
if cfg.Options().LocalAnnEnabled {
// v4 broadcasts
bcd, err := discover.NewLocal(myID, fmt.Sprintf(":%d", cfg.Options().LocalAnnPort), addrList, relaySvc)
if err != nil {
l.Warnln("IPv4 local discovery:", err)
} else {
cachedDiscovery.Add(bcd, 0, 0)
}
// v6 multicasts
mcd, err := discover.NewLocal(myID, cfg.Options().LocalAnnMCAddr, addrList, relaySvc)
if err != nil {
l.Warnln("IPv6 local discovery:", err)
} else {
cachedDiscovery.Add(mcd, 0, 0)
}
}
// GUI
setupGUI(mainSvc, cfg, m, apiSub, cachedDiscovery, relaySvc)
// Start connection management
connectionSvc := newConnectionSvc(cfg, myID, m, tlsCfg, discoverer, relaySvc)
connectionSvc := newConnectionSvc(cfg, myID, m, tlsCfg, cachedDiscovery, relaySvc)
mainSvc.Add(connectionSvc)
if cpuProfile {
@ -844,7 +876,7 @@ func startAuditing(mainSvc *suture.Supervisor) {
l.Infoln("Audit log in", auditFile)
}
func setupGUI(mainSvc *suture.Supervisor, cfg *config.Wrapper, m *model.Model, apiSub *events.BufferedSubscription, discoverer *discover.Discoverer) {
func setupGUI(mainSvc *suture.Supervisor, cfg *config.Wrapper, m *model.Model, apiSub *events.BufferedSubscription, discoverer *discover.CachingMux, relaySvc *relay.Svc) {
opts := cfg.Options()
guiCfg := overrideGUIConfig(cfg.GUI(), guiAddress, guiAuthentication, guiAPIKey)
@ -873,7 +905,7 @@ func setupGUI(mainSvc *suture.Supervisor, cfg *config.Wrapper, m *model.Model, a
urlShow := fmt.Sprintf("%s://%s/", proto, net.JoinHostPort(hostShow, strconv.Itoa(addr.Port)))
l.Infoln("Starting web GUI on", urlShow)
api, err := newAPISvc(myID, guiCfg, guiAssets, m, apiSub, discoverer)
api, err := newAPISvc(myID, guiCfg, guiAssets, m, apiSub, discoverer, relaySvc)
if err != nil {
l.Fatalln("Cannot start GUI:", err)
}
@ -944,28 +976,6 @@ func shutdown() {
stop <- exitSuccess
}
func discovery(addrList *addressLister, relaySvc *relay.Svc) *discover.Discoverer {
opts := cfg.Options()
disc := discover.NewDiscoverer(myID, opts.ListenAddress, relaySvc)
if opts.LocalAnnEnabled {
l.Infoln("Starting local discovery announcements")
disc.StartLocal(opts.LocalAnnPort, opts.LocalAnnMCAddr)
}
if opts.GlobalAnnEnabled {
go func() {
// Defer starting global announce server, giving time to connect
// to relay servers.
time.Sleep(5 * time.Second)
l.Infoln("Starting global discovery announcements")
disc.StartGlobal(opts.GlobalAnnServers, addrList)
}()
}
return disc
}
func ensureDir(dir string, mode int) {
fi, err := os.Stat(dir)
if os.IsNotExist(err) {

11
cmd/syncthing/verbose.go
View File

@ -8,6 +8,7 @@ package main
import (
"fmt"
"strings"
"github.com/syncthing/syncthing/lib/events"
)
@ -139,6 +140,16 @@ func (s *verboseSvc) formatEvent(ev events.Event) string {
data := ev.Data.(map[string]string)
device := data["device"]
return fmt.Sprintf("Device %v was resumed", device)
case events.ExternalPortMappingChanged:
data := ev.Data.(map[string]int)
port := data["port"]
return fmt.Sprintf("External port mapping changed; new port is %d.", port)
case events.RelayStateChanged:
data := ev.Data.(map[string][]string)
newRelays := data["new"]
return fmt.Sprintf("Relay state changed; connected relay(s) are %s.", strings.Join(newRelays, ", "))
}
return fmt.Sprintf("%s %#v", ev.Type, ev)

4
gui/assets/css/overrides.css
View File

@ -249,3 +249,7 @@ ul.three-columns li, ul.two-columns li {
position: static;
}
}
.popover {
min-width: 250px;
}

26
gui/index.html
View File

@ -379,28 +379,28 @@
<th><span class="fa fa-fw fa-tachometer"></span>&nbsp;<span translate>CPU Utilization</span></th>
<td class="text-right">{{system.cpuPercent | alwaysNumber | natural:1}}%</td>
</tr>
<tr ng-if="system.extAnnounceOK != undefined && announceServersTotal > 0">
<th><span class="fa fa-fw fa-bullhorn"></span>&nbsp;<span translate>Global Discovery</span></th>
<tr ng-if="system.discoveryEnabled">
<th><span class="fa fa-fw fa-map-signs"></span>&nbsp;<span translate>Discovery</span></th>
<td class="text-right">
<span ng-if="announceServersFailed.length == 0" class="data text-success">
<span>OK</span>
<span ng-if="discoveryFailed.length == 0" class="data text-success">
<span>{{discoveryTotal}}/{{discoveryTotal}}</span>
</span>
<span ng-if="announceServersFailed.length != 0" class="data" ng-class="{'text-danger': announceServersFailed.length == announceServersTotal}">
<span popover data-trigger="hover" data-placement="bottom" data-content="{{announceServersFailed.join('\n')}}">
{{announceServersTotal-announceServersFailed.length}}/{{announceServersTotal}}
<span ng-if="discoveryFailed.length != 0" class="data" ng-class="{'text-danger': discoveryFailed.length == discoveryTotal}">
<span popover data-trigger="hover" data-placement="bottom" data-html="true" data-content="{{discoveryFailed.join('<br>\n')}}">
{{discoveryTotal-discoveryFailed.length}}/{{discoveryTotal}}
</span>
</span>
</td>
</tr>
<tr ng-if="system.relayClientStatus != undefined && relayClientsTotal > 0">
<tr ng-if="system.relaysEnabled">
<th><span class="fa fa-fw fa-sitemap"></span>&nbsp;<span translate>Relays</span></th>
<td class="text-right">
<span ng-if="relayClientsFailed.length == 0" class="data text-success">
<span>OK</span>
<span ng-if="relaysFailed.length == 0" class="data text-success">
<span>{{relaysTotal}}/{{relaysTotal}}</span>
</span>
<span ng-if="relayClientsFailed.length != 0" class="data" ng-class="{'text-danger': relayClientsFailed.length == relayClientsTotal}">
<span popover data-trigger="hover" data-placement="bottom" data-content="{{relayClientsFailed.join('\n')}}">
{{relayClientsTotal-relayClientsFailed.length}}/{{relayClientsTotal}}
<span ng-if="relaysFailed.length != 0" class="data" ng-class="{'text-danger': relaysFailed.length == relaysTotal}">
<span popover data-trigger="hover" data-placement="bottom" data-html="true" data-content="{{relaysFailed.join('<br>\n')}}">
{{relaysTotal-relaysFailed.length}}/{{relaysTotal}}
</span>
</span>
</td>

23
gui/syncthing/core/syncthingController.js
View File

@ -378,24 +378,25 @@ angular.module('syncthing.core')
$scope.myID = data.myID;
$scope.system = data;
$scope.announceServersTotal = data.extAnnounceOK ? Object.keys(data.extAnnounceOK).length : 0;
var failedAnnounce = [];
for (var server in data.extAnnounceOK) {
if (!data.extAnnounceOK[server]) {
failedAnnounce.push(server);
$scope.discoveryTotal = data.discoveryMethods;
var discoveryFailed = [];
for (var disco in data.discoveryErrors) {
if (data.discoveryErrors[disco]) {
discoveryFailed.push(disco + ": " + data.discoveryErrors[disco]);
}
}
$scope.announceServersFailed = failedAnnounce;
$scope.discoveryFailed = discoveryFailed;
$scope.relayClientsTotal = data.relayClientStatus ? Object.keys(data.relayClientStatus).length : 0;
var failedRelays = [];
var relaysFailed = [];
var relaysTotal = 0;
for (var relay in data.relayClientStatus) {
if (!data.relayClientStatus[relay]) {
failedRelays.push(relay);
relaysFailed.push(relay);
}
relaysTotal++;
}
$scope.relayClientsFailed = failedRelays;
$scope.relaysFailed = relaysFailed;
$scope.relaysTotal = relaysTotal;
console.log("refreshSystem", data);
}).error($scope.emitHTTPError);

2
gui/syncthing/device/editDeviceModalView.html
View File

@ -13,7 +13,7 @@
<label translate for="deviceID">Device ID</label>
<input ng-if="!editingExisting" name="deviceID" id="deviceID" class="form-control text-monospace" type="text" ng-model="currentDevice.deviceID" required valid-deviceid list="discovery-list" />
<datalist id="discovery-list" ng-if="!editingExisting">
<option ng-repeat="(id,address) in discovery" value="{{ id }}" />
<option ng-repeat="(id, data) in discovery" value="{{id}}" />
</datalist>
<div ng-if="editingExisting" class="well well-sm text-monospace">{{currentDevice.deviceID}}</div>
<p class="help-block">

8
lib/auto/gui.files.go
View File

File diff suppressed because one or more lines are too long

47
lib/beacon/beacon.go
View File

@ -6,7 +6,12 @@
package beacon
import "net"
import (
"net"
stdsync "sync"
"github.com/thejerf/suture"
)
type recv struct {
data []byte
@ -14,34 +19,30 @@ type recv struct {
}
type Interface interface {
suture.Service
Send(data []byte)
Recv() ([]byte, net.Addr)
Error() error
}
type readerFrom interface {
ReadFrom([]byte) (int, net.Addr, error)
}
func genericReader(conn readerFrom, outbox chan<- recv) {
bs := make([]byte, 65536)
for {
n, addr, err := conn.ReadFrom(bs)
if err != nil {
l.Warnln("multicast read:", err)
return
}
if debug {
l.Debugf("recv %d bytes from %s", n, addr)
}
c := make([]byte, n)
copy(c, bs)
select {
case outbox <- recv{c, addr}:
default:
if debug {
l.Debugln("dropping message")
}
}
}
type errorHolder struct {
err error
mut stdsync.Mutex // uses stdlib sync as I want this to be trivially embeddable, and there is no risk of blocking
}
func (e *errorHolder) setError(err error) {
e.mut.Lock()
e.err = err
e.mut.Unlock()
}
func (e *errorHolder) Error() error {
e.mut.Lock()
err := e.err
e.mut.Unlock()
return err
}

70
lib/beacon/broadcast.go
View File

@ -19,6 +19,8 @@ type Broadcast struct {
port int
inbox chan []byte
outbox chan recv
br *broadcastReader
bw *broadcastWriter
}
func NewBroadcast(port int) *Broadcast {
@ -41,14 +43,16 @@ func NewBroadcast(port int) *Broadcast {
outbox: make(chan recv, 16),
}
b.Add(&broadcastReader{
b.br = &broadcastReader{
port: port,
outbox: b.outbox,
})
b.Add(&broadcastWriter{
}
b.Add(b.br)
b.bw = &broadcastWriter{
port: port,
inbox: b.inbox,
})
}
b.Add(b.bw)
return b
}
@ -62,11 +66,18 @@ func (b *Broadcast) Recv() ([]byte, net.Addr) {
return recv.data, recv.src
}
func (b *Broadcast) Error() error {
if err := b.br.Error(); err != nil {
return err
}
return b.bw.Error()
}
type broadcastWriter struct {
port int
inbox chan []byte
conn *net.UDPConn
failed bool // Have we already logged a failure reason?
port int
inbox chan []byte
conn *net.UDPConn
errorHolder
}
func (w *broadcastWriter) Serve() {
@ -78,22 +89,21 @@ func (w *broadcastWriter) Serve() {
var err error
w.conn, err = net.ListenUDP("udp4", nil)
if err != nil {
if !w.failed {
l.Warnln("Local discovery over IPv4 unavailable:", err)
w.failed = true
if debug {
l.Debugln(err)
}
w.setError(err)
return
}
defer w.conn.Close()
w.failed = false
for bs := range w.inbox {
addrs, err := net.InterfaceAddrs()
if err != nil {
if debug {
l.Debugln("Local discovery (broadcast writer):", err)
l.Debugln(err)
}
w.setError(err)
continue
}
@ -117,13 +127,16 @@ func (w *broadcastWriter) Serve() {
for _, ip := range dsts {
dst := &net.UDPAddr{IP: ip, Port: w.port}
w.conn.SetWriteDeadline(time.Now().Add(10 * time.Second))
w.conn.SetWriteDeadline(time.Now().Add(time.Second))
_, err := w.conn.WriteTo(bs, dst)
w.conn.SetWriteDeadline(time.Time{})
if err, ok := err.(net.Error); ok && err.Timeout() {
// Write timeouts should not happen. We treat it as a fatal
// error on the socket.
l.Infoln("Local discovery (broadcast writer):", err)
w.failed = true
if debug {
l.Debugln(err)
}
w.setError(err)
return
} else if err, ok := err.(net.Error); ok && err.Temporary() {
// A transient error. Lets hope for better luck in the future.
@ -133,11 +146,14 @@ func (w *broadcastWriter) Serve() {
continue
} else if err != nil {
// Some other error that we don't expect. Bail and retry.
l.Infoln("Local discovery (broadcast writer):", err)
w.failed = true
if debug {
l.Debugln(err)
}
w.setError(err)
return
} else if debug {
l.Debugf("sent %d bytes to %s", len(bs), dst)
w.setError(nil)
}
}
}
@ -155,7 +171,7 @@ type broadcastReader struct {
port int
outbox chan recv
conn *net.UDPConn
failed bool
errorHolder
}
func (r *broadcastReader) Serve() {
@ -167,10 +183,10 @@ func (r *broadcastReader) Serve() {
var err error
r.conn, err = net.ListenUDP("udp4", &net.UDPAddr{Port: r.port})
if err != nil {
if !r.failed {
l.Warnln("Local discovery over IPv4 unavailable:", err)
r.failed = true
if debug {
l.Debugln(err)
}
r.setError(err)
return
}
defer r.conn.Close()
@ -179,14 +195,14 @@ func (r *broadcastReader) Serve() {
for {
n, addr, err := r.conn.ReadFrom(bs)
if err != nil {
if !r.failed {
l.Infoln("Local discovery (broadcast reader):", err)
r.failed = true
if debug {
l.Debugln(err)
}
r.setError(err)
return
}
r.failed = false
r.setError(nil)
if debug {
l.Debugf("recv %d bytes from %s", n, addr)

255
lib/beacon/multicast.go
View File

@ -8,39 +8,200 @@ package beacon
import (
"errors"
"fmt"
"net"
"time"
"github.com/thejerf/suture"
"golang.org/x/net/ipv6"
)
type Multicast struct {
conn *ipv6.PacketConn
*suture.Supervisor
addr *net.UDPAddr
inbox chan []byte
outbox chan recv
intfs []net.Interface
mr *multicastReader
mw *multicastWriter
}
func NewMulticast(addr string) (*Multicast, error) {
gaddr, err := net.ResolveUDPAddr("udp6", addr)
if err != nil {
return nil, err
func NewMulticast(addr string) *Multicast {
m := &Multicast{
Supervisor: suture.New("multicastBeacon", suture.Spec{
// Don't retry too frenetically: an error to open a socket or
// whatever is usually something that is either permanent or takes
// a while to get solved...
FailureThreshold: 2,
FailureBackoff: 60 * time.Second,
// Only log restarts in debug mode.
Log: func(line string) {
if debug {
l.Debugln(line)
}
},
}),
inbox: make(chan []byte),
outbox: make(chan recv, 16),
}
conn, err := net.ListenPacket("udp6", addr)
m.mr = &multicastReader{
addr: addr,
outbox: m.outbox,
stop: make(chan struct{}),
}
m.Add(m.mr)
m.mw = &multicastWriter{
addr: addr,
inbox: m.inbox,
stop: make(chan struct{}),
}
m.Add(m.mw)
return m
}
func (m *Multicast) Send(data []byte) {
m.inbox <- data
}
func (m *Multicast) Recv() ([]byte, net.Addr) {
recv := <-m.outbox
return recv.data, recv.src
}
func (m *Multicast) Error() error {
if err := m.mr.Error(); err != nil {
return err
}
return m.mw.Error()
}
type multicastWriter struct {
addr string
inbox <-chan []byte
errorHolder
stop chan struct{}
}
func (w *multicastWriter) Serve() {
if debug {
l.Debugln(w, "starting")
defer l.Debugln(w, "stopping")
}
gaddr, err := net.ResolveUDPAddr("udp6", w.addr)
if err != nil {
return nil, err
if debug {
l.Debugln(err)
}
w.setError(err)
return
}
conn, err := net.ListenPacket("udp6", ":0")
if err != nil {
if debug {
l.Debugln(err)
}
w.setError(err)
return
}
pconn := ipv6.NewPacketConn(conn)
wcm := &ipv6.ControlMessage{
HopLimit: 1,
}
for bs := range w.inbox {
intfs, err := net.Interfaces()
if err != nil {
if debug {
l.Debugln(err)
}
w.setError(err)
return
}
var success int
for _, intf := range intfs {
wcm.IfIndex = intf.Index
pconn.SetWriteDeadline(time.Now().Add(time.Second))
_, err = pconn.WriteTo(bs, wcm, gaddr)
pconn.SetWriteDeadline(time.Time{})
if err != nil && debug {
l.Debugln(err, "on write to", gaddr, intf.Name)
} else if debug {
l.Debugf("sent %d bytes to %v on %s", len(bs), gaddr, intf.Name)
success++
}
}
if success > 0 {
w.setError(nil)
} else {
if debug {
l.Debugln(err)
}
w.setError(err)
}
}
}
func (w *multicastWriter) Stop() {
close(w.stop)
}
func (w *multicastWriter) String() string {
return fmt.Sprintf("multicastWriter@%p", w)
}
type multicastReader struct {
addr string
outbox chan<- recv
errorHolder
stop chan struct{}
}
func (r *multicastReader) Serve() {
if debug {
l.Debugln(r, "starting")
defer l.Debugln(r, "stopping")
}
gaddr, err := net.ResolveUDPAddr("udp6", r.addr)
if err != nil {
if debug {
l.Debugln(err)
}
r.setError(err)
return
}
conn, err := net.ListenPacket("udp6", r.addr)
if err != nil {
if debug {
l.Debugln(err)
}
r.setError(err)
return
}
intfs, err := net.Interfaces()
if err != nil {
return nil, err
if debug {
l.Debugln(err)
}
r.setError(err)
return
}
p := ipv6.NewPacketConn(conn)
pconn := ipv6.NewPacketConn(conn)
joined := 0
for _, intf := range intfs {
err := p.JoinGroup(&intf, &net.UDPAddr{IP: gaddr.IP})
err := pconn.JoinGroup(&intf, &net.UDPAddr{IP: gaddr.IP})
if debug {
if err != nil {
l.Debugln("IPv6 join", intf.Name, "failed:", err)
@ -52,57 +213,43 @@ func NewMulticast(addr string) (*Multicast, error) {
}
if joined == 0 {
return nil, errors.New("no multicast interfaces available")
if debug {
l.Debugln("no multicast interfaces available")
}
r.setError(errors.New("no multicast interfaces available"))
return
}
b := &Multicast{
conn: p,
addr: gaddr,
inbox: make(chan []byte),
outbox: make(chan recv, 16),
intfs: intfs,
}
bs := make([]byte, 65536)
for {
n, _, addr, err := pconn.ReadFrom(bs)
if err != nil {
if debug {
l.Debugln(err)
}
r.setError(err)
continue
}
if debug {
l.Debugf("recv %d bytes from %s", n, addr)
}
go genericReader(ipv6ReaderAdapter{b.conn}, b.outbox)
go b.writer()
return b, nil
}
func (b *Multicast) Send(data []byte) {
b.inbox <- data
}
func (b *Multicast) Recv() ([]byte, net.Addr) {
recv := <-b.outbox
return recv.data, recv.src
}
func (b *Multicast) writer() {
wcm := &ipv6.ControlMessage{
HopLimit: 1,
}
for bs := range b.inbox {
for _, intf := range b.intfs {
wcm.IfIndex = intf.Index
_, err := b.conn.WriteTo(bs, wcm, b.addr)
if err != nil && debug {
l.Debugln(err, "on write to", b.addr)
} else if debug {
l.Debugf("sent %d bytes to %v on %s", len(bs), b.addr, intf.Name)
c := make([]byte, n)
copy(c, bs)
select {
case r.outbox <- recv{c, addr}:
default:
if debug {
l.Debugln("dropping message")
}
}
}
}
// This makes ReadFrom on an *ipv6.PacketConn behave like ReadFrom on a
// net.PacketConn.
type ipv6ReaderAdapter struct {
c *ipv6.PacketConn
func (r *multicastReader) Stop() {
close(r.stop)
}
func (i ipv6ReaderAdapter) ReadFrom(bs []byte) (int, net.Addr, error) {
n, _, src, err := i.c.ReadFrom(bs)
return n, src, err
func (r *multicastReader) String() string {
return fmt.Sprintf("multicastReader@%p", r)
}

35
lib/config/config.go
View File

@ -31,6 +31,21 @@ const (
MaxRescanIntervalS = 365 * 24 * 60 * 60
)
var (
// DefaultDiscoveryServers should be substituted when the configuration
// contains <globalAnnounceServer>default</globalAnnounceServer>. This is
// done by the "consumer" of the configuration, as we don't want these
// saved to the config.
DefaultDiscoveryServers = []string{
"https://v4-1.discover.syncthing.net/?id=SR7AARM-TCBUZ5O-VFAXY4D-CECGSDE-3Q6IZ4G-XG7AH75-OBIXJQV-QJ6NLQA", // 194.126.249.5, Sweden
"https://v4-2.discover.syncthing.net/?id=AQEHEO2-XOS7QRA-X2COH5K-PO6OPVA-EWOSEGO-KZFMD32-XJ4ZV46-CUUVKAS", // 45.55.230.38, USA
"https://v4-3.discover.syncthing.net/?id=7WT2BVR-FX62ZOW-TNVVW25-6AHFJGD-XEXQSBW-VO3MPL2-JBTLL4T-P4572Q4", // 128.199.95.124, Singapore
"https://v6-1.discover.syncthing.net/?id=SR7AARM-TCBUZ5O-VFAXY4D-CECGSDE-3Q6IZ4G-XG7AH75-OBIXJQV-QJ6NLQA", // 2001:470:28:4d6::5, Sweden
"https://v6-2.discover.syncthing.net/?id=AQEHEO2-XOS7QRA-X2COH5K-PO6OPVA-EWOSEGO-KZFMD32-XJ4ZV46-CUUVKAS", // 2604:a880:800:10::182:a001, USA
"https://v6-3.discover.syncthing.net/?id=7WT2BVR-FX62ZOW-TNVVW25-6AHFJGD-XEXQSBW-VO3MPL2-JBTLL4T-P4572Q4", // 2400:6180:0:d0::d9:d001, Singapore
}
)
type Configuration struct {
Version int `xml:"version,attr" json:"version"`
Folders []FolderConfiguration `xml:"folder" json:"folders"`
@ -215,7 +230,7 @@ type FolderDeviceConfiguration struct {
type OptionsConfiguration struct {
ListenAddress []string `xml:"listenAddress" json:"listenAddress" default:"tcp://0.0.0.0:22000"`
GlobalAnnServers []string `xml:"globalAnnounceServer" json:"globalAnnounceServers" json:"globalAnnounceServer" default:"udp4://announce.syncthing.net:22027, udp6://announce-v6.syncthing.net:22027"`
GlobalAnnServers []string `xml:"globalAnnounceServer" json:"globalAnnounceServers" json:"globalAnnounceServer" default:"default"`
GlobalAnnEnabled bool `xml:"globalAnnounceEnabled" json:"globalAnnounceEnabled" default:"true"`
LocalAnnEnabled bool `xml:"localAnnounceEnabled" json:"localAnnounceEnabled" default:"true"`
LocalAnnPort int `xml:"localAnnouncePort" json:"localAnnouncePort" default:"21027"`
@ -498,17 +513,21 @@ func convertV11V12(cfg *Configuration) {
}
// Use new discovery server
for i, addr := range cfg.Options.GlobalAnnServers {
var newDiscoServers []string
var useDefault bool
for _, addr := range cfg.Options.GlobalAnnServers {
if addr == "udp4://announce.syncthing.net:22026" {
cfg.Options.GlobalAnnServers[i] = "udp4://announce.syncthing.net:22027"
useDefault = true
} else if addr == "udp6://announce-v6.syncthing.net:22026" {
cfg.Options.GlobalAnnServers[i] = "udp6://announce-v6.syncthing.net:22027"
} else if addr == "udp4://194.126.249.5:22026" {
cfg.Options.GlobalAnnServers[i] = "udp4://194.126.249.5:22027"
} else if addr == "udp6://[2001:470:28:4d6::5]:22026" {
cfg.Options.GlobalAnnServers[i] = "udp6://[2001:470:28:4d6::5]:22027"
useDefault = true
} else {
newDiscoServers = append(newDiscoServers, addr)
}
}
if useDefault {
newDiscoServers = append(newDiscoServers, "default")
}
cfg.Options.GlobalAnnServers = newDiscoServers
// Use new multicast group
if cfg.Options.LocalAnnMCAddr == "[ff32::5222]:21026" {

2
lib/config/config_test.go
View File

@ -32,7 +32,7 @@ func init() {
func TestDefaultValues(t *testing.T) {
expected := OptionsConfiguration{
ListenAddress: []string{"tcp://0.0.0.0:22000"},
GlobalAnnServers: []string{"udp4://announce.syncthing.net:22027", "udp6://announce-v6.syncthing.net:22027"},
GlobalAnnServers: []string{"default"},
GlobalAnnEnabled: true,
LocalAnnEnabled: true,
LocalAnnPort: 21027,

12
lib/config/wrapper.go
View File

@ -317,3 +317,15 @@ func (w *Wrapper) Save() error {
events.Default.Log(events.ConfigSaved, w.cfg)
return nil
}
func (w *Wrapper) GlobalDiscoveryServers() []string {
var servers []string
for _, srv := range w.cfg.Options.GlobalAnnServers {
if srv == "default" {
servers = append(servers, DefaultDiscoveryServers...)
} else {
servers = append(servers, srv)
}
}
return uniqueStrings(servers)
}

192
lib/discover/cache.go Normal file
View File

@ -0,0 +1,192 @@
package discover
import (
stdsync "sync"
"time"
"github.com/syncthing/protocol"
"github.com/syncthing/syncthing/lib/sync"
"github.com/thejerf/suture"
)
// The CachingMux aggregates results from multiple Finders. Each Finder has
// an associated cache time and negative cache time. The cache time sets how
// long we cache and return successfull lookup results, the negative cache
// time sets how long we refrain from asking about the same device ID after
// receiving a negative answer. The value of zero disables caching (positive
// or negative).
type CachingMux struct {
*suture.Supervisor
finders []cachedFinder
caches []*cache
mut sync.Mutex
}
// A cachedFinder is a Finder with associated cache timeouts.
type cachedFinder struct {
Finder
cacheTime time.Duration
negCacheTime time.Duration
}
func NewCachingMux() *CachingMux {
return &CachingMux{
Supervisor: suture.NewSimple("discover.cachingMux"),
mut: sync.NewMutex(),
}
}
// Add registers a new Finder, with associated cache timeouts.
func (m *CachingMux) Add(finder Finder, cacheTime, negCacheTime time.Duration) {
m.mut.Lock()
m.finders = append(m.finders, cachedFinder{finder, cacheTime, negCacheTime})
m.caches = append(m.caches, newCache())
m.mut.Unlock()
if svc, ok := finder.(suture.Service); ok {
m.Supervisor.Add(svc)
}
}
// Lookup attempts to resolve the device ID using any of the added Finders,
// while obeying the cache settings.
func (m *CachingMux) Lookup(deviceID protocol.DeviceID) (direct []string, relays []Relay, err error) {
m.mut.Lock()
for i, finder := range m.finders {
if cacheEntry, ok := m.caches[i].Get(deviceID); ok {
// We have a cache entry. Lets see what it says.
if cacheEntry.found && time.Since(cacheEntry.when) < finder.cacheTime {
// It's a positive, valid entry. Use it.
if debug {
l.Debugln("cached discovery entry for", deviceID, "at", finder.String())
l.Debugln(" ", cacheEntry)
}
direct = append(direct, cacheEntry.Direct...)
relays = append(relays, cacheEntry.Relays...)
continue
}
if !cacheEntry.found && time.Since(cacheEntry.when) < finder.negCacheTime {
// It's a negative, valid entry. We should not make another
// attempt right now.
if debug {
l.Debugln("negative cache entry for", deviceID, "at", finder.String())
}
continue
}
// It's expired. Ignore and continue.
}
// Perform the actual lookup and cache the result.
if td, tr, err := finder.Lookup(deviceID); err == nil {
if debug {
l.Debugln("lookup for", deviceID, "at", finder.String())
l.Debugln(" ", td)
l.Debugln(" ", tr)
}
direct = append(direct, td...)
relays = append(relays, tr...)
m.caches[i].Set(deviceID, CacheEntry{
Direct: td,
Relays: tr,
when: time.Now(),
found: len(td)+len(tr) > 0,
})
}
}
m.mut.Unlock()
if debug {
l.Debugln("lookup results for", deviceID)
l.Debugln(" ", direct)
l.Debugln(" ", relays)
}
return direct, relays, nil
}
func (m *CachingMux) String() string {
return "discovery cache"
}
func (m *CachingMux) Error() error {
return nil
}
func (m *CachingMux) ChildErrors() map[string]error {
m.mut.Lock()
children := make(map[string]error, len(m.finders))
for _, f := range m.finders {
children[f.String()] = f.Error()
}
m.mut.Unlock()
return children
}
func (m *CachingMux) Cache() map[protocol.DeviceID]CacheEntry {
// Res will be the "total" cache, i.e. the union of our cache and all our
// children's caches.
res := make(map[protocol.DeviceID]CacheEntry)
m.mut.Lock()
for i := range m.finders {
// Each finder[i] has a corresponding cache at cache[i]. Go through it
// and populate the total, if it's newer than what's already in there.
// We skip any negative cache entries.
for k, v := range m.caches[i].Cache() {
if v.found && v.when.After(res[k].when) {
res[k] = v
}
}
// Then ask the finder itself for it's cache and do the same. If this
// finder is a global discovery client, it will have no cache. If it's
// a local discovery client, this will be it's current state.
for k, v := range m.finders[i].Cache() {
if v.found && v.when.After(res[k].when) {
res[k] = v
}
}
}
m.mut.Unlock()
return res
}
// A cache can be embedded wherever useful
type cache struct {
entries map[protocol.DeviceID]CacheEntry
mut stdsync.Mutex
}
func newCache() *cache {
return &cache{
entries: make(map[protocol.DeviceID]CacheEntry),
}
}
func (c *cache) Set(id protocol.DeviceID, ce CacheEntry) {
c.mut.Lock()
c.entries[id] = ce
c.mut.Unlock()
}
func (c *cache) Get(id protocol.DeviceID) (CacheEntry, bool) {
c.mut.Lock()
ce, ok := c.entries[id]
c.mut.Unlock()
return ce, ok
}
func (c *cache) Cache() map[protocol.DeviceID]CacheEntry {
c.mut.Lock()
m := make(map[protocol.DeviceID]CacheEntry, len(c.entries))
for k, v := range c.entries {
m[k] = v
}
c.mut.Unlock()
return m
}

54
lib/discover/client.go
View File

@ -1,54 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package discover
import (
"fmt"
"net/url"
"time"
"github.com/syncthing/protocol"
)
type Announcer interface {
Announcement() Announce
}
type Factory func(*url.URL, Announcer) (Client, error)
var (
factories = make(map[string]Factory)
DefaultErrorRetryInternval = 60 * time.Second
DefaultGlobalBroadcastInterval = 1800 * time.Second
)
func Register(proto string, factory Factory) {
factories[proto] = factory
}
func New(addr string, announcer Announcer) (Client, error) {
uri, err := url.Parse(addr)
if err != nil {
return nil, err
}
factory, ok := factories[uri.Scheme]
if !ok {
return nil, fmt.Errorf("Unsupported scheme: %s", uri.Scheme)
}
client, err := factory(uri, announcer)
if err != nil {
return nil, err
}
return client, nil
}
type Client interface {
Lookup(device protocol.DeviceID) (Announce, error)
StatusOK() bool
Address() string
Stop()
}

239
lib/discover/client_test.go
View File

@ -1,239 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package discover
import (
"fmt"
"net"
"time"
"testing"
"github.com/syncthing/protocol"
"github.com/syncthing/syncthing/lib/sync"
)
var device protocol.DeviceID
func init() {
device, _ = protocol.DeviceIDFromString("P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2")
}
type FakeAnnouncer struct {
pkt Announce
}
func (f *FakeAnnouncer) Announcement() Announce {
return f.pkt
}
func TestUDP4Success(t *testing.T) {
conn, err := net.ListenUDP("udp4", nil)
if err != nil {
t.Fatal(err)
}
port := conn.LocalAddr().(*net.UDPAddr).Port
address := fmt.Sprintf("udp4://127.0.0.1:%d", port)
pkt := Announce{
Magic: AnnouncementMagic,
This: Device{
device[:],
[]string{"tcp://123.123.123.123:1234"},
nil,
},
}
ann := &FakeAnnouncer{
pkt: pkt,
}
client, err := New(address, ann)
if err != nil {
t.Fatal(err)
}
udpclient := client.(*UDPClient)
if udpclient.errorRetryInterval != DefaultErrorRetryInternval {
t.Fatal("Incorrect retry interval")
}
if udpclient.listenAddress.IP != nil || udpclient.listenAddress.Port != 0 {
t.Fatal("Wrong listen IP or port", udpclient.listenAddress)
}
if client.Address() != address {
t.Fatal("Incorrect address")
}
buf := make([]byte, 2048)
// First announcement
conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
_, err = conn.Read(buf)
if err != nil {
t.Fatal(err)
}
// Announcement verification
conn.SetDeadline(time.Now().Add(time.Millisecond * 1100))
_, addr, err := conn.ReadFromUDP(buf)
if err != nil {
t.Fatal(err)
}
// Reply to it.
_, err = conn.WriteToUDP(pkt.MustMarshalXDR(), addr)
if err != nil {
t.Fatal(err)
}
// We should get nothing else
conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
_, err = conn.Read(buf)
if err == nil {
t.Fatal("Expected error")
}
// Status should be ok
if !client.StatusOK() {
t.Fatal("Wrong status")
}
// Do a lookup in a separate routine
addrs := []string{}
wg := sync.NewWaitGroup()
wg.Add(1)
go func() {
pkt, err := client.Lookup(device)
if err == nil {
for _, addr := range pkt.This.Addresses {
addrs = append(addrs, addr)
}
}
wg.Done()
}()
// Receive the lookup and reply
conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
_, addr, err = conn.ReadFromUDP(buf)
if err != nil {
t.Fatal(err)
}
conn.WriteToUDP(pkt.MustMarshalXDR(), addr)
// Wait for the lookup to arrive, verify that the number of answers is correct
wg.Wait()
if len(addrs) != 1 || addrs[0] != "tcp://123.123.123.123:1234" {
t.Fatal("Wrong number of answers")
}
client.Stop()
}
func TestUDP4Failure(t *testing.T) {
conn, err := net.ListenUDP("udp4", nil)
if err != nil {
t.Fatal(err)
}
port := conn.LocalAddr().(*net.UDPAddr).Port
address := fmt.Sprintf("udp4://127.0.0.1:%d/?listenaddress=127.0.0.1&retry=5", port)
pkt := Announce{
Magic: AnnouncementMagic,
This: Device{
device[:],
[]string{"tcp://123.123.123.123:1234"},
nil,
},
}
ann := &FakeAnnouncer{
pkt: pkt,
}
client, err := New(address, ann)
if err != nil {
t.Fatal(err)
}
udpclient := client.(*UDPClient)
if udpclient.errorRetryInterval != time.Second*5 {
t.Fatal("Incorrect retry interval")
}
if !udpclient.listenAddress.IP.Equal(net.IPv4(127, 0, 0, 1)) || udpclient.listenAddress.Port != 0 {
t.Fatal("Wrong listen IP or port", udpclient.listenAddress)
}
if client.Address() != address {
t.Fatal("Incorrect address")
}
buf := make([]byte, 2048)
// First announcement
conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
_, err = conn.Read(buf)
if err != nil {
t.Fatal(err)
}
// Announcement verification
conn.SetDeadline(time.Now().Add(time.Millisecond * 1100))
_, _, err = conn.ReadFromUDP(buf)
if err != nil {
t.Fatal(err)
}
// Don't reply
// We should get nothing else
conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
_, err = conn.Read(buf)
if err == nil {
t.Fatal("Expected error")
}
// Status should be failure
if client.StatusOK() {
t.Fatal("Wrong status")
}
// Do a lookup in a separate routine
addrs := []string{}
wg := sync.NewWaitGroup()
wg.Add(1)
go func() {
pkt, err := client.Lookup(device)
if err == nil {
for _, addr := range pkt.This.Addresses {
addrs = append(addrs, addr)
}
}
wg.Done()
}()
// Receive the lookup and don't reply
conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
_, _, err = conn.ReadFromUDP(buf)
if err != nil {
t.Fatal(err)
}
// Wait for the lookup to timeout, verify that the number of answers is none
wg.Wait()
if len(addrs) != 0 {
t.Fatal("Wrong number of answers")
}
client.Stop()
}

261
lib/discover/client_udp.go
View File

@ -1,261 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package discover
import (
"encoding/hex"
"io"
"net"
"net/url"
"strconv"
"time"
"github.com/syncthing/protocol"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/sync"
)
func init() {
for _, proto := range []string{"udp", "udp4", "udp6"} {
Register(proto, func(uri *url.URL, announcer Announcer) (Client, error) {
c := &UDPClient{
announcer: announcer,
wg: sync.NewWaitGroup(),
mut: sync.NewRWMutex(),
}
err := c.Start(uri)
if err != nil {
return nil, err
}
return c, nil
})
}
}
type UDPClient struct {
url *url.URL
stop chan struct{}
wg sync.WaitGroup
listenAddress *net.UDPAddr
globalBroadcastInterval time.Duration
errorRetryInterval time.Duration
announcer Announcer
status bool
mut sync.RWMutex
}
func (d *UDPClient) Start(uri *url.URL) error {
d.url = uri
d.stop = make(chan struct{})
params := uri.Query()
// The address must not have a port, as otherwise both announce and lookup
// sockets would try to bind to the same port.
addr, err := net.ResolveUDPAddr(d.url.Scheme, params.Get("listenaddress")+":0")
if err != nil {
return err
}
d.listenAddress = addr
broadcastSeconds, err := strconv.ParseUint(params.Get("broadcast"), 0, 0)
if err != nil {
d.globalBroadcastInterval = DefaultGlobalBroadcastInterval
} else {
d.globalBroadcastInterval = time.Duration(broadcastSeconds) * time.Second
}
retrySeconds, err := strconv.ParseUint(params.Get("retry"), 0, 0)
if err != nil {
d.errorRetryInterval = DefaultErrorRetryInternval
} else {
d.errorRetryInterval = time.Duration(retrySeconds) * time.Second
}
d.wg.Add(1)
go d.broadcast()
return nil
}
func (d *UDPClient) broadcast() {
defer d.wg.Done()
conn, err := net.ListenUDP(d.url.Scheme, d.listenAddress)
for err != nil {
if debug {
l.Debugf("discover %s: broadcast listen: %v; trying again in %v", d.url, err, d.errorRetryInterval)
}
select {
case <-d.stop:
return
case <-time.After(d.errorRetryInterval):
}
conn, err = net.ListenUDP(d.url.Scheme, d.listenAddress)
}
defer conn.Close()
remote, err := net.ResolveUDPAddr(d.url.Scheme, d.url.Host)
for err != nil {
if debug {
l.Debugf("discover %s: broadcast resolve: %v; trying again in %v", d.url, err, d.errorRetryInterval)
}
select {
case <-d.stop:
return
case <-time.After(d.errorRetryInterval):
}
remote, err = net.ResolveUDPAddr(d.url.Scheme, d.url.Host)
}
timer := time.NewTimer(0)
eventSub := events.Default.Subscribe(events.ExternalPortMappingChanged)
defer events.Default.Unsubscribe(eventSub)
for {
select {
case <-d.stop:
return
case <-eventSub.C():
ok := d.sendAnnouncement(remote, conn)
d.mut.Lock()
d.status = ok
d.mut.Unlock()
case <-timer.C:
ok := d.sendAnnouncement(remote, conn)
d.mut.Lock()
d.status = ok
d.mut.Unlock()
if ok {
timer.Reset(d.globalBroadcastInterval)
} else {
timer.Reset(d.errorRetryInterval)
}
}
}
}
func (d *UDPClient) sendAnnouncement(remote net.Addr, conn *net.UDPConn) bool {
if debug {
l.Debugf("discover %s: broadcast: Sending self announcement to %v", d.url, remote)
}
ann := d.announcer.Announcement()
pkt, err := ann.MarshalXDR()
if err != nil {
return false
}
myID := protocol.DeviceIDFromBytes(ann.This.ID)
_, err = conn.WriteTo(pkt, remote)
if err != nil {
if debug {
l.Debugf("discover %s: broadcast: Failed to send self announcement: %s", d.url, err)
}
return false
}
// Verify that the announce server responds positively for our device ID
time.Sleep(1 * time.Second)
ann, err = d.Lookup(myID)
if err != nil && debug {
l.Debugf("discover %s: broadcast: Self-lookup failed: %v", d.url, err)
} else if debug {
l.Debugf("discover %s: broadcast: Self-lookup returned: %v", d.url, ann.This.Addresses)
}
return len(ann.This.Addresses) > 0
}
func (d *UDPClient) Lookup(device protocol.DeviceID) (Announce, error) {
extIP, err := net.ResolveUDPAddr(d.url.Scheme, d.url.Host)
if err != nil {
if debug {
l.Debugf("discover %s: Lookup(%s): %s", d.url, device, err)
}
return Announce{}, err
}
conn, err := net.DialUDP(d.url.Scheme, d.listenAddress, extIP)
if err != nil {
if debug {
l.Debugf("discover %s: Lookup(%s): %s", d.url, device, err)
}
return Announce{}, err
}
defer conn.Close()
err = conn.SetDeadline(time.Now().Add(5 * time.Second))
if err != nil {
if debug {
l.Debugf("discover %s: Lookup(%s): %s", d.url, device, err)
}
return Announce{}, err
}
buf := Query{QueryMagic, device[:]}.MustMarshalXDR()
_, err = conn.Write(buf)
if err != nil {
if debug {
l.Debugf("discover %s: Lookup(%s): %s", d.url, device, err)
}
return Announce{}, err
}
buf = make([]byte, 2048)
n, err := conn.Read(buf)
if err != nil {
if err, ok := err.(net.Error); ok && err.Timeout() {
// Expected if the server doesn't know about requested device ID
return Announce{}, err
}
if debug {
l.Debugf("discover %s: Lookup(%s): %s", d.url, device, err)
}
return Announce{}, err
}
var pkt Announce
err = pkt.UnmarshalXDR(buf[:n])
if err != nil && err != io.EOF {
if debug {
l.Debugf("discover %s: Lookup(%s): %s\n%s", d.url, device, err, hex.Dump(buf[:n]))
}
return Announce{}, err
}
if debug {
l.Debugf("discover %s: Lookup(%s) result: %v relays: %v", d.url, device, pkt.This.Addresses, pkt.This.Relays)
}
return pkt, nil
}
func (d *UDPClient) Stop() {
if d.stop != nil {
close(d.stop)
d.wg.Wait()
}
}
func (d *UDPClient) StatusOK() bool {
d.mut.RLock()
defer d.mut.RUnlock()
return d.status
}
func (d *UDPClient) Address() string {
return d.url.String()
}

549
lib/discover/discover.go
View File

@ -1,4 +1,4 @@
// Copyright (C) 2014 The Syncthing Authors.
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
@ -7,539 +7,48 @@
package discover
import (
"bytes"
"encoding/hex"
"errors"
"fmt"
"io"
"net"
"net/url"
"sort"
"time"
"github.com/syncthing/protocol"
"github.com/syncthing/syncthing/lib/beacon"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/sync"
"github.com/thejerf/suture"
)
type Discoverer struct {
myID protocol.DeviceID
listenAddrs []string
relayStatusProvider relayStatusProvider
localBcastIntv time.Duration
localBcastStart time.Time
cacheLifetime time.Duration
negCacheCutoff time.Duration
beacons []beacon.Interface
extAddr externalAddr
localBcastTick <-chan time.Time
forcedBcastTick chan time.Time
registryLock sync.RWMutex
addressRegistry map[protocol.DeviceID][]CacheEntry
relayRegistry map[protocol.DeviceID][]CacheEntry
lastLookup map[protocol.DeviceID]time.Time
clients []Client
mut sync.RWMutex
}
type relayStatusProvider interface {
ClientStatus() map[string]bool
}
type externalAddr interface {
ExternalAddresses() []string
// A Finder provides lookup services of some kind.
type Finder interface {
Lookup(deviceID protocol.DeviceID) (direct []string, relays []Relay, err error)
Error() error
String() string
Cache() map[protocol.DeviceID]CacheEntry
}
type CacheEntry struct {
Address string
Seen time.Time
Direct []string `json:"direct"`
Relays []Relay `json:"relays"`
when time.Time // When did we get the result
found bool // Is it a success (cacheTime applies) or a failure (negCacheTime applies)?
}
var (
ErrIncorrectMagic = errors.New("incorrect magic number")
)
func NewDiscoverer(id protocol.DeviceID, addresses []string, relayStatusProvider relayStatusProvider) *Discoverer {
return &Discoverer{
myID: id,
listenAddrs: addresses,
relayStatusProvider: relayStatusProvider,
localBcastIntv: 30 * time.Second,
cacheLifetime: 5 * time.Minute,
negCacheCutoff: 3 * time.Minute,
addressRegistry: make(map[protocol.DeviceID][]CacheEntry),
relayRegistry: make(map[protocol.DeviceID][]CacheEntry),
lastLookup: make(map[protocol.DeviceID]time.Time),
registryLock: sync.NewRWMutex(),
mut: sync.NewRWMutex(),
}
// A FinderService is a Finder that has background activity and must be run as
// a suture.Service.
type FinderService interface {
Finder
suture.Service
}
func (d *Discoverer) StartLocal(localPort int, localMCAddr string) {
if localPort > 0 {
d.startLocalIPv4Broadcasts(localPort)
}
if len(localMCAddr) > 0 {
d.startLocalIPv6Multicasts(localMCAddr)
}
if len(d.beacons) == 0 {
l.Warnln("Local discovery unavailable")
return
}
d.localBcastTick = time.Tick(d.localBcastIntv)
d.forcedBcastTick = make(chan time.Time)
d.localBcastStart = time.Now()
go d.sendLocalAnnouncements()
type FinderMux interface {
Finder
ChildStatus() map[string]error
}
func (d *Discoverer) startLocalIPv4Broadcasts(localPort int) {
bb := beacon.NewBroadcast(localPort)
d.beacons = append(d.beacons, bb)
go d.recvAnnouncements(bb)
bb.ServeBackground()
// The RelayStatusProvider answers questions about current relay status.
type RelayStatusProvider interface {
Relays() []string
RelayStatus(uri string) (time.Duration, bool)
}
func (d *Discoverer) startLocalIPv6Multicasts(localMCAddr string) {
mb, err := beacon.NewMulticast(localMCAddr)
if err != nil {
if debug {
l.Debugln("beacon.NewMulticast:", err)
}
l.Infoln("Local discovery over IPv6 unavailable")
return
}
d.beacons = append(d.beacons, mb)
go d.recvAnnouncements(mb)
}
func (d *Discoverer) StartGlobal(servers []string, extAddr externalAddr) {
d.mut.Lock()
defer d.mut.Unlock()
if len(d.clients) > 0 {
d.stopGlobal()
}
d.extAddr = extAddr
wg := sync.NewWaitGroup()
clients := make(chan Client, len(servers))
for _, address := range servers {
wg.Add(1)
go func(addr string) {
defer wg.Done()
client, err := New(addr, d)
if err != nil {
l.Infoln("Error creating discovery client", addr, err)
return
}
clients <- client
}(address)
}
wg.Wait()
close(clients)
for client := range clients {
d.clients = append(d.clients, client)
}
}
func (d *Discoverer) StopGlobal() {
d.mut.Lock()
defer d.mut.Unlock()
d.stopGlobal()
}
func (d *Discoverer) stopGlobal() {
for _, client := range d.clients {
client.Stop()
}
d.clients = []Client{}
}
func (d *Discoverer) ExtAnnounceOK() map[string]bool {
d.mut.RLock()
defer d.mut.RUnlock()
ret := make(map[string]bool)
for _, client := range d.clients {
ret[client.Address()] = client.StatusOK()
}
return ret
}
// Lookup returns a list of addresses the device is available at, as well as
// a list of relays the device is supposed to be available on sorted by the
// sum of latencies between this device, and the device in question.
func (d *Discoverer) Lookup(device protocol.DeviceID) ([]string, []string) {
d.registryLock.RLock()
cachedAddresses := d.filterCached(d.addressRegistry[device])
cachedRelays := d.filterCached(d.relayRegistry[device])
lastLookup := d.lastLookup[device]
d.registryLock.RUnlock()
d.mut.RLock()
defer d.mut.RUnlock()
relays := make([]string, len(cachedRelays))
for i := range cachedRelays {
relays[i] = cachedRelays[i].Address
}
if len(cachedAddresses) > 0 {
// There are cached address entries.
addrs := make([]string, len(cachedAddresses))
for i := range cachedAddresses {
addrs[i] = cachedAddresses[i].Address
}
return addrs, relays
}
if time.Since(lastLookup) < d.negCacheCutoff {
// We have recently tried to lookup this address and failed. Lets
// chill for a while.
return nil, relays
}
if len(d.clients) != 0 && time.Since(d.localBcastStart) > d.localBcastIntv {
// Only perform external lookups if we have at least one external
// server client and one local announcement interval has passed. This is
// to avoid finding local peers on their remote address at startup.
results := make(chan Announce, len(d.clients))
wg := sync.NewWaitGroup()
for _, client := range d.clients {
wg.Add(1)
go func(c Client) {
defer wg.Done()
ann, err := c.Lookup(device)
if err == nil {
results <- ann
}
}(client)
}
wg.Wait()
close(results)
cachedAddresses := []CacheEntry{}
availableRelays := []Relay{}
seenAddresses := make(map[string]struct{})
seenRelays := make(map[string]struct{})
now := time.Now()
var addrs []string
for result := range results {
for _, addr := range result.This.Addresses {
_, ok := seenAddresses[addr]
if !ok {
cachedAddresses = append(cachedAddresses, CacheEntry{
Address: addr,
Seen: now,
})
seenAddresses[addr] = struct{}{}
addrs = append(addrs, addr)
}
}
for _, relay := range result.This.Relays {
_, ok := seenRelays[relay.Address]
if !ok {
availableRelays = append(availableRelays, relay)
seenRelays[relay.Address] = struct{}{}
}
}
}
relays = RelayAddressesSortedByLatency(availableRelays)
cachedRelays := make([]CacheEntry, len(relays))
for i := range relays {
cachedRelays[i] = CacheEntry{
Address: relays[i],
Seen: now,
}
}
d.registryLock.Lock()
d.addressRegistry[device] = cachedAddresses
d.relayRegistry[device] = cachedRelays
d.lastLookup[device] = time.Now()
d.registryLock.Unlock()
return addrs, relays
}
return nil, relays
}
func (d *Discoverer) Hint(device string, addrs []string) {
resAddrs := resolveAddrs(addrs)
var id protocol.DeviceID
id.UnmarshalText([]byte(device))
d.registerDevice(nil, Device{
Addresses: resAddrs,
ID: id[:],
})
}
func (d *Discoverer) All() map[protocol.DeviceID][]CacheEntry {
d.registryLock.RLock()
devices := make(map[protocol.DeviceID][]CacheEntry, len(d.addressRegistry))
for device, addrs := range d.addressRegistry {
addrsCopy := make([]CacheEntry, len(addrs))
copy(addrsCopy, addrs)
devices[device] = addrsCopy
}
d.registryLock.RUnlock()
return devices
}
func (d *Discoverer) Announcement() Announce {
return d.announcementPkt(true)
}
func (d *Discoverer) announcementPkt(allowExternal bool) Announce {
var addrs []string
if allowExternal && d.extAddr != nil {
addrs = d.extAddr.ExternalAddresses()
} else {
addrs = resolveAddrs(d.listenAddrs)
}
var relayAddrs []string
if d.relayStatusProvider != nil {
status := d.relayStatusProvider.ClientStatus()
for uri, ok := range status {
if ok {
relayAddrs = append(relayAddrs, uri)
}
}
}
return Announce{
Magic: AnnouncementMagic,
This: Device{d.myID[:], addrs, measureLatency(relayAddrs)},
}
}
func (d *Discoverer) sendLocalAnnouncements() {
var pkt = d.announcementPkt(false)
msg := pkt.MustMarshalXDR()
for {
for _, b := range d.beacons {
b.Send(msg)
}
select {
case <-d.localBcastTick:
case <-d.forcedBcastTick:
}
}
}
func (d *Discoverer) recvAnnouncements(b beacon.Interface) {
for {
buf, addr := b.Recv()
var pkt Announce
err := pkt.UnmarshalXDR(buf)
if err != nil && err != io.EOF {
if debug {
l.Debugf("discover: Failed to unmarshal local announcement from %s:\n%s", addr, hex.Dump(buf))
}
continue
}
if debug {
l.Debugf("discover: Received local announcement from %s for %s", addr, protocol.DeviceIDFromBytes(pkt.This.ID))
}
var newDevice bool
if bytes.Compare(pkt.This.ID, d.myID[:]) != 0 {
newDevice = d.registerDevice(addr, pkt.This)
}
if newDevice {
select {
case d.forcedBcastTick <- time.Now():
}
}
}
}
func (d *Discoverer) registerDevice(addr net.Addr, device Device) bool {
var id protocol.DeviceID
copy(id[:], device.ID)
d.registryLock.Lock()
defer d.registryLock.Unlock()
current := d.filterCached(d.addressRegistry[id])
orig := current
for _, deviceAddr := range device.Addresses {
uri, err := url.Parse(deviceAddr)
if err != nil {
if debug {
l.Debugf("discover: Failed to parse address %s: %s", deviceAddr, err)
}
continue
}
host, port, err := net.SplitHostPort(uri.Host)
if err != nil {
if debug {
l.Debugf("discover: Failed to split address host %s: %s", deviceAddr, err)
}
continue
}
if host == "" {
uri.Host = net.JoinHostPort(addr.(*net.UDPAddr).IP.String(), port)
deviceAddr = uri.String()
}
for i := range current {
if current[i].Address == deviceAddr {
current[i].Seen = time.Now()
goto done
}
}
current = append(current, CacheEntry{
Address: deviceAddr,
Seen: time.Now(),
})
done:
}
if debug {
l.Debugf("discover: Caching %s addresses: %v", id, current)
}
d.addressRegistry[id] = current
if len(current) > len(orig) {
addrs := make([]string, len(current))
for i := range current {
addrs[i] = current[i].Address
}
events.Default.Log(events.DeviceDiscovered, map[string]interface{}{
"device": id.String(),
"addrs": addrs,
})
}
return len(current) > len(orig)
}
func (d *Discoverer) filterCached(c []CacheEntry) []CacheEntry {
for i := 0; i < len(c); {
if ago := time.Since(c[i].Seen); ago > d.cacheLifetime {
if debug {
l.Debugf("discover: Removing cached entry %s - seen %v ago", c[i].Address, ago)
}
c[i] = c[len(c)-1]
c = c[:len(c)-1]
} else {
i++
}
}
return c
}
func addrToAddr(addr *net.TCPAddr) string {
if len(addr.IP) == 0 || addr.IP.IsUnspecified() {
return fmt.Sprintf(":%d", addr.Port)
} else if bs := addr.IP.To4(); bs != nil {
return fmt.Sprintf("%s:%d", bs.String(), addr.Port)
} else if bs := addr.IP.To16(); bs != nil {
return fmt.Sprintf("[%s]:%d", bs.String(), addr.Port)
}
return ""
}
func resolveAddrs(addrs []string) []string {
var raddrs []string
for _, addrStr := range addrs {
uri, err := url.Parse(addrStr)
if err != nil {
continue
}
addrRes, err := net.ResolveTCPAddr("tcp", uri.Host)
if err != nil {
continue
}
addr := addrToAddr(addrRes)
if len(addr) > 0 {
uri.Host = addr
raddrs = append(raddrs, uri.String())
}
}
return raddrs
}
func measureLatency(relayAdresses []string) []Relay {
relays := make([]Relay, 0, len(relayAdresses))
for i, addr := range relayAdresses {
relay := Relay{
Address: addr,
Latency: int32(time.Hour / time.Millisecond),
}
relays = append(relays, relay)
if latency, err := osutil.GetLatencyForURL(addr); err == nil {
if debug {
l.Debugf("Relay %s latency %s", addr, latency)
}
relays[i].Latency = int32(latency / time.Millisecond)
} else {
l.Debugf("Failed to get relay %s latency %s", addr, err)
}
}
return relays
}
// RelayAddressesSortedByLatency adds local latency to the relay, and sorts them
// by sum latency, and returns the addresses.
func RelayAddressesSortedByLatency(input []Relay) []string {
relays := make([]Relay, len(input))
copy(relays, input)
for i, relay := range relays {
if latency, err := osutil.GetLatencyForURL(relay.Address); err == nil {
relays[i].Latency += int32(latency / time.Millisecond)
} else {
relays[i].Latency += int32(time.Hour / time.Millisecond)
}
}
sort.Sort(relayList(relays))
addresses := make([]string, 0, len(relays))
for _, relay := range relays {
addresses = append(addresses, relay.Address)
}
return addresses
}
type relayList []Relay
func (l relayList) Len() int {
return len(l)
}
func (l relayList) Less(a, b int) bool {
return l[a].Latency < l[b].Latency
}
func (l relayList) Swap(a, b int) {
l[a], l[b] = l[b], l[a]
// The AddressLister answers questions about what addresses we are listening
// on.
type AddressLister interface {
ExternalAddresses() []string
AllAddresses() []string
}

163
lib/discover/discover_test.go
View File

@ -1,163 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package discover
import (
"net/url"
"time"
"testing"
"github.com/syncthing/protocol"
)
type DummyClient struct {
url *url.URL
lookups []protocol.DeviceID
lookupRet Announce
stops int
statusRet bool
statusChecks int
}
func (c *DummyClient) Lookup(device protocol.DeviceID) (Announce, error) {
c.lookups = append(c.lookups, device)
return c.lookupRet, nil
}
func (c *DummyClient) StatusOK() bool {
c.statusChecks++
return c.statusRet
}
func (c *DummyClient) Stop() {
c.stops++
}
func (c *DummyClient) Address() string {
return c.url.String()
}
func TestGlobalDiscovery(t *testing.T) {
c1 := &DummyClient{
statusRet: false,
lookupRet: Announce{
Magic: AnnouncementMagic,
This: Device{
ID: protocol.LocalDeviceID[:],
Addresses: []string{"test.com:1234"},
Relays: nil,
},
Extra: nil,
},
}
c2 := &DummyClient{
statusRet: true,
lookupRet: Announce{
Magic: AnnouncementMagic,
This: Device{
ID: protocol.LocalDeviceID[:],
Addresses: nil,
Relays: nil,
},
Extra: nil,
},
}
c3 := &DummyClient{
statusRet: true,
lookupRet: Announce{
Magic: AnnouncementMagic,
This: Device{
ID: protocol.LocalDeviceID[:],
Addresses: []string{"best.com:2345"},
Relays: nil,
},
Extra: nil,
},
}
clients := []*DummyClient{c1, c2}
Register("test1", func(uri *url.URL, ann Announcer) (Client, error) {
c := clients[0]
clients = clients[1:]
c.url = uri
return c, nil
})
Register("test2", func(uri *url.URL, ann Announcer) (Client, error) {
c3.url = uri
return c3, nil
})
d := NewDiscoverer(device, []string{}, nil)
d.localBcastStart = time.Time{}
servers := []string{
"test1://123.123.123.123:1234",
"test1://23.23.23.23:234",
"test2://234.234.234.234.2345",
}
d.StartGlobal(servers, nil)
if len(d.clients) != 3 {
t.Fatal("Wrong number of clients")
}
status := d.ExtAnnounceOK()
for _, c := range []*DummyClient{c1, c2, c3} {
if status[c.url.String()] != c.statusRet || c.statusChecks != 1 {
t.Fatal("Wrong status")
}
}
addrs, _ := d.Lookup(device)
if len(addrs) != 2 {
t.Fatal("Wrong number of addresses", addrs)
}
for _, addr := range []string{"test.com:1234", "best.com:2345"} {
found := false
for _, laddr := range addrs {
if laddr == addr {
found = true
break
}
}
if !found {
t.Fatal("Couldn't find", addr)
}
}
for _, c := range []*DummyClient{c1, c2, c3} {
if len(c.lookups) != 1 || c.lookups[0] != device {
t.Fatal("Wrong lookups")
}
}
addrs, _ = d.Lookup(device)
if len(addrs) != 2 {
t.Fatal("Wrong number of addresses", addrs)
}
// Answer should be cached, so number of lookups should have not increased
for _, c := range []*DummyClient{c1, c2, c3} {
if len(c.lookups) != 1 || c.lookups[0] != device {
t.Fatal("Wrong lookups")
}
}
d.StopGlobal()
for _, c := range []*DummyClient{c1, c2, c3} {
if c.stops != 1 {
t.Fatal("Wrong number of stops")
}
}
}

71
lib/discover/doc.go
View File

@ -1,8 +1,75 @@
// Copyright (C) 2014 The Syncthing Authors.
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
// Package discover implements the device discovery protocol.
/*
Package discover implements the local and global device discovery protocols.
Global Discovery
================
Announcements
-------------
A device should announce itself at startup. It does this by an HTTPS POST to
the announce server URL (with the path usually being "/", but this is of
course up to the discovery server). The POST has a JSON payload listing direct
connection addresses (if any) and relay addresses (if any).
{
direct: ["tcp://192.0.2.45:22000", "tcp://:22202"],
relays: [{"url": "relay://192.0.2.99:22028", "latency": 142}]
}
It's OK for either of the "direct" or "relays" fields to be either the empty
list ([]), null, or missing entirely. An announcment with both fields missing
or empty is however not useful...
Any empty or unspecified IP addresses (i.e. addresses like tcp://:22000,
tcp://0.0.0.0:22000, tcp://[::]:22000) are interpreted as referring to the
source IP address of the announcement.
The device ID of the announcing device is not part of the announcement.
Instead, the server requires that the client perform certificate
authentication. The device ID is deduced from the presented certificate.
The server response is empty, with code 200 (OK) on success. If no certificate
was presented, status 403 (Forbidden) is returned. If the posted data doesn't
conform to the expected format, 400 (Bad Request) is returned.
In successfull responses, the server may return a "Reannounce-After" header
containing the number of seconds after which the client should perform a new
announcement.
In error responses, the server may return a "Retry-After" header containing
the number of seconds after which the client should retry.
Performing announcements significantly more often than indicated by the
Reannounce-After or Retry-After headers may result in the client being
throttled. In such cases the server may respond with status code 429 (Too Many
Requests).
Queries
=======
Queries are performed as HTTPS GET requests to the announce server URL. The
requested device ID is passed as the query parameter "device", in canonical
string form, i.e. https://announce.syncthing.net/?device=ABC12345-....
Successfull responses will have status code 200 (OK) and carry a JSON payload
of the same format as the announcement above. The response will not contain
empty or unspecified addresses.
If the "device" query parameter is missing or malformed, the status code 400
(Bad Request) is returned.
If the device ID is of a valid format but not found in the registry, 404 (Not
Found) is returned.
If the client has exceeded a rate limit, the server may respond with 429 (Too
Many Requests).
*/
package discover

385
lib/discover/global.go Normal file
View File

@ -0,0 +1,385 @@
package discover
import (
"bytes"
"crypto/tls"
"encoding/json"
"errors"
"io"
"net/http"
"net/url"
"strconv"
stdsync "sync"
"time"
"github.com/syncthing/protocol"
"github.com/syncthing/syncthing/lib/events"
)
type globalClient struct {
server string
addrList AddressLister
relayStat RelayStatusProvider
announceClient httpClient
queryClient httpClient
noAnnounce bool
stop chan struct{}
errorHolder
}
type httpClient interface {
Get(url string) (*http.Response, error)
Post(url, ctype string, data io.Reader) (*http.Response, error)
}
const (
defaultReannounceInterval = 30 * time.Minute
announceErrorRetryInterval = 5 * time.Minute
)
type announcement struct {
Direct []string `json:"direct"`
Relays []Relay `json:"relays"`
}
type serverOptions struct {
insecure bool // don't check certificate
noAnnounce bool // don't announce
id string // expected server device ID
}
func NewGlobal(server string, cert tls.Certificate, addrList AddressLister, relayStat RelayStatusProvider) (FinderService, error) {
server, opts, err := parseOptions(server)
if err != nil {
return nil, err
}
var devID protocol.DeviceID
if opts.id != "" {
devID, err = protocol.DeviceIDFromString(opts.id)
if err != nil {
return nil, err
}
}
// The http.Client used for announcements. It needs to have our
// certificate to prove our identity, and may or may not verify the server
// certificate depending on the insecure setting.
var announceClient httpClient = &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: opts.insecure,
Certificates: []tls.Certificate{cert},
},
},
}
if opts.id != "" {
announceClient = newIDCheckingHTTPClient(announceClient, devID)
}
// The http.Client used for queries. We don't need to present our
// certificate here, so lets not include it. May be insecure if requested.
var queryClient httpClient = &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: opts.insecure,
},
},
}
if opts.id != "" {
queryClient = newIDCheckingHTTPClient(queryClient, devID)
}
cl := &globalClient{
server: server,
addrList: addrList,
relayStat: relayStat,
announceClient: announceClient,
queryClient: queryClient,
noAnnounce: opts.noAnnounce,
stop: make(chan struct{}),
}
cl.setError(errors.New("not announced"))
return cl, nil
}
// Lookup returns the list of addresses where the given device is available;
// direct, and via relays.
func (c *globalClient) Lookup(device protocol.DeviceID) (direct []string, relays []Relay, err error) {
qURL, err := url.Parse(c.server)
if err != nil {
return nil, nil, err
}
q := qURL.Query()
q.Set("device", device.String())
qURL.RawQuery = q.Encode()
resp, err := c.queryClient.Get(qURL.String())
if err != nil {
if debug {
l.Debugln("globalClient.Lookup", qURL.String(), err)
}
return nil, nil, err
}
if resp.StatusCode != 200 {
resp.Body.Close()
if debug {
l.Debugln("globalClient.Lookup", qURL.String(), resp.Status)
}
return nil, nil, errors.New(resp.Status)
}
// TODO: Handle 429 and Retry-After?
var ann announcement
err = json.NewDecoder(resp.Body).Decode(&ann)
resp.Body.Close()
return ann.Direct, ann.Relays, err
}
func (c *globalClient) String() string {
return "global@" + c.server
}
func (c *globalClient) Serve() {
if c.noAnnounce {
// We're configured to not do announcements, only lookups. To maintain
// the same interface, we just pause here if Serve() is run.
<-c.stop
return
}
timer := time.NewTimer(0)
defer timer.Stop()
eventSub := events.Default.Subscribe(events.ExternalPortMappingChanged | events.RelayStateChanged)
defer events.Default.Unsubscribe(eventSub)
for {
select {
case <-eventSub.C():
c.sendAnnouncement(timer)
case <-timer.C:
c.sendAnnouncement(timer)
case <-c.stop:
return
}
}
}
func (c *globalClient) sendAnnouncement(timer *time.Timer) {
var ann announcement
if c.addrList != nil {
ann.Direct = c.addrList.ExternalAddresses()
}
if c.relayStat != nil {
for _, relay := range c.relayStat.Relays() {
latency, ok := c.relayStat.RelayStatus(relay)
if ok {
ann.Relays = append(ann.Relays, Relay{
URL: relay,
Latency: int32(latency / time.Millisecond),
})
}
}
}
if len(ann.Direct)+len(ann.Relays) == 0 {
c.setError(errors.New("nothing to announce"))
if debug {
l.Debugln("Nothing to announce")
}
timer.Reset(announceErrorRetryInterval)
return
}
// The marshal doesn't fail, I promise.
postData, _ := json.Marshal(ann)
if debug {
l.Debugf("Announcement: %s", postData)
}
resp, err := c.announceClient.Post(c.server, "application/json", bytes.NewReader(postData))
if err != nil {
if debug {
l.Debugln("announce POST:", err)
}
c.setError(err)
timer.Reset(announceErrorRetryInterval)
return
}
if debug {
l.Debugln("announce POST:", resp.Status)
}
resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode > 299 {
if debug {
l.Debugln("announce POST:", resp.Status)
}
c.setError(errors.New(resp.Status))
if h := resp.Header.Get("Retry-After"); h != "" {
// The server has a recommendation on when we should
// retry. Follow it.
if secs, err := strconv.Atoi(h); err == nil && secs > 0 {
if debug {
l.Debugln("announce Retry-After:", secs, err)
}
timer.Reset(time.Duration(secs) * time.Second)
return
}
}
timer.Reset(announceErrorRetryInterval)
return
}
c.setError(nil)
if h := resp.Header.Get("Reannounce-After"); h != "" {
// The server has a recommendation on when we should
// reannounce. Follow it.
if secs, err := strconv.Atoi(h); err == nil && secs > 0 {
if debug {
l.Debugln("announce Reannounce-After:", secs, err)
}
timer.Reset(time.Duration(secs) * time.Second)
return
}
}
timer.Reset(defaultReannounceInterval)
}
func (c *globalClient) Stop() {
close(c.stop)
}
func (c *globalClient) Cache() map[protocol.DeviceID]CacheEntry {
// The globalClient doesn't do caching
return nil
}
// parseOptions parses and strips away any ?query=val options, setting the
// corresponding field in the serverOptions struct. Unknown query options are
// ignored and removed.
func parseOptions(dsn string) (server string, opts serverOptions, err error) {
p, err := url.Parse(dsn)
if err != nil {
return "", serverOptions{}, err
}
// Grab known options from the query string
q := p.Query()
opts.id = q.Get("id")
opts.insecure = opts.id != "" || queryBool(q, "insecure")
opts.noAnnounce = queryBool(q, "noannounce")
// Check for disallowed combinations
if p.Scheme == "http" {
if !opts.insecure {
return "", serverOptions{}, errors.New("http without insecure not supported")
}
if !opts.noAnnounce {
return "", serverOptions{}, errors.New("http without noannounce not supported")
}
} else if p.Scheme != "https" {
return "", serverOptions{}, errors.New("unsupported scheme " + p.Scheme)
}
// Remove the query string
p.RawQuery = ""
server = p.String()
return
}
// queryBool returns the query parameter parsed as a boolean. An empty value
// ("?foo") is considered true, as is any value string except false
// ("?foo=false").
func queryBool(q url.Values, key string) bool {
if _, ok := q[key]; !ok {
return false
}
return q.Get(key) != "false"
}
type idCheckingHTTPClient struct {
httpClient
id protocol.DeviceID
}
func newIDCheckingHTTPClient(client httpClient, id protocol.DeviceID) *idCheckingHTTPClient {
return &idCheckingHTTPClient{
httpClient: client,
id: id,
}
}
func (c *idCheckingHTTPClient) check(resp *http.Response) error {
if resp.TLS == nil {
return errors.New("security: not TLS")
}
if len(resp.TLS.PeerCertificates) == 0 {
return errors.New("security: no certificates")
}
id := protocol.NewDeviceID(resp.TLS.PeerCertificates[0].Raw)
if !id.Equals(c.id) {
return errors.New("security: incorrect device id")
}
return nil
}
func (c *idCheckingHTTPClient) Get(url string) (*http.Response, error) {
resp, err := c.httpClient.Get(url)
if err != nil {
return nil, err
}
if err := c.check(resp); err != nil {
return nil, err
}
return resp, nil
}
func (c *idCheckingHTTPClient) Post(url, ctype string, data io.Reader) (*http.Response, error) {
resp, err := c.httpClient.Post(url, ctype, data)
if err != nil {
return nil, err
}
if err := c.check(resp); err != nil {
return nil, err
}
return resp, nil
}
type errorHolder struct {
err error
mut stdsync.Mutex // uses stdlib sync as I want this to be trivially embeddable, and there is no risk of blocking
}
func (e *errorHolder) setError(err error) {
e.mut.Lock()
e.err = err
e.mut.Unlock()
}
func (e *errorHolder) Error() error {
e.mut.Lock()
err := e.err
e.mut.Unlock()
return err
}

253
lib/discover/global_test.go Normal file
View File

@ -0,0 +1,253 @@
package discover
import (
"crypto/tls"
"io/ioutil"
"net"
"net/http"
"strings"
"testing"
"time"
"github.com/syncthing/protocol"
"github.com/syncthing/syncthing/lib/tlsutil"
)
func TestParseOptions(t *testing.T) {
testcases := []struct {
in string
out string
opts serverOptions
}{
{"https://example.com/", "https://example.com/", serverOptions{}},
{"https://example.com/?insecure", "https://example.com/", serverOptions{insecure: true}},
{"https://example.com/?insecure=true", "https://example.com/", serverOptions{insecure: true}},
{"https://example.com/?insecure=yes", "https://example.com/", serverOptions{insecure: true}},
{"https://example.com/?insecure=false&noannounce", "https://example.com/", serverOptions{noAnnounce: true}},
{"https://example.com/?id=abc", "https://example.com/", serverOptions{id: "abc", insecure: true}},
}
for _, tc := range testcases {
res, opts, err := parseOptions(tc.in)
if err != nil {
t.Errorf("Unexpected err %v for %v", err, tc.in)
continue
}
if res != tc.out {
t.Errorf("Incorrect server, %v!= %v for %v", res, tc.out, tc.in)
}
if opts != tc.opts {
t.Errorf("Incorrect options, %v!= %v for %v", opts, tc.opts, tc.in)
}
}
}
func TestGlobalOverHTTP(t *testing.T) {
// HTTP works for queries, but is obviously insecure and we can't do
// announces over it (as we don't present a certificate). As such, http://
// is only allowed in combination with the "insecure" and "noannounce"
// parameters.
if _, err := NewGlobal("http://192.0.2.42/", tls.Certificate{}, nil, nil); err == nil {
t.Fatal("http is not allowed without insecure and noannounce")
}
if _, err := NewGlobal("http://192.0.2.42/?insecure", tls.Certificate{}, nil, nil); err == nil {
t.Fatal("http is not allowed without noannounce")
}
if _, err := NewGlobal("http://192.0.2.42/?noannounce", tls.Certificate{}, nil, nil); err == nil {
t.Fatal("http is not allowed without insecure")
}
// Now lets check that lookups work over HTTP, given the correct options.
list, err := net.Listen("tcp4", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer list.Close()
s := new(fakeDiscoveryServer)
mux := http.NewServeMux()
mux.HandleFunc("/", s.handler)
go http.Serve(list, mux)
direct, relays, err := testLookup("http://" + list.Addr().String() + "?insecure&noannounce")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(direct) != 1 || direct[0] != "tcp://192.0.2.42::22000" {
t.Errorf("incorrect direct list: %+v", direct)
}
if len(relays) != 1 || relays[0] != (Relay{URL: "relay://192.0.2.43:443", Latency: 42}) {
t.Errorf("incorrect relays list: %+v", direct)
}
}
func TestGlobalOverHTTPS(t *testing.T) {
dir, err := ioutil.TempDir("", "syncthing")
if err != nil {
t.Fatal(err)
}
// Generate a server certificate, using fewer bits than usual to hurry the
// process along a bit.
cert, err := tlsutil.NewCertificate(dir+"/cert.pem", dir+"/key.pem", "syncthing", 1024)
if err != nil {
t.Fatal(err)
}
list, err := tls.Listen("tcp4", "127.0.0.1:0", &tls.Config{Certificates: []tls.Certificate{cert}})
if err != nil {
t.Fatal(err)
}
defer list.Close()
s := new(fakeDiscoveryServer)
mux := http.NewServeMux()
mux.HandleFunc("/", s.handler)
go http.Serve(list, mux)
// With default options the lookup code expects the server certificate to
// check out according to the usual CA chains etc. That won't be the case
// here so we expect the lookup to fail.
url := "https://" + list.Addr().String()
if _, _, err := testLookup(url); err == nil {
t.Fatalf("unexpected nil error when we should have got a certificate error")
}
// With "insecure" set, whatever certificate is on the other side should
// be accepted.
url = "https://" + list.Addr().String() + "?insecure"
if direct, relays, err := testLookup(url); err != nil {
t.Fatalf("unexpected error: %v", err)
} else {
if len(direct) != 1 || direct[0] != "tcp://192.0.2.42::22000" {
t.Errorf("incorrect direct list: %+v", direct)
}
if len(relays) != 1 || relays[0] != (Relay{URL: "relay://192.0.2.43:443", Latency: 42}) {
t.Errorf("incorrect relays list: %+v", direct)
}
}
// With "id" set to something incorrect, the checks should fail again.
url = "https://" + list.Addr().String() + "?id=" + protocol.LocalDeviceID.String()
if _, _, err := testLookup(url); err == nil {
t.Fatalf("unexpected nil error for incorrect discovery server ID")
}
// With the correct device ID, the check should pass and we should get a
// lookup response.
id := protocol.NewDeviceID(cert.Certificate[0])
url = "https://" + list.Addr().String() + "?id=" + id.String()
if direct, relays, err := testLookup(url); err != nil {
t.Fatalf("unexpected error: %v", err)
} else {
if len(direct) != 1 || direct[0] != "tcp://192.0.2.42::22000" {
t.Errorf("incorrect direct list: %+v", direct)
}
if len(relays) != 1 || relays[0] != (Relay{URL: "relay://192.0.2.43:443", Latency: 42}) {
t.Errorf("incorrect relays list: %+v", direct)
}
}
}
func TestGlobalAnnounce(t *testing.T) {
dir, err := ioutil.TempDir("", "syncthing")
if err != nil {
t.Fatal(err)
}
// Generate a server certificate, using fewer bits than usual to hurry the
// process along a bit.
cert, err := tlsutil.NewCertificate(dir+"/cert.pem", dir+"/key.pem", "syncthing", 1024)
if err != nil {
t.Fatal(err)
}
list, err := tls.Listen("tcp4", "127.0.0.1:0", &tls.Config{Certificates: []tls.Certificate{cert}})
if err != nil {
t.Fatal(err)
}
defer list.Close()
s := new(fakeDiscoveryServer)
mux := http.NewServeMux()
mux.HandleFunc("/", s.handler)
go http.Serve(list, mux)
url := "https://" + list.Addr().String() + "?insecure"
disco, err := NewGlobal(url, cert, new(fakeAddressLister), new(fakeRelayStatus))
if err != nil {
t.Fatal(err)
}
go disco.Serve()
defer disco.Stop()
// The discovery thing should attempt an announcement immediately. We wait
// for it to succeed, a while.
t0 := time.Now()
for err := disco.Error(); err != nil; err = disco.Error() {
if time.Since(t0) > 10*time.Second {
t.Fatal("announce failed:", err)
}
time.Sleep(100 * time.Millisecond)
}
if !strings.Contains(string(s.announce), "tcp://0.0.0.0:22000") {
t.Errorf("announce missing direct address: %s", s.announce)
}
if !strings.Contains(string(s.announce), "relay://192.0.2.42:443") {
t.Errorf("announce missing relay address: %s", s.announce)
}
}
func testLookup(url string) ([]string, []Relay, error) {
disco, err := NewGlobal(url, tls.Certificate{}, nil, nil)
if err != nil {
return nil, nil, err
}
go disco.Serve()
defer disco.Stop()
return disco.Lookup(protocol.LocalDeviceID)
}
type fakeDiscoveryServer struct {
announce []byte
}
func (s *fakeDiscoveryServer) handler(w http.ResponseWriter, r *http.Request) {
if r.Method == "POST" {
s.announce, _ = ioutil.ReadAll(r.Body)
w.WriteHeader(204)
} else {
w.Header().Set("Content-Type", "application/json")
w.Write([]byte(`{"direct":["tcp://192.0.2.42::22000"], "relays":[{"url": "relay://192.0.2.43:443", "latency": 42}]}`))
}
}
type fakeAddressLister struct{}
func (f *fakeAddressLister) ExternalAddresses() []string {
return []string{"tcp://0.0.0.0:22000"}
}
func (f *fakeAddressLister) AllAddresses() []string {
return []string{"tcp://0.0.0.0:22000", "tcp://192.168.0.1:22000"}
}
type fakeRelayStatus struct{}
func (f *fakeRelayStatus) Relays() []string {
return []string{"relay://192.0.2.42:443"}
}
func (f *fakeRelayStatus) RelayStatus(uri string) (time.Duration, bool) {
return 42 * time.Millisecond, true
}

270
lib/discover/local.go Normal file
View File

@ -0,0 +1,270 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package discover
import (
"bytes"
"encoding/hex"
"errors"
"fmt"
"io"
"net"
"net/url"
"strconv"
"time"
"github.com/syncthing/protocol"
"github.com/syncthing/syncthing/lib/beacon"
"github.com/syncthing/syncthing/lib/events"
"github.com/thejerf/suture"
)
type localClient struct {
*suture.Supervisor
myID protocol.DeviceID
addrList AddressLister
relayStat RelayStatusProvider
name string
beacon beacon.Interface
localBcastStart time.Time
localBcastTick <-chan time.Time
forcedBcastTick chan time.Time
*cache
}
const (
BroadcastInterval = 30 * time.Second
CacheLifeTime = 3 * BroadcastInterval
)
var (
ErrIncorrectMagic = errors.New("incorrect magic number")
)
func NewLocal(id protocol.DeviceID, addr string, addrList AddressLister, relayStat RelayStatusProvider) (FinderService, error) {
c := &localClient{
Supervisor: suture.NewSimple("local"),
myID: id,
addrList: addrList,
relayStat: relayStat,
localBcastTick: time.Tick(BroadcastInterval),
forcedBcastTick: make(chan time.Time),
localBcastStart: time.Now(),
cache: newCache(),
}
host, port, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
if len(host) == 0 {
// A broadcast client
c.name = "IPv4 local"
bcPort, err := strconv.Atoi(port)
if err != nil {
return nil, err
}
c.startLocalIPv4Broadcasts(bcPort)
} else {
// A multicast client
c.name = "IPv6 local"
c.startLocalIPv6Multicasts(addr)
}
go c.sendLocalAnnouncements()
return c, nil
}
func (c *localClient) startLocalIPv4Broadcasts(localPort int) {
c.beacon = beacon.NewBroadcast(localPort)
c.Add(c.beacon)
go c.recvAnnouncements(c.beacon)
}
func (c *localClient) startLocalIPv6Multicasts(localMCAddr string) {
c.beacon = beacon.NewMulticast(localMCAddr)
c.Add(c.beacon)
go c.recvAnnouncements(c.beacon)
}
// Lookup returns a list of addresses the device is available at. Local
// discovery never returns relays.
func (c *localClient) Lookup(device protocol.DeviceID) (direct []string, relays []Relay, err error) {
if cache, ok := c.Get(device); ok {
if time.Since(cache.when) < CacheLifeTime {
direct = cache.Direct
relays = cache.Relays
}
}
return
}
func (c *localClient) String() string {
return c.name
}
func (c *localClient) Error() error {
return c.beacon.Error()
}
func (c *localClient) announcementPkt() Announce {
addrs := c.addrList.AllAddresses()
var relays []Relay
for _, relay := range c.relayStat.Relays() {
latency, ok := c.relayStat.RelayStatus(relay)
if ok {
relays = append(relays, Relay{
URL: relay,
Latency: int32(latency / time.Millisecond),
})
}
}
return Announce{
Magic: AnnouncementMagic,
This: Device{
ID: c.myID[:],
Addresses: addrs,
Relays: relays,
},
}
}
func (c *localClient) sendLocalAnnouncements() {
var pkt = c.announcementPkt()
msg := pkt.MustMarshalXDR()
for {
c.beacon.Send(msg)
select {
case <-c.localBcastTick:
case <-c.forcedBcastTick:
}
}
}
func (c *localClient) recvAnnouncements(b beacon.Interface) {
for {
buf, addr := b.Recv()
var pkt Announce
err := pkt.UnmarshalXDR(buf)
if err != nil && err != io.EOF {
if debug {
l.Debugf("discover: Failed to unmarshal local announcement from %s:\n%s", addr, hex.Dump(buf))
}
continue
}
if debug {
l.Debugf("discover: Received local announcement from %s for %s", addr, protocol.DeviceIDFromBytes(pkt.This.ID))
}
var newDevice bool
if bytes.Compare(pkt.This.ID, c.myID[:]) != 0 {
newDevice = c.registerDevice(addr, pkt.This)
}
if newDevice {
select {
case c.forcedBcastTick <- time.Now():
}
}
}
}
func (c *localClient) registerDevice(src net.Addr, device Device) bool {
var id protocol.DeviceID
copy(id[:], device.ID)
// Remember whether we already had a valid cache entry for this device.
ce, existsAlready := c.Get(id)
isNewDevice := !existsAlready || time.Since(ce.when) > CacheLifeTime
// Any empty or unspecified addresses should be set to the source address
// of the announcement. We also skip any addresses we can't parse.
var validAddresses []string
for _, addr := range device.Addresses {
u, err := url.Parse(addr)
if err != nil {
continue
}
tcpAddr, err := net.ResolveTCPAddr("tcp", u.Host)
if err != nil {
continue
}
if len(tcpAddr.IP) == 0 || tcpAddr.IP.IsUnspecified() {
host, _, err := net.SplitHostPort(src.String())
if err != nil {
continue
}
u.Host = fmt.Sprintf("%s:%d", host, tcpAddr.Port)
validAddresses = append(validAddresses, u.String())
} else {
validAddresses = append(validAddresses, addr)
}
}
c.Set(id, CacheEntry{
Direct: validAddresses,
Relays: device.Relays,
when: time.Now(),
found: true,
})
if isNewDevice {
events.Default.Log(events.DeviceDiscovered, map[string]interface{}{
"device": id.String(),
"addrs": device.Addresses,
"relays": device.Relays,
})
}
return isNewDevice
}
func addrToAddr(addr *net.TCPAddr) string {
if len(addr.IP) == 0 || addr.IP.IsUnspecified() {
return fmt.Sprintf(":%c", addr.Port)
} else if bs := addr.IP.To4(); bs != nil {
return fmt.Sprintf("%s:%c", bs.String(), addr.Port)
} else if bs := addr.IP.To16(); bs != nil {
return fmt.Sprintf("[%s]:%c", bs.String(), addr.Port)
}
return ""
}
func resolveAddrs(addrs []string) []string {
var raddrs []string
for _, addrStr := range addrs {
uri, err := url.Parse(addrStr)
if err != nil {
continue
}
addrRes, err := net.ResolveTCPAddr("tcp", uri.Host)
if err != nil {
continue
}
addr := addrToAddr(addrRes)
if len(addr) > 0 {
uri.Host = addr
raddrs = append(raddrs, uri.String())
}
}
return raddrs
}

6
lib/discover/packets.go → lib/discover/localpackets.go
View File

@ -5,7 +5,7 @@
// You can obtain one at http://mozilla.org/MPL/2.0/.
//go:generate -command genxdr go run ../../Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go
//go:generate genxdr -o packets_xdr.go packets.go
//go:generate genxdr -o localpackets_xdr.go localpackets.go
package discover
@ -26,8 +26,8 @@ type Announce struct {
}
type Relay struct {
Address string // max:256
Latency int32
URL string `json:"url"` // max:2083
Latency int32 `json:"latency"`
}
type Device struct {

14
lib/discover/packets_xdr.go → lib/discover/localpackets_xdr.go
View File

@ -192,10 +192,10 @@ Relay Structure:
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Length of Address |
| Length of URL |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ Address (variable length) \
\ URL (variable length) \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Latency |
@ -203,7 +203,7 @@ Relay Structure:
struct Relay {
string Address<256>;
string URL<256>;
int Latency;
}
@ -234,10 +234,10 @@ func (o Relay) AppendXDR(bs []byte) ([]byte, error) {
}
func (o Relay) EncodeXDRInto(xw *xdr.Writer) (int, error) {
if l := len(o.Address); l > 256 {
return xw.Tot(), xdr.ElementSizeExceeded("Address", l, 256)
if l := len(o.URL); l > 256 {
return xw.Tot(), xdr.ElementSizeExceeded("URL", l, 256)
}
xw.WriteString(o.Address)
xw.WriteString(o.URL)
xw.WriteUint32(uint32(o.Latency))
return xw.Tot(), xw.Error()
}
@ -254,7 +254,7 @@ func (o *Relay) UnmarshalXDR(bs []byte) error {
}
func (o *Relay) DecodeXDRFrom(xr *xdr.Reader) error {
o.Address = xr.ReadStringMax(256)
o.URL = xr.ReadStringMax(256)
o.Latency = int32(xr.ReadUint32())
return xr.Error()
}

3
lib/events/events.go
View File

@ -40,6 +40,7 @@ const (
FolderErrors
FolderScanProgress
ExternalPortMappingChanged
RelayStateChanged
AllEvents = (1 << iota) - 1
)
@ -90,6 +91,8 @@ func (t EventType) String() string {
return "FolderScanProgress"
case ExternalPortMappingChanged:
return "ExternalPortMappingChanged"
case RelayStateChanged:
return "RelayStateChanged"
default:
return "Unknown"
}

2
lib/ignore/ignore.go
View File

@ -12,6 +12,7 @@ import (
"crypto/md5"
"fmt"
"io"
"log"
"os"
"path/filepath"
"regexp"
@ -236,6 +237,7 @@ func parseIgnoreFile(fd io.Reader, currentFile string, seen map[string]bool) ([]
includeFile := filepath.Join(filepath.Dir(currentFile), line[len("#include "):])
includes, err := loadIgnoreFile(includeFile, seen)
if err != nil {
log.Println(err)
return err
}
patterns = append(patterns, includes...)

148
lib/relay/relay.go
View File

@ -12,18 +12,23 @@ import (
"net"
"net/http"
"net/url"
"sort"
"time"
"github.com/syncthing/relaysrv/client"
"github.com/syncthing/relaysrv/protocol"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/discover"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/sync"
"github.com/thejerf/suture"
)
const (
eventBroadcasterCheckInterval = 10 * time.Second
)
type Svc struct {
*suture.Supervisor
cfg *config.Wrapper
@ -71,7 +76,12 @@ func NewSvc(cfg *config.Wrapper, tlsCfg *tls.Config) *Svc {
stop: make(chan struct{}),
}
eventBc := &eventBroadcaster{
svc: svc,
}
svc.Add(receiver)
svc.Add(eventBc)
return svc
}
@ -132,7 +142,7 @@ func (s *Svc) CommitConfiguration(from, to config.Configuration) bool {
continue
}
dynRelays := make([]discover.Relay, 0, len(ann.Relays))
var dynRelayAddrs []string
for _, relayAnn := range ann.Relays {
ruri, err := url.Parse(relayAnn.URL)
if err != nil {
@ -144,13 +154,11 @@ func (s *Svc) CommitConfiguration(from, to config.Configuration) bool {
if debug {
l.Debugln("Found", ruri, "via", uri)
}
dynRelays = append(dynRelays, discover.Relay{
Address: ruri.String(),
})
dynRelayAddrs = append(dynRelayAddrs, ruri.String())
}
dynRelayAddrs := discover.RelayAddressesSortedByLatency(dynRelays)
if len(dynRelayAddrs) > 0 {
dynRelayAddrs = relayAddressesSortedByLatency(dynRelayAddrs)
closestRelay := dynRelayAddrs[0]
if debug {
l.Debugln("Picking", closestRelay, "as closest dynamic relay from", uri)
@ -193,7 +201,14 @@ func (s *Svc) CommitConfiguration(from, to config.Configuration) bool {
return true
}
func (s *Svc) ClientStatus() map[string]bool {
type Status struct {
URL string
OK bool
Latency int
}
// Relays return the list of relays that currently have an OK status.
func (s *Svc) Relays() []string {
if s == nil {
// A nil client does not have a status, really. Yet we may be called
// this way, for raisins...
@ -201,12 +216,34 @@ func (s *Svc) ClientStatus() map[string]bool {
}
s.mut.RLock()
status := make(map[string]bool, len(s.clients))
for uri, client := range s.clients {
status[uri] = client.StatusOK()
relays := make([]string, 0, len(s.clients))
for uri := range s.clients {
relays = append(relays, uri)
}
s.mut.RUnlock()
return status
sort.Strings(relays)
return relays
}
// RelayStatus returns the latency and OK status for a given relay.
func (s *Svc) RelayStatus(uri string) (time.Duration, bool) {
if s == nil {
// A nil client does not have a status, really. Yet we may be called
// this way, for raisins...
return time.Hour, false
}
s.mut.RLock()
client, ok := s.clients[uri]
s.mut.RUnlock()
if !ok || !client.StatusOK() {
return time.Hour, false
}
return client.Latency(), true
}
// Accept returns a new *tls.Conn. The connection is already handshaken.
@ -266,6 +303,55 @@ func (r *invitationReceiver) Stop() {
close(r.stop)
}
// The eventBroadcaster sends a RelayStateChanged event when the relay status
// changes. We need this somewhat ugly polling mechanism as there's currently
// no way to get the event feed directly from the relay lib. This may be
// somethign to revisit later, possibly.
type eventBroadcaster struct {
svc *Svc
stop chan struct{}
}
func (e *eventBroadcaster) Serve() {
timer := time.NewTicker(eventBroadcasterCheckInterval)
defer timer.Stop()
var prevOKRelays []string
for {
select {
case <-timer.C:
curOKRelays := e.svc.Relays()
changed := len(curOKRelays) != len(prevOKRelays)
if !changed {
for i := range curOKRelays {
if curOKRelays[i] != prevOKRelays[i] {
changed = true
break
}
}
}
if changed {
events.Default.Log(events.RelayStateChanged, map[string][]string{
"old": prevOKRelays,
"new": curOKRelays,
})
}
prevOKRelays = curOKRelays
case <-e.stop:
return
}
}
}
func (e *eventBroadcaster) Stop() {
close(e.stop)
}
// This is the announcement recieved from the relay server;
// {"relays": [{"url": "relay://10.20.30.40:5060"}, ...]}
type dynamicAnnouncement struct {
@ -273,3 +359,43 @@ type dynamicAnnouncement struct {
URL string
}
}
// relayAddressesSortedByLatency adds local latency to the relay, and sorts them
// by sum latency, and returns the addresses.
func relayAddressesSortedByLatency(input []string) []string {
relays := make(relayList, len(input))
for i, relay := range input {
if latency, err := osutil.GetLatencyForURL(relay); err == nil {
relays[i] = relayWithLatency{relay, int(latency / time.Millisecond)}
} else {
relays[i] = relayWithLatency{relay, int(time.Hour / time.Millisecond)}
}
}
sort.Sort(relays)
addresses := make([]string, len(relays))
for i, relay := range relays {
addresses[i] = relay.relay
}
return addresses
}
type relayWithLatency struct {
relay string
latency int
}
type relayList []relayWithLatency
func (l relayList) Len() int {
return len(l)
}
func (l relayList) Less(a, b int) bool {
return l[a].latency < l[b].latency
}
func (l relayList) Swap(a, b int) {
l[a], l[b] = l[b], l[a]
}

3
test/h1/config.xml
View File

@ -46,8 +46,7 @@
</gui>
<options>
<listenAddress>tcp://127.0.0.1:22001</listenAddress>
<globalAnnounceServer>udp4://announce.syncthing.net:22027</globalAnnounceServer>
<globalAnnounceServer>udp6://announce-v6.syncthing.net:22027</globalAnnounceServer>
<globalAnnounceServer>default</globalAnnounceServer>
<globalAnnounceEnabled>false</globalAnnounceEnabled>
<localAnnounceEnabled>true</localAnnounceEnabled>
<localAnnouncePort>21025</localAnnouncePort>

3
test/h2/config.xml
View File

@ -53,8 +53,7 @@
</gui>
<options>
<listenAddress>tcp://127.0.0.1:22002</listenAddress>
<globalAnnounceServer>udp4://announce.syncthing.net:22027</globalAnnounceServer>
<globalAnnounceServer>udp6://announce-v6.syncthing.net:22027</globalAnnounceServer>
<globalAnnounceServer>default</globalAnnounceServer>
<globalAnnounceEnabled>true</globalAnnounceEnabled>
<localAnnounceEnabled>true</localAnnounceEnabled>
<localAnnouncePort>21025</localAnnouncePort>

3
test/h3/config.xml
View File

@ -39,8 +39,7 @@
</gui>
<options>
<listenAddress>tcp://127.0.0.1:22003</listenAddress>
<globalAnnounceServer>udp4://announce.syncthing.net:22027</globalAnnounceServer>
<globalAnnounceServer>udp6://announce-v6.syncthing.net:22027</globalAnnounceServer>
<globalAnnounceServer>default</globalAnnounceServer>
<globalAnnounceEnabled>false</globalAnnounceEnabled>
<localAnnounceEnabled>false</localAnnounceEnabled>
<localAnnouncePort>21025</localAnnouncePort>

3
test/h4/config.xml
View File

@ -18,8 +18,7 @@
</gui>
<options>
<listenAddress>tcp://127.0.0.1:22004</listenAddress>
<globalAnnounceServer>udp4://announce.syncthing.net:22027</globalAnnounceServer>
<globalAnnounceServer>udp6://announce-v6.syncthing.net:22027</globalAnnounceServer>
<globalAnnounceServer>default</globalAnnounceServer>
<globalAnnounceEnabled>false</globalAnnounceEnabled>
<localAnnounceEnabled>false</localAnnounceEnabled>
<localAnnouncePort>21025</localAnnouncePort>