Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Send external announcements

This commit is contained in:
Jakob Borg 2013-12-22 16:29:23 -05:00
parent 8e65d36691
commit e48222ada0
3 changed files with 350 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

244
discover/discover.go
View File

@ -4,31 +4,75 @@ served by something more standardized, such as mDNS / DNS-SD. In practice, this
was much easier and quicker to get up and running.
The mode of operation is to periodically (currently once every 30 seconds)
transmit a broadcast UDP packet to the well known port number 21025. The packet
has the following format:
broadcast an Announcement packet to UDP port 21025. The packet has the
following format:
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Magic Number |
| Magic Number (0x20121025) |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Port Number | Length of NodeID |
| Port Number | Reserved |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Length of NodeID |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ NodeID (variable length) \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
This is the XDR encoding of:
struct Announcement {
unsigned int Magic;
unsigned short Port;
string NodeID<>;
}
(Hence NodeID is padded to a multiple of 32 bits)
The sending node's address is not encoded -- it is taken to be the source
address of the announcement. Every time such a packet is received, a local
table that maps NodeID to Address is updated. When the local node wants to
connect to another node with the address specification 'dynamic', this table is
consulted.
For external discovery, an identical packet is sent every 30 minutes to the
external discovery server. The server keeps information for up to 60 minutes.
To query the server, and UDP packet with the format below is sent.
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Magic Number (0x19760309) |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Length of NodeID |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ NodeID (variable length) \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
This is the XDR encoding of:
struct Announcement {
unsigned int Magic;
string NodeID<>;
}
(Hence NodeID is padded to a multiple of 32 bits)
It is answered with an announcement packet for the queried node ID if the
information is available. There is no answer for queries about unknown nodes. A
reasonable timeout is recommended instead. (This, combined with server side
rate limits for packets per source IP and queries per node ID, prevents the
server from being used as an amplifier in a DDoS attack.)
*/
package discover
import (
"encoding/binary"
"errors"
"fmt"
"log"
"net"
@ -36,14 +80,34 @@ import (
"time"
)
const (
AnnouncementPort = 21025
AnnouncementMagic = 0x20121025
QueryMagic = 0x19760309
)
var (
errBadMagic = errors.New("bad magic")
errFormat = errors.New("incorrect packet format")
)
type Discoverer struct {
MyID string
ListenPort int
BroadcastIntv time.Duration
MyID string
ListenPort int
BroadcastIntv time.Duration
ExtListenPort int
ExtBroadcastIntv time.Duration
conn *net.UDPConn
registry map[string]string
registryLock sync.RWMutex
extServer string
}
type packet struct {
magic uint32 // AnnouncementMagic or QueryMagic
port uint16 // unset if magic == QueryMagic
id string
}
// We tolerate a certain amount of errors because we might be running in
@ -51,50 +115,71 @@ type Discoverer struct {
// When we hit this many errors in succession, we stop.
const maxErrors = 30
func NewDiscoverer(id string, port int) (*Discoverer, error) {
local4 := &net.UDPAddr{IP: net.IP{0, 0, 0, 0}, Port: 21025}
func NewDiscoverer(id string, port int, extPort int, extServer string) (*Discoverer, error) {
local4 := &net.UDPAddr{IP: net.IP{0, 0, 0, 0}, Port: AnnouncementPort}
conn, err := net.ListenUDP("udp4", local4)
if err != nil {
return nil, err
}
disc := &Discoverer{
MyID: id,
ListenPort: port,
BroadcastIntv: 30 * time.Second,
conn: conn,
registry: make(map[string]string),
MyID: id,
ListenPort: port,
BroadcastIntv: 30 * time.Second,
ExtListenPort: extPort,
ExtBroadcastIntv: 1800 * time.Second,
conn: conn,
registry: make(map[string]string),
extServer: extServer,
}
go disc.sendAnnouncements()
go disc.recvAnnouncements()
if disc.ListenPort > 0 {
disc.sendAnnouncements()
}
if len(disc.extServer) > 0 && disc.ExtListenPort > 0 {
disc.sendExtAnnouncements()
}
return disc, nil
}
func (d *Discoverer) sendAnnouncements() {
remote4 := &net.UDPAddr{IP: net.IP{255, 255, 255, 255}, Port: 21025}
remote4 := &net.UDPAddr{IP: net.IP{255, 255, 255, 255}, Port: AnnouncementPort}
idbs := []byte(d.MyID)
buf := make([]byte, 4+4+4+len(idbs))
buf := encodePacket(packet{AnnouncementMagic, uint16(d.ListenPort), d.MyID})
go d.writeAnnouncements(buf, remote4, d.BroadcastIntv)
}
binary.BigEndian.PutUint32(buf, uint32(0x121025))
binary.BigEndian.PutUint16(buf[4:], uint16(d.ListenPort))
binary.BigEndian.PutUint16(buf[6:], uint16(len(idbs)))
copy(buf[8:], idbs)
func (d *Discoverer) sendExtAnnouncements() {
extIPs, err := net.LookupIP(d.extServer)
if err != nil {
log.Printf("discover/external: %v; no external announcements", err)
return
}
buf := encodePacket(packet{AnnouncementMagic, uint16(d.ExtListenPort), d.MyID})
for _, extIP := range extIPs {
remote4 := &net.UDPAddr{IP: extIP, Port: AnnouncementPort}
go d.writeAnnouncements(buf, remote4, d.ExtBroadcastIntv)
}
}
func (d *Discoverer) writeAnnouncements(buf []byte, remote *net.UDPAddr, intv time.Duration) {
var errCounter = 0
var err error
for errCounter < maxErrors {
_, _, err = d.conn.WriteMsgUDP(buf, nil, remote4)
_, _, err = d.conn.WriteMsgUDP(buf, nil, remote)
if err != nil {
errCounter++
} else {
errCounter = 0
}
time.Sleep(d.BroadcastIntv)
time.Sleep(intv)
}
log.Println("discover/write: stopping due to too many errors:", err)
log.Println("discover/write: %v: stopping due to too many errors:", remote, err)
}
func (d *Discoverer) recvAnnouncements() {
@ -102,26 +187,27 @@ func (d *Discoverer) recvAnnouncements() {
var errCounter = 0
var err error
for errCounter < maxErrors {
_, addr, err := d.conn.ReadFromUDP(buf)
n, addr, err := d.conn.ReadFromUDP(buf)
if err != nil {
errCounter++
time.Sleep(time.Second)
continue
}
errCounter = 0
magic := binary.BigEndian.Uint32(buf)
if magic != 0x121025 {
pkt, err := decodePacket(buf[:n])
if err != nil || pkt.magic != AnnouncementMagic {
errCounter++
time.Sleep(time.Second)
continue
}
port := binary.BigEndian.Uint16(buf[4:])
l := binary.BigEndian.Uint16(buf[6:])
idbs := buf[8 : l+8]
id := string(idbs)
if id != d.MyID {
nodeAddr := fmt.Sprintf("%s:%d", addr.IP.String(), port)
errCounter = 0
if pkt.id != d.MyID {
nodeAddr := fmt.Sprintf("%s:%d", addr.IP.String(), pkt.port)
d.registryLock.Lock()
if d.registry[id] != nodeAddr {
d.registry[id] = nodeAddr
if d.registry[pkt.id] != nodeAddr {
d.registry[pkt.id] = nodeAddr
}
d.registryLock.Unlock()
}
@ -135,3 +221,87 @@ func (d *Discoverer) Lookup(node string) (string, bool) {
addr, ok := d.registry[node]
return addr, ok
}
func encodePacket(pkt packet) []byte {
var idbs = []byte(pkt.id)
var l = len(idbs) + pad(len(idbs)) + 4 + 4
if pkt.magic == AnnouncementMagic {
l += 4
}
var buf = make([]byte, l)
var offset = 0
binary.BigEndian.PutUint32(buf[offset:], pkt.magic)
offset += 4
if pkt.magic == AnnouncementMagic {
binary.BigEndian.PutUint16(buf[offset:], uint16(pkt.port))
offset += 4
}
binary.BigEndian.PutUint32(buf[offset:], uint32(len(idbs)))
offset += 4
copy(buf[offset:], idbs)
return buf
}
func decodePacket(buf []byte) (*packet, error) {
var p packet
var offset int
if len(buf) < 4 {
// short packet
return nil, errFormat
}
p.magic = binary.BigEndian.Uint32(buf[offset:])
offset += 4
if p.magic != AnnouncementMagic && p.magic != QueryMagic {
return nil, errBadMagic
}
if p.magic == AnnouncementMagic {
if len(buf) < offset+4 {
// short packet
return nil, errFormat
}
p.port = binary.BigEndian.Uint16(buf[offset:])
offset += 2
reserved := binary.BigEndian.Uint16(buf[offset:])
if reserved != 0 {
return nil, errFormat
}
offset += 2
}
if len(buf) < offset+4 {
// short packet
return nil, errFormat
}
l := binary.BigEndian.Uint32(buf[offset:])
offset += 4
if len(buf) < offset+int(l)+pad(int(l)) {
// short packet
return nil, errFormat
}
idbs := buf[offset : offset+int(l)]
p.id = string(idbs)
offset += int(l) + pad(int(l))
if len(buf[offset:]) > 0 {
// extra data
return nil, errFormat
}
return &p, nil
}
func pad(l int) int {
d := l % 4
if d == 0 {
return 0
}
return 4 - d
}

111
discover/discover_test.go Normal file
View File

@ -0,0 +1,111 @@
package discover
import (
"bytes"
"reflect"
"testing"
)
var testdata = []struct {
data []byte
packet *packet
err error
}{
{
[]byte{0x20, 0x12, 0x10, 0x25,
0x12, 0x34, 0x00, 0x00,
0x00, 0x00, 0x00, 0x05,
0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x00, 0x00, 0x00},
&packet{
magic: 0x20121025,
port: 0x1234,
id: "hello",
},
nil,
},
{
[]byte{0x20, 0x12, 0x10, 0x25,
0x34, 0x56, 0x00, 0x00,
0x00, 0x00, 0x00, 0x08,
0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x21, 0x21, 0x21},
&packet{
magic: 0x20121025,
port: 0x3456,
id: "hello!!!",
},
nil,
},
{
[]byte{0x19, 0x76, 0x03, 0x09,
0x00, 0x00, 0x00, 0x06,
0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x21, 0x00, 0x00},
&packet{
magic: 0x19760309,
id: "hello!",
},
nil,
},
{
[]byte{0x20, 0x12, 0x10, 0x25,
0x12, 0x34, 0x12, 0x34, // reserved bits not set to zero
0x00, 0x00, 0x00, 0x06,
0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x21, 0x00, 0x00},
nil,
errFormat,
},
{
[]byte{0x20, 0x12, 0x10, 0x25,
0x12, 0x34, 0x00, 0x00,
0x00, 0x00, 0x00, 0x06,
0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x21}, // missing padding
nil,
errFormat,
},
{
[]byte{0x19, 0x77, 0x03, 0x09, // incorrect magic
0x00, 0x00, 0x00, 0x06,
0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x21, 0x00, 0x00},
nil,
errBadMagic,
},
{
[]byte{0x19, 0x76, 0x03, 0x09,
0x6c, 0x6c, 0x6c, 0x6c, // length exceeds packet size
0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x21, 0x00, 0x00},
nil,
errFormat,
},
{
[]byte{0x19, 0x76, 0x03, 0x09,
0x00, 0x00, 0x00, 0x06,
0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x21, 0x00, 0x00,
0x23}, // extra data at the end
nil,
errFormat,
},
}
func TestDecodePacket(t *testing.T) {
for i, test := range testdata {
p, err := decodePacket(test.data)
if err != test.err {
t.Errorf("%d: unexpected error %v", i, err)
} else {
if !reflect.DeepEqual(p, test.packet) {
t.Errorf("%d: incorrect packet\n%v\n%v", i, test.packet, p)
}
}
}
}
func TestEncodePacket(t *testing.T) {
for i, test := range testdata {
if test.err != nil {
continue
}
buf := encodePacket(*test.packet)
if bytes.Compare(buf, test.data) != 0 {
t.Errorf("%d: incorrect encoded packet\n% x\n% 0x", i, test.data, buf)
}
}
}

46
main.go
View File

@ -21,21 +21,29 @@ import (
)
type Options struct {
ConfDir string `short:"c" long:"cfg" description:"Configuration directory" default:"~/.syncthing" value-name:"DIR"`
Listen string `short:"l" long:"listen" description:"Listen address" default:":22000" value-name:"ADDR"`
ReadOnly bool `long:"ro" description:"Repository is read only"`
Delete bool `long:"delete" description:"Delete files from repo when deleted from cluster"`
NoSymlinks bool `long:"no-symlinks" description:"Don't follow first level symlinks in the repo"`
ScanInterval time.Duration `long:"scan-intv" description:"Repository scan interval" default:"60s" value-name:"INTV"`
ConnInterval time.Duration `long:"conn-intv" description:"Node reconnect interval" default:"60s" value-name:"INTV"`
Debug DebugOptions `group:"Debugging Options"`
ConfDir string `short:"c" long:"cfg" description:"Configuration directory" default:"~/.syncthing" value-name:"DIR"`
Listen string `short:"l" long:"listen" description:"Listen address" default:":22000" value-name:"ADDR"`
ReadOnly bool `short:"r" long:"ro" description:"Repository is read only"`
Delete bool `short:"d" long:"delete" description:"Delete files from repo when deleted from cluster"`
NoSymlinks bool `long:"no-symlinks" description:"Don't follow first level symlinks in the repo"`
ScanInterval time.Duration `long:"scan-intv" description:"Repository scan interval" default:"60s" value-name:"INTV"`
ConnInterval time.Duration `long:"conn-intv" description:"Node reconnect interval" default:"60s" value-name:"INTV"`
Discovery DiscoveryOptions `group:"Discovery Options"`
Debug DebugOptions `group:"Debugging Options"`
}
type DebugOptions struct {
TraceFile bool `long:"trace-file"`
TraceNet bool `long:"trace-net"`
TraceIdx bool `long:"trace-idx"`
Profiler string `long:"profiler"`
Profiler string `long:"profiler" value-name:"ADDR"`
}
type DiscoveryOptions struct {
ExternalServer string `long:"ext-server" description:"External discovery server" value-name:"NAME" default:"syncthing.nym.se"`
ExternalPort int `short:"e" long:"ext-port" description:"External listen port" value-name:"PORT" default:"22000"`
NoExternalDiscovery bool `short:"n" long:"no-ext-announce" description:"Do not announce presence externally"`
NoLocalDiscovery bool `short:"N" long:"no-local-announce" description:"Do not announce presence locally"`
}
var opts Options
@ -206,8 +214,6 @@ listen:
continue listen
}
}
warnln("Connect from unknown node", remoteID)
conn.Close()
}
}
@ -217,10 +223,22 @@ func connect(myID string, addr string, nodeAddrs map[string][]string, m *Model,
fatalErr(err)
port, _ := strconv.Atoi(portstr)
infoln("Starting local discovery")
disc, err := discover.NewDiscoverer(myID, port)
if opts.Discovery.NoLocalDiscovery {
port = -1
} else {
infoln("Sending local discovery announcements")
}
if opts.Discovery.NoExternalDiscovery {
opts.Discovery.ExternalPort = -1
} else {
infoln("Sending external discovery announcements")
}
disc, err := discover.NewDiscoverer(myID, port, opts.Discovery.ExternalPort, opts.Discovery.ExternalServer)
if err != nil {
warnln("No local discovery possible")
warnf("No discovery possible (%v)", err)
}
for {