From f28367bcfc94f2648c99ca05403993c5d56ec3cd Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Mon, 22 Sep 2014 21:42:11 +0200 Subject: [PATCH 01/61] Move top level packages to internal. --- .gitignore | 1 + common_test.go | 76 ++++ counting.go | 64 +++ debug.go | 17 + doc.go | 6 + header.go | 45 ++ message.go | 139 ++++++ message_xdr.go | 964 +++++++++++++++++++++++++++++++++++++++++ nativemodel_darwin.go | 42 ++ nativemodel_unix.go | 33 ++ nativemodel_windows.go | 72 +++ nodeid.go | 159 +++++++ nodeid_test.go | 78 ++++ protocol.go | 640 +++++++++++++++++++++++++++ protocol_test.go | 383 ++++++++++++++++ wireformat.go | 58 +++ 16 files changed, 2777 insertions(+) create mode 100644 .gitignore create mode 100644 common_test.go create mode 100644 counting.go create mode 100644 debug.go create mode 100644 doc.go create mode 100644 header.go create mode 100644 message.go create mode 100644 message_xdr.go create mode 100644 nativemodel_darwin.go create mode 100644 nativemodel_unix.go create mode 100644 nativemodel_windows.go create mode 100644 nodeid.go create mode 100644 nodeid_test.go create mode 100644 protocol.go create mode 100644 protocol_test.go create mode 100644 wireformat.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..2211df63d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.txt diff --git a/common_test.go b/common_test.go new file mode 100644 index 000000000..9d387825c --- /dev/null +++ b/common_test.go @@ -0,0 +1,76 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package protocol + +import ( + "io" + "time" +) + +type TestModel struct { + data []byte + repo string + name string + offset int64 + size int + closedCh chan bool +} + +func newTestModel() *TestModel { + return &TestModel{ + closedCh: make(chan bool), + } +} + +func (t *TestModel) Index(nodeID NodeID, repo string, files []FileInfo) { +} + +func (t *TestModel) IndexUpdate(nodeID NodeID, repo string, files []FileInfo) { +} + +func (t *TestModel) Request(nodeID NodeID, repo, name string, offset int64, size int) ([]byte, error) { + t.repo = repo + t.name = name + t.offset = offset + t.size = size + return t.data, nil +} + +func (t *TestModel) Close(nodeID NodeID, err error) { + close(t.closedCh) +} + +func (t *TestModel) ClusterConfig(nodeID NodeID, config ClusterConfigMessage) { +} + +func (t *TestModel) isClosed() bool { + select { + case <-t.closedCh: + return true + case <-time.After(1 * time.Second): + return false // Timeout + } +} + +type ErrPipe struct { + io.PipeWriter + written int + max int + err error + closed bool +} + +func (e *ErrPipe) Write(data []byte) (int, error) { + if e.closed { + return 0, e.err + } + if e.written+len(data) > e.max { + n, _ := e.PipeWriter.Write(data[:e.max-e.written]) + e.PipeWriter.CloseWithError(e.err) + e.closed = true + return n, e.err + } + return e.PipeWriter.Write(data) +} diff --git a/counting.go b/counting.go new file mode 100644 index 000000000..512774fba --- /dev/null +++ b/counting.go @@ -0,0 +1,64 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package protocol + +import ( + "io" + "sync/atomic" + "time" +) + +type countingReader struct { + io.Reader + tot uint64 // bytes + last int64 // unix nanos +} + +var ( + totalIncoming uint64 + totalOutgoing uint64 +) + +func (c *countingReader) Read(bs []byte) (int, error) { + n, err := c.Reader.Read(bs) + atomic.AddUint64(&c.tot, uint64(n)) + atomic.AddUint64(&totalIncoming, uint64(n)) + atomic.StoreInt64(&c.last, time.Now().UnixNano()) + return n, err +} + +func (c *countingReader) Tot() uint64 { + return atomic.LoadUint64(&c.tot) +} + +func (c *countingReader) Last() time.Time { + return time.Unix(0, atomic.LoadInt64(&c.last)) +} + +type countingWriter struct { + io.Writer + tot uint64 // bytes + last int64 // unix nanos +} + +func (c *countingWriter) Write(bs []byte) (int, error) { + n, err := c.Writer.Write(bs) + atomic.AddUint64(&c.tot, uint64(n)) + atomic.AddUint64(&totalOutgoing, uint64(n)) + atomic.StoreInt64(&c.last, time.Now().UnixNano()) + return n, err +} + +func (c *countingWriter) Tot() uint64 { + return atomic.LoadUint64(&c.tot) +} + +func (c *countingWriter) Last() time.Time { + return time.Unix(0, atomic.LoadInt64(&c.last)) +} + +func TotalInOut() (uint64, uint64) { + return atomic.LoadUint64(&totalIncoming), atomic.LoadUint64(&totalOutgoing) +} diff --git a/debug.go b/debug.go new file mode 100644 index 000000000..6c586b90e --- /dev/null +++ b/debug.go @@ -0,0 +1,17 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package protocol + +import ( + "os" + "strings" + + "github.com/syncthing/syncthing/internal/logger" +) + +var ( + debug = strings.Contains(os.Getenv("STTRACE"), "protocol") || os.Getenv("STTRACE") == "all" + l = logger.DefaultLogger +) diff --git a/doc.go b/doc.go new file mode 100644 index 000000000..8c6b524e6 --- /dev/null +++ b/doc.go @@ -0,0 +1,6 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +// Package protocol implements the Block Exchange Protocol. +package protocol diff --git a/header.go b/header.go new file mode 100644 index 000000000..6fd2ebb8f --- /dev/null +++ b/header.go @@ -0,0 +1,45 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package protocol + +import "github.com/calmh/xdr" + +type header struct { + version int + msgID int + msgType int + compression bool +} + +func (h header) encodeXDR(xw *xdr.Writer) (int, error) { + u := encodeHeader(h) + return xw.WriteUint32(u) +} + +func (h *header) decodeXDR(xr *xdr.Reader) error { + u := xr.ReadUint32() + *h = decodeHeader(u) + return xr.Error() +} + +func encodeHeader(h header) uint32 { + var isComp uint32 + if h.compression { + isComp = 1 << 0 // the zeroth bit is the compression bit + } + return uint32(h.version&0xf)<<28 + + uint32(h.msgID&0xfff)<<16 + + uint32(h.msgType&0xff)<<8 + + isComp +} + +func decodeHeader(u uint32) header { + return header{ + version: int(u>>28) & 0xf, + msgID: int(u>>16) & 0xfff, + msgType: int(u>>8) & 0xff, + compression: u&1 == 1, + } +} diff --git a/message.go b/message.go new file mode 100644 index 000000000..779817a7b --- /dev/null +++ b/message.go @@ -0,0 +1,139 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package protocol + +import "fmt" + +type IndexMessage struct { + Repository string // max:64 + Files []FileInfo +} + +type FileInfo struct { + Name string // max:8192 + Flags uint32 + Modified int64 + Version uint64 + LocalVersion uint64 + Blocks []BlockInfo +} + +func (f FileInfo) String() string { + return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%d, Size:%d, Blocks:%v}", + f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.Blocks) +} + +func (f FileInfo) Size() (bytes int64) { + if IsDeleted(f.Flags) || IsDirectory(f.Flags) { + return 128 + } + for _, b := range f.Blocks { + bytes += int64(b.Size) + } + return +} + +func (f FileInfo) IsDeleted() bool { + return IsDeleted(f.Flags) +} + +func (f FileInfo) IsInvalid() bool { + return IsInvalid(f.Flags) +} + +// Used for unmarshalling a FileInfo structure but skipping the actual block list +type FileInfoTruncated struct { + Name string // max:8192 + Flags uint32 + Modified int64 + Version uint64 + LocalVersion uint64 + NumBlocks uint32 +} + +// Returns a statistical guess on the size, not the exact figure +func (f FileInfoTruncated) Size() int64 { + if IsDeleted(f.Flags) || IsDirectory(f.Flags) { + return 128 + } + if f.NumBlocks < 2 { + return BlockSize / 2 + } else { + return int64(f.NumBlocks-1)*BlockSize + BlockSize/2 + } +} + +func (f FileInfoTruncated) IsDeleted() bool { + return IsDeleted(f.Flags) +} + +func (f FileInfoTruncated) IsInvalid() bool { + return IsInvalid(f.Flags) +} + +type FileIntf interface { + Size() int64 + IsDeleted() bool + IsInvalid() bool +} + +type BlockInfo struct { + Offset int64 // noencode (cache only) + Size uint32 + Hash []byte // max:64 +} + +func (b BlockInfo) String() string { + return fmt.Sprintf("Block{%d/%d/%x}", b.Offset, b.Size, b.Hash) +} + +type RequestMessage struct { + Repository string // max:64 + Name string // max:8192 + Offset uint64 + Size uint32 +} + +type ResponseMessage struct { + Data []byte +} + +type ClusterConfigMessage struct { + ClientName string // max:64 + ClientVersion string // max:64 + Repositories []Repository // max:64 + Options []Option // max:64 +} + +func (o *ClusterConfigMessage) GetOption(key string) string { + for _, option := range o.Options { + if option.Key == key { + return option.Value + } + } + return "" +} + +type Repository struct { + ID string // max:64 + Nodes []Node // max:64 +} + +type Node struct { + ID []byte // max:32 + Flags uint32 + MaxLocalVersion uint64 +} + +type Option struct { + Key string // max:64 + Value string // max:1024 +} + +type CloseMessage struct { + Reason string // max:1024 +} + +type EmptyMessage struct{} diff --git a/message_xdr.go b/message_xdr.go new file mode 100644 index 000000000..11a11da34 --- /dev/null +++ b/message_xdr.go @@ -0,0 +1,964 @@ +// ************************************************************ +// This file is automatically generated by genxdr. Do not edit. +// ************************************************************ + +package protocol + +import ( + "bytes" + "io" + + "github.com/calmh/xdr" +) + +/* + +IndexMessage Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Repository | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Repository (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Number of Files | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Zero or more FileInfo Structures \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct IndexMessage { + string Repository<64>; + FileInfo Files<>; +} + +*/ + +func (o IndexMessage) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.encodeXDR(xw) +} + +func (o IndexMessage) MarshalXDR() []byte { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o IndexMessage) AppendXDR(bs []byte) []byte { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + o.encodeXDR(xw) + return []byte(aw) +} + +func (o IndexMessage) encodeXDR(xw *xdr.Writer) (int, error) { + if len(o.Repository) > 64 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteString(o.Repository) + xw.WriteUint32(uint32(len(o.Files))) + for i := range o.Files { + _, err := o.Files[i].encodeXDR(xw) + if err != nil { + return xw.Tot(), err + } + } + return xw.Tot(), xw.Error() +} + +func (o *IndexMessage) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.decodeXDR(xr) +} + +func (o *IndexMessage) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.decodeXDR(xr) +} + +func (o *IndexMessage) decodeXDR(xr *xdr.Reader) error { + o.Repository = xr.ReadStringMax(64) + _FilesSize := int(xr.ReadUint32()) + o.Files = make([]FileInfo, _FilesSize) + for i := range o.Files { + (&o.Files[i]).decodeXDR(xr) + } + return xr.Error() +} + +/* + +FileInfo Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Name | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Name (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Flags | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | ++ Modified (64 bits) + +| | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | ++ Version (64 bits) + +| | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | ++ Local Version (64 bits) + +| | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Number of Blocks | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Zero or more BlockInfo Structures \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct FileInfo { + string Name<8192>; + unsigned int Flags; + hyper Modified; + unsigned hyper Version; + unsigned hyper LocalVersion; + BlockInfo Blocks<>; +} + +*/ + +func (o FileInfo) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.encodeXDR(xw) +} + +func (o FileInfo) MarshalXDR() []byte { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o FileInfo) AppendXDR(bs []byte) []byte { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + o.encodeXDR(xw) + return []byte(aw) +} + +func (o FileInfo) encodeXDR(xw *xdr.Writer) (int, error) { + if len(o.Name) > 8192 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteString(o.Name) + xw.WriteUint32(o.Flags) + xw.WriteUint64(uint64(o.Modified)) + xw.WriteUint64(o.Version) + xw.WriteUint64(o.LocalVersion) + xw.WriteUint32(uint32(len(o.Blocks))) + for i := range o.Blocks { + _, err := o.Blocks[i].encodeXDR(xw) + if err != nil { + return xw.Tot(), err + } + } + return xw.Tot(), xw.Error() +} + +func (o *FileInfo) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.decodeXDR(xr) +} + +func (o *FileInfo) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.decodeXDR(xr) +} + +func (o *FileInfo) decodeXDR(xr *xdr.Reader) error { + o.Name = xr.ReadStringMax(8192) + o.Flags = xr.ReadUint32() + o.Modified = int64(xr.ReadUint64()) + o.Version = xr.ReadUint64() + o.LocalVersion = xr.ReadUint64() + _BlocksSize := int(xr.ReadUint32()) + o.Blocks = make([]BlockInfo, _BlocksSize) + for i := range o.Blocks { + (&o.Blocks[i]).decodeXDR(xr) + } + return xr.Error() +} + +/* + +FileInfoTruncated Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Name | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Name (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Flags | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | ++ Modified (64 bits) + +| | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | ++ Version (64 bits) + +| | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | ++ Local Version (64 bits) + +| | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Num Blocks | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct FileInfoTruncated { + string Name<8192>; + unsigned int Flags; + hyper Modified; + unsigned hyper Version; + unsigned hyper LocalVersion; + unsigned int NumBlocks; +} + +*/ + +func (o FileInfoTruncated) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.encodeXDR(xw) +} + +func (o FileInfoTruncated) MarshalXDR() []byte { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o FileInfoTruncated) AppendXDR(bs []byte) []byte { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + o.encodeXDR(xw) + return []byte(aw) +} + +func (o FileInfoTruncated) encodeXDR(xw *xdr.Writer) (int, error) { + if len(o.Name) > 8192 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteString(o.Name) + xw.WriteUint32(o.Flags) + xw.WriteUint64(uint64(o.Modified)) + xw.WriteUint64(o.Version) + xw.WriteUint64(o.LocalVersion) + xw.WriteUint32(o.NumBlocks) + return xw.Tot(), xw.Error() +} + +func (o *FileInfoTruncated) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.decodeXDR(xr) +} + +func (o *FileInfoTruncated) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.decodeXDR(xr) +} + +func (o *FileInfoTruncated) decodeXDR(xr *xdr.Reader) error { + o.Name = xr.ReadStringMax(8192) + o.Flags = xr.ReadUint32() + o.Modified = int64(xr.ReadUint64()) + o.Version = xr.ReadUint64() + o.LocalVersion = xr.ReadUint64() + o.NumBlocks = xr.ReadUint32() + return xr.Error() +} + +/* + +BlockInfo Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Size | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Hash | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Hash (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct BlockInfo { + unsigned int Size; + opaque Hash<64>; +} + +*/ + +func (o BlockInfo) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.encodeXDR(xw) +} + +func (o BlockInfo) MarshalXDR() []byte { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o BlockInfo) AppendXDR(bs []byte) []byte { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + o.encodeXDR(xw) + return []byte(aw) +} + +func (o BlockInfo) encodeXDR(xw *xdr.Writer) (int, error) { + xw.WriteUint32(o.Size) + if len(o.Hash) > 64 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteBytes(o.Hash) + return xw.Tot(), xw.Error() +} + +func (o *BlockInfo) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.decodeXDR(xr) +} + +func (o *BlockInfo) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.decodeXDR(xr) +} + +func (o *BlockInfo) decodeXDR(xr *xdr.Reader) error { + o.Size = xr.ReadUint32() + o.Hash = xr.ReadBytesMax(64) + return xr.Error() +} + +/* + +RequestMessage Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Repository | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Repository (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Name | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Name (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | ++ Offset (64 bits) + +| | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Size | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct RequestMessage { + string Repository<64>; + string Name<8192>; + unsigned hyper Offset; + unsigned int Size; +} + +*/ + +func (o RequestMessage) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.encodeXDR(xw) +} + +func (o RequestMessage) MarshalXDR() []byte { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o RequestMessage) AppendXDR(bs []byte) []byte { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + o.encodeXDR(xw) + return []byte(aw) +} + +func (o RequestMessage) encodeXDR(xw *xdr.Writer) (int, error) { + if len(o.Repository) > 64 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteString(o.Repository) + if len(o.Name) > 8192 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteString(o.Name) + xw.WriteUint64(o.Offset) + xw.WriteUint32(o.Size) + return xw.Tot(), xw.Error() +} + +func (o *RequestMessage) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.decodeXDR(xr) +} + +func (o *RequestMessage) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.decodeXDR(xr) +} + +func (o *RequestMessage) decodeXDR(xr *xdr.Reader) error { + o.Repository = xr.ReadStringMax(64) + o.Name = xr.ReadStringMax(8192) + o.Offset = xr.ReadUint64() + o.Size = xr.ReadUint32() + return xr.Error() +} + +/* + +ResponseMessage Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Data | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Data (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct ResponseMessage { + opaque Data<>; +} + +*/ + +func (o ResponseMessage) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.encodeXDR(xw) +} + +func (o ResponseMessage) MarshalXDR() []byte { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o ResponseMessage) AppendXDR(bs []byte) []byte { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + o.encodeXDR(xw) + return []byte(aw) +} + +func (o ResponseMessage) encodeXDR(xw *xdr.Writer) (int, error) { + xw.WriteBytes(o.Data) + return xw.Tot(), xw.Error() +} + +func (o *ResponseMessage) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.decodeXDR(xr) +} + +func (o *ResponseMessage) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.decodeXDR(xr) +} + +func (o *ResponseMessage) decodeXDR(xr *xdr.Reader) error { + o.Data = xr.ReadBytes() + return xr.Error() +} + +/* + +ClusterConfigMessage Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Client Name | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Client Name (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Client Version | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Client Version (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Number of Repositories | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Zero or more Repository Structures \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Number of Options | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Zero or more Option Structures \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct ClusterConfigMessage { + string ClientName<64>; + string ClientVersion<64>; + Repository Repositories<64>; + Option Options<64>; +} + +*/ + +func (o ClusterConfigMessage) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.encodeXDR(xw) +} + +func (o ClusterConfigMessage) MarshalXDR() []byte { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o ClusterConfigMessage) AppendXDR(bs []byte) []byte { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + o.encodeXDR(xw) + return []byte(aw) +} + +func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) { + if len(o.ClientName) > 64 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteString(o.ClientName) + if len(o.ClientVersion) > 64 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteString(o.ClientVersion) + if len(o.Repositories) > 64 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteUint32(uint32(len(o.Repositories))) + for i := range o.Repositories { + _, err := o.Repositories[i].encodeXDR(xw) + if err != nil { + return xw.Tot(), err + } + } + if len(o.Options) > 64 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteUint32(uint32(len(o.Options))) + for i := range o.Options { + _, err := o.Options[i].encodeXDR(xw) + if err != nil { + return xw.Tot(), err + } + } + return xw.Tot(), xw.Error() +} + +func (o *ClusterConfigMessage) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.decodeXDR(xr) +} + +func (o *ClusterConfigMessage) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.decodeXDR(xr) +} + +func (o *ClusterConfigMessage) decodeXDR(xr *xdr.Reader) error { + o.ClientName = xr.ReadStringMax(64) + o.ClientVersion = xr.ReadStringMax(64) + _RepositoriesSize := int(xr.ReadUint32()) + if _RepositoriesSize > 64 { + return xdr.ErrElementSizeExceeded + } + o.Repositories = make([]Repository, _RepositoriesSize) + for i := range o.Repositories { + (&o.Repositories[i]).decodeXDR(xr) + } + _OptionsSize := int(xr.ReadUint32()) + if _OptionsSize > 64 { + return xdr.ErrElementSizeExceeded + } + o.Options = make([]Option, _OptionsSize) + for i := range o.Options { + (&o.Options[i]).decodeXDR(xr) + } + return xr.Error() +} + +/* + +Repository Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of ID | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ ID (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Number of Nodes | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Zero or more Node Structures \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct Repository { + string ID<64>; + Node Nodes<64>; +} + +*/ + +func (o Repository) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.encodeXDR(xw) +} + +func (o Repository) MarshalXDR() []byte { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o Repository) AppendXDR(bs []byte) []byte { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + o.encodeXDR(xw) + return []byte(aw) +} + +func (o Repository) encodeXDR(xw *xdr.Writer) (int, error) { + if len(o.ID) > 64 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteString(o.ID) + if len(o.Nodes) > 64 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteUint32(uint32(len(o.Nodes))) + for i := range o.Nodes { + _, err := o.Nodes[i].encodeXDR(xw) + if err != nil { + return xw.Tot(), err + } + } + return xw.Tot(), xw.Error() +} + +func (o *Repository) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.decodeXDR(xr) +} + +func (o *Repository) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.decodeXDR(xr) +} + +func (o *Repository) decodeXDR(xr *xdr.Reader) error { + o.ID = xr.ReadStringMax(64) + _NodesSize := int(xr.ReadUint32()) + if _NodesSize > 64 { + return xdr.ErrElementSizeExceeded + } + o.Nodes = make([]Node, _NodesSize) + for i := range o.Nodes { + (&o.Nodes[i]).decodeXDR(xr) + } + return xr.Error() +} + +/* + +Node Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of ID | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ ID (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Flags | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | ++ Max Local Version (64 bits) + +| | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct Node { + opaque ID<32>; + unsigned int Flags; + unsigned hyper MaxLocalVersion; +} + +*/ + +func (o Node) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.encodeXDR(xw) +} + +func (o Node) MarshalXDR() []byte { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o Node) AppendXDR(bs []byte) []byte { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + o.encodeXDR(xw) + return []byte(aw) +} + +func (o Node) encodeXDR(xw *xdr.Writer) (int, error) { + if len(o.ID) > 32 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteBytes(o.ID) + xw.WriteUint32(o.Flags) + xw.WriteUint64(o.MaxLocalVersion) + return xw.Tot(), xw.Error() +} + +func (o *Node) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.decodeXDR(xr) +} + +func (o *Node) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.decodeXDR(xr) +} + +func (o *Node) decodeXDR(xr *xdr.Reader) error { + o.ID = xr.ReadBytesMax(32) + o.Flags = xr.ReadUint32() + o.MaxLocalVersion = xr.ReadUint64() + return xr.Error() +} + +/* + +Option Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Key | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Key (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Value | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Value (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct Option { + string Key<64>; + string Value<1024>; +} + +*/ + +func (o Option) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.encodeXDR(xw) +} + +func (o Option) MarshalXDR() []byte { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o Option) AppendXDR(bs []byte) []byte { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + o.encodeXDR(xw) + return []byte(aw) +} + +func (o Option) encodeXDR(xw *xdr.Writer) (int, error) { + if len(o.Key) > 64 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteString(o.Key) + if len(o.Value) > 1024 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteString(o.Value) + return xw.Tot(), xw.Error() +} + +func (o *Option) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.decodeXDR(xr) +} + +func (o *Option) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.decodeXDR(xr) +} + +func (o *Option) decodeXDR(xr *xdr.Reader) error { + o.Key = xr.ReadStringMax(64) + o.Value = xr.ReadStringMax(1024) + return xr.Error() +} + +/* + +CloseMessage Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Reason | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Reason (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct CloseMessage { + string Reason<1024>; +} + +*/ + +func (o CloseMessage) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.encodeXDR(xw) +} + +func (o CloseMessage) MarshalXDR() []byte { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o CloseMessage) AppendXDR(bs []byte) []byte { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + o.encodeXDR(xw) + return []byte(aw) +} + +func (o CloseMessage) encodeXDR(xw *xdr.Writer) (int, error) { + if len(o.Reason) > 1024 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteString(o.Reason) + return xw.Tot(), xw.Error() +} + +func (o *CloseMessage) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.decodeXDR(xr) +} + +func (o *CloseMessage) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.decodeXDR(xr) +} + +func (o *CloseMessage) decodeXDR(xr *xdr.Reader) error { + o.Reason = xr.ReadStringMax(1024) + return xr.Error() +} + +/* + +EmptyMessage Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct EmptyMessage { +} + +*/ + +func (o EmptyMessage) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.encodeXDR(xw) +} + +func (o EmptyMessage) MarshalXDR() []byte { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o EmptyMessage) AppendXDR(bs []byte) []byte { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + o.encodeXDR(xw) + return []byte(aw) +} + +func (o EmptyMessage) encodeXDR(xw *xdr.Writer) (int, error) { + return xw.Tot(), xw.Error() +} + +func (o *EmptyMessage) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.decodeXDR(xr) +} + +func (o *EmptyMessage) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.decodeXDR(xr) +} + +func (o *EmptyMessage) decodeXDR(xr *xdr.Reader) error { + return xr.Error() +} diff --git a/nativemodel_darwin.go b/nativemodel_darwin.go new file mode 100644 index 000000000..9ac402fe6 --- /dev/null +++ b/nativemodel_darwin.go @@ -0,0 +1,42 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +// +build darwin + +package protocol + +// Darwin uses NFD normalization + +import "code.google.com/p/go.text/unicode/norm" + +type nativeModel struct { + next Model +} + +func (m nativeModel) Index(nodeID NodeID, repo string, files []FileInfo) { + for i := range files { + files[i].Name = norm.NFD.String(files[i].Name) + } + m.next.Index(nodeID, repo, files) +} + +func (m nativeModel) IndexUpdate(nodeID NodeID, repo string, files []FileInfo) { + for i := range files { + files[i].Name = norm.NFD.String(files[i].Name) + } + m.next.IndexUpdate(nodeID, repo, files) +} + +func (m nativeModel) Request(nodeID NodeID, repo string, name string, offset int64, size int) ([]byte, error) { + name = norm.NFD.String(name) + return m.next.Request(nodeID, repo, name, offset, size) +} + +func (m nativeModel) ClusterConfig(nodeID NodeID, config ClusterConfigMessage) { + m.next.ClusterConfig(nodeID, config) +} + +func (m nativeModel) Close(nodeID NodeID, err error) { + m.next.Close(nodeID, err) +} diff --git a/nativemodel_unix.go b/nativemodel_unix.go new file mode 100644 index 000000000..23fbe0b6b --- /dev/null +++ b/nativemodel_unix.go @@ -0,0 +1,33 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +// +build !windows,!darwin + +package protocol + +// Normal Unixes uses NFC and slashes, which is the wire format. + +type nativeModel struct { + next Model +} + +func (m nativeModel) Index(nodeID NodeID, repo string, files []FileInfo) { + m.next.Index(nodeID, repo, files) +} + +func (m nativeModel) IndexUpdate(nodeID NodeID, repo string, files []FileInfo) { + m.next.IndexUpdate(nodeID, repo, files) +} + +func (m nativeModel) Request(nodeID NodeID, repo string, name string, offset int64, size int) ([]byte, error) { + return m.next.Request(nodeID, repo, name, offset, size) +} + +func (m nativeModel) ClusterConfig(nodeID NodeID, config ClusterConfigMessage) { + m.next.ClusterConfig(nodeID, config) +} + +func (m nativeModel) Close(nodeID NodeID, err error) { + m.next.Close(nodeID, err) +} diff --git a/nativemodel_windows.go b/nativemodel_windows.go new file mode 100644 index 000000000..9841d63f7 --- /dev/null +++ b/nativemodel_windows.go @@ -0,0 +1,72 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +// +build windows + +package protocol + +// Windows uses backslashes as file separator and disallows a bunch of +// characters in the filename + +import ( + "path/filepath" + "strings" +) + +var disallowedCharacters = string([]rune{ + '<', '>', ':', '"', '|', '?', '*', + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, + 31, +}) + +type nativeModel struct { + next Model +} + +func (m nativeModel) Index(nodeID NodeID, repo string, files []FileInfo) { + for i, f := range files { + if strings.ContainsAny(f.Name, disallowedCharacters) { + if f.IsDeleted() { + // Don't complain if the file is marked as deleted, since it + // can't possibly exist here anyway. + continue + } + files[i].Flags |= FlagInvalid + l.Warnf("File name %q contains invalid characters; marked as invalid.", f.Name) + } + files[i].Name = filepath.FromSlash(f.Name) + } + m.next.Index(nodeID, repo, files) +} + +func (m nativeModel) IndexUpdate(nodeID NodeID, repo string, files []FileInfo) { + for i, f := range files { + if strings.ContainsAny(f.Name, disallowedCharacters) { + if f.IsDeleted() { + // Don't complain if the file is marked as deleted, since it + // can't possibly exist here anyway. + continue + } + files[i].Flags |= FlagInvalid + l.Warnf("File name %q contains invalid characters; marked as invalid.", f.Name) + } + files[i].Name = filepath.FromSlash(files[i].Name) + } + m.next.IndexUpdate(nodeID, repo, files) +} + +func (m nativeModel) Request(nodeID NodeID, repo string, name string, offset int64, size int) ([]byte, error) { + name = filepath.FromSlash(name) + return m.next.Request(nodeID, repo, name, offset, size) +} + +func (m nativeModel) ClusterConfig(nodeID NodeID, config ClusterConfigMessage) { + m.next.ClusterConfig(nodeID, config) +} + +func (m nativeModel) Close(nodeID NodeID, err error) { + m.next.Close(nodeID, err) +} diff --git a/nodeid.go b/nodeid.go new file mode 100644 index 000000000..9079781b9 --- /dev/null +++ b/nodeid.go @@ -0,0 +1,159 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package protocol + +import ( + "bytes" + "crypto/sha256" + "encoding/base32" + "errors" + "fmt" + "regexp" + "strings" + + "github.com/syncthing/syncthing/internal/luhn" +) + +type NodeID [32]byte + +var LocalNodeID = NodeID{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} + +// NewNodeID generates a new node ID from the raw bytes of a certificate +func NewNodeID(rawCert []byte) NodeID { + var n NodeID + hf := sha256.New() + hf.Write(rawCert) + hf.Sum(n[:0]) + return n +} + +func NodeIDFromString(s string) (NodeID, error) { + var n NodeID + err := n.UnmarshalText([]byte(s)) + return n, err +} + +func NodeIDFromBytes(bs []byte) NodeID { + var n NodeID + if len(bs) != len(n) { + panic("incorrect length of byte slice representing node ID") + } + copy(n[:], bs) + return n +} + +// String returns the canonical string representation of the node ID +func (n NodeID) String() string { + id := base32.StdEncoding.EncodeToString(n[:]) + id = strings.Trim(id, "=") + id, err := luhnify(id) + if err != nil { + // Should never happen + panic(err) + } + id = chunkify(id) + return id +} + +func (n NodeID) GoString() string { + return n.String() +} + +func (n NodeID) Compare(other NodeID) int { + return bytes.Compare(n[:], other[:]) +} + +func (n NodeID) Equals(other NodeID) bool { + return bytes.Compare(n[:], other[:]) == 0 +} + +func (n *NodeID) MarshalText() ([]byte, error) { + return []byte(n.String()), nil +} + +func (n *NodeID) UnmarshalText(bs []byte) error { + id := string(bs) + id = strings.Trim(id, "=") + id = strings.ToUpper(id) + id = untypeoify(id) + id = unchunkify(id) + + var err error + switch len(id) { + case 56: + // New style, with check digits + id, err = unluhnify(id) + if err != nil { + return err + } + fallthrough + case 52: + // Old style, no check digits + dec, err := base32.StdEncoding.DecodeString(id + "====") + if err != nil { + return err + } + copy(n[:], dec) + return nil + default: + return errors.New("node ID invalid: incorrect length") + } +} + +func luhnify(s string) (string, error) { + if len(s) != 52 { + panic("unsupported string length") + } + + res := make([]string, 0, 4) + for i := 0; i < 4; i++ { + p := s[i*13 : (i+1)*13] + l, err := luhn.Base32.Generate(p) + if err != nil { + return "", err + } + res = append(res, fmt.Sprintf("%s%c", p, l)) + } + return res[0] + res[1] + res[2] + res[3], nil +} + +func unluhnify(s string) (string, error) { + if len(s) != 56 { + return "", fmt.Errorf("unsupported string length %d", len(s)) + } + + res := make([]string, 0, 4) + for i := 0; i < 4; i++ { + p := s[i*14 : (i+1)*14-1] + l, err := luhn.Base32.Generate(p) + if err != nil { + return "", err + } + if g := fmt.Sprintf("%s%c", p, l); g != s[i*14:(i+1)*14] { + return "", errors.New("check digit incorrect") + } + res = append(res, p) + } + return res[0] + res[1] + res[2] + res[3], nil +} + +func chunkify(s string) string { + s = regexp.MustCompile("(.{7})").ReplaceAllString(s, "$1-") + s = strings.Trim(s, "-") + return s +} + +func unchunkify(s string) string { + s = strings.Replace(s, "-", "", -1) + s = strings.Replace(s, " ", "", -1) + return s +} + +func untypeoify(s string) string { + s = strings.Replace(s, "0", "O", -1) + s = strings.Replace(s, "1", "I", -1) + s = strings.Replace(s, "8", "B", -1) + return s +} diff --git a/nodeid_test.go b/nodeid_test.go new file mode 100644 index 000000000..5b861b6de --- /dev/null +++ b/nodeid_test.go @@ -0,0 +1,78 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package protocol + +import "testing" + +var formatted = "P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" +var formatCases = []string{ + "P56IOI-7MZJNU-2IQGDR-EYDM2M-GTMGL3-BXNPQ6-W5BTBB-Z4TJXZ-WICQ", + "P56IOI-7MZJNU2Y-IQGDR-EYDM2M-GTI-MGL3-BXNPQ6-W5BM-TBB-Z4TJXZ-WICQ2", + "P56IOI7 MZJNU2I QGDREYD M2MGTMGL 3BXNPQ6W 5BTB BZ4T JXZWICQ", + "P56IOI7 MZJNU2Y IQGDREY DM2MGTI MGL3BXN PQ6W5BM TBBZ4TJ XZWICQ2", + "P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQ", + "p56ioi7mzjnu2iqgdreydm2mgtmgl3bxnpq6w5btbbz4tjxzwicq", + "P56IOI7MZJNU2YIQGDREYDM2MGTIMGL3BXNPQ6W5BMTBBZ4TJXZWICQ2", + "P561017MZJNU2YIQGDREYDM2MGTIMGL3BXNPQ6W5BMT88Z4TJXZWICQ2", + "p56ioi7mzjnu2yiqgdreydm2mgtimgl3bxnpq6w5bmtbbz4tjxzwicq2", + "p561017mzjnu2yiqgdreydm2mgtimgl3bxnpq6w5bmt88z4tjxzwicq2", +} + +func TestFormatNodeID(t *testing.T) { + for i, tc := range formatCases { + var id NodeID + err := id.UnmarshalText([]byte(tc)) + if err != nil { + t.Errorf("#%d UnmarshalText(%q); %v", i, tc, err) + } else if f := id.String(); f != formatted { + t.Errorf("#%d FormatNodeID(%q)\n\t%q !=\n\t%q", i, tc, f, formatted) + } + } +} + +var validateCases = []struct { + s string + ok bool +}{ + {"", false}, + {"P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2", true}, + {"P56IOI7-MZJNU2-IQGDREY-DM2MGT-MGL3BXN-PQ6W5B-TBBZ4TJ-XZWICQ", true}, + {"P56IOI7 MZJNU2I QGDREYD M2MGTMGL 3BXNPQ6W 5BTB BZ4T JXZWICQ", true}, + {"P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQ", true}, + {"P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQCCCC", false}, + {"p56ioi7mzjnu2iqgdreydm2mgtmgl3bxnpq6w5btbbz4tjxzwicq", true}, + {"p56ioi7mzjnu2iqgdreydm2mgtmgl3bxnpq6w5btbbz4tjxzwicqCCCC", false}, +} + +func TestValidateNodeID(t *testing.T) { + for _, tc := range validateCases { + var id NodeID + err := id.UnmarshalText([]byte(tc.s)) + if (err == nil && !tc.ok) || (err != nil && tc.ok) { + t.Errorf("ValidateNodeID(%q); %v != %v", tc.s, err, tc.ok) + } + } +} + +func TestMarshallingNodeID(t *testing.T) { + n0 := NodeID{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 10, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32} + n1 := NodeID{} + n2 := NodeID{} + + bs, _ := n0.MarshalText() + n1.UnmarshalText(bs) + bs, _ = n1.MarshalText() + n2.UnmarshalText(bs) + + if n2.String() != n0.String() { + t.Errorf("String marshalling error; %q != %q", n2.String(), n0.String()) + } + if !n2.Equals(n0) { + t.Error("Equals error") + } + if n2.Compare(n0) != 0 { + t.Error("Compare error") + } +} diff --git a/protocol.go b/protocol.go new file mode 100644 index 000000000..86fd6199f --- /dev/null +++ b/protocol.go @@ -0,0 +1,640 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package protocol + +import ( + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "io" + "sync" + "time" + + lz4 "github.com/bkaradzic/go-lz4" +) + +const ( + BlockSize = 128 * 1024 +) + +const ( + messageTypeClusterConfig = 0 + messageTypeIndex = 1 + messageTypeRequest = 2 + messageTypeResponse = 3 + messageTypePing = 4 + messageTypePong = 5 + messageTypeIndexUpdate = 6 + messageTypeClose = 7 +) + +const ( + stateInitial = iota + stateCCRcvd + stateIdxRcvd +) + +const ( + FlagDeleted uint32 = 1 << 12 + FlagInvalid = 1 << 13 + FlagDirectory = 1 << 14 + FlagNoPermBits = 1 << 15 +) + +const ( + FlagShareTrusted uint32 = 1 << 0 + FlagShareReadOnly = 1 << 1 + FlagIntroducer = 1 << 2 + FlagShareBits = 0x000000ff +) + +var ( + ErrClusterHash = fmt.Errorf("configuration error: mismatched cluster hash") + ErrClosed = errors.New("connection closed") +) + +type Model interface { + // An index was received from the peer node + Index(nodeID NodeID, repo string, files []FileInfo) + // An index update was received from the peer node + IndexUpdate(nodeID NodeID, repo string, files []FileInfo) + // A request was made by the peer node + Request(nodeID NodeID, repo string, name string, offset int64, size int) ([]byte, error) + // A cluster configuration message was received + ClusterConfig(nodeID NodeID, config ClusterConfigMessage) + // The peer node closed the connection + Close(nodeID NodeID, err error) +} + +type Connection interface { + ID() NodeID + Name() string + Index(repo string, files []FileInfo) error + IndexUpdate(repo string, files []FileInfo) error + Request(repo string, name string, offset int64, size int) ([]byte, error) + ClusterConfig(config ClusterConfigMessage) + Statistics() Statistics +} + +type rawConnection struct { + id NodeID + name string + receiver Model + state int + + cr *countingReader + cw *countingWriter + + awaiting [4096]chan asyncResult + awaitingMut sync.Mutex + + idxMut sync.Mutex // ensures serialization of Index calls + + nextID chan int + outbox chan hdrMsg + closed chan struct{} + once sync.Once + + compressionThreshold int // compress messages larger than this many bytes + + rdbuf0 []byte // used & reused by readMessage + rdbuf1 []byte // used & reused by readMessage +} + +type asyncResult struct { + val []byte + err error +} + +type hdrMsg struct { + hdr header + msg encodable +} + +type encodable interface { + AppendXDR([]byte) []byte +} + +const ( + pingTimeout = 30 * time.Second + pingIdleTime = 60 * time.Second +) + +func NewConnection(nodeID NodeID, reader io.Reader, writer io.Writer, receiver Model, name string, compress bool) Connection { + cr := &countingReader{Reader: reader} + cw := &countingWriter{Writer: writer} + + compThres := 1<<31 - 1 // compression disabled + if compress { + compThres = 128 // compress messages that are 128 bytes long or larger + } + c := rawConnection{ + id: nodeID, + name: name, + receiver: nativeModel{receiver}, + state: stateInitial, + cr: cr, + cw: cw, + outbox: make(chan hdrMsg), + nextID: make(chan int), + closed: make(chan struct{}), + compressionThreshold: compThres, + } + + go c.readerLoop() + go c.writerLoop() + go c.pingerLoop() + go c.idGenerator() + + return wireFormatConnection{&c} +} + +func (c *rawConnection) ID() NodeID { + return c.id +} + +func (c *rawConnection) Name() string { + return c.name +} + +// Index writes the list of file information to the connected peer node +func (c *rawConnection) Index(repo string, idx []FileInfo) error { + select { + case <-c.closed: + return ErrClosed + default: + } + c.idxMut.Lock() + c.send(-1, messageTypeIndex, IndexMessage{repo, idx}) + c.idxMut.Unlock() + return nil +} + +// IndexUpdate writes the list of file information to the connected peer node as an update +func (c *rawConnection) IndexUpdate(repo string, idx []FileInfo) error { + select { + case <-c.closed: + return ErrClosed + default: + } + c.idxMut.Lock() + c.send(-1, messageTypeIndexUpdate, IndexMessage{repo, idx}) + c.idxMut.Unlock() + return nil +} + +// Request returns the bytes for the specified block after fetching them from the connected peer. +func (c *rawConnection) Request(repo string, name string, offset int64, size int) ([]byte, error) { + var id int + select { + case id = <-c.nextID: + case <-c.closed: + return nil, ErrClosed + } + + c.awaitingMut.Lock() + if ch := c.awaiting[id]; ch != nil { + panic("id taken") + } + rc := make(chan asyncResult, 1) + c.awaiting[id] = rc + c.awaitingMut.Unlock() + + ok := c.send(id, messageTypeRequest, RequestMessage{repo, name, uint64(offset), uint32(size)}) + if !ok { + return nil, ErrClosed + } + + res, ok := <-rc + if !ok { + return nil, ErrClosed + } + return res.val, res.err +} + +// ClusterConfig send the cluster configuration message to the peer and returns any error +func (c *rawConnection) ClusterConfig(config ClusterConfigMessage) { + c.send(-1, messageTypeClusterConfig, config) +} + +func (c *rawConnection) ping() bool { + var id int + select { + case id = <-c.nextID: + case <-c.closed: + return false + } + + rc := make(chan asyncResult, 1) + c.awaitingMut.Lock() + c.awaiting[id] = rc + c.awaitingMut.Unlock() + + ok := c.send(id, messageTypePing, nil) + if !ok { + return false + } + + res, ok := <-rc + return ok && res.err == nil +} + +func (c *rawConnection) readerLoop() (err error) { + defer func() { + c.close(err) + }() + + for { + select { + case <-c.closed: + return ErrClosed + default: + } + + hdr, msg, err := c.readMessage() + if err != nil { + return err + } + + switch hdr.msgType { + case messageTypeIndex: + if c.state < stateCCRcvd { + return fmt.Errorf("protocol error: index message in state %d", c.state) + } + c.handleIndex(msg.(IndexMessage)) + c.state = stateIdxRcvd + + case messageTypeIndexUpdate: + if c.state < stateIdxRcvd { + return fmt.Errorf("protocol error: index update message in state %d", c.state) + } + c.handleIndexUpdate(msg.(IndexMessage)) + + case messageTypeRequest: + if c.state < stateIdxRcvd { + return fmt.Errorf("protocol error: request message in state %d", c.state) + } + // Requests are handled asynchronously + go c.handleRequest(hdr.msgID, msg.(RequestMessage)) + + case messageTypeResponse: + if c.state < stateIdxRcvd { + return fmt.Errorf("protocol error: response message in state %d", c.state) + } + c.handleResponse(hdr.msgID, msg.(ResponseMessage)) + + case messageTypePing: + c.send(hdr.msgID, messageTypePong, EmptyMessage{}) + + case messageTypePong: + c.handlePong(hdr.msgID) + + case messageTypeClusterConfig: + if c.state != stateInitial { + return fmt.Errorf("protocol error: cluster config message in state %d", c.state) + } + go c.receiver.ClusterConfig(c.id, msg.(ClusterConfigMessage)) + c.state = stateCCRcvd + + case messageTypeClose: + return errors.New(msg.(CloseMessage).Reason) + + default: + return fmt.Errorf("protocol error: %s: unknown message type %#x", c.id, hdr.msgType) + } + } +} + +func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) { + if cap(c.rdbuf0) < 8 { + c.rdbuf0 = make([]byte, 8) + } else { + c.rdbuf0 = c.rdbuf0[:8] + } + _, err = io.ReadFull(c.cr, c.rdbuf0) + if err != nil { + return + } + + hdr = decodeHeader(binary.BigEndian.Uint32(c.rdbuf0[0:4])) + msglen := int(binary.BigEndian.Uint32(c.rdbuf0[4:8])) + + if debug { + l.Debugf("read header %v (msglen=%d)", hdr, msglen) + } + + if cap(c.rdbuf0) < msglen { + c.rdbuf0 = make([]byte, msglen) + } else { + c.rdbuf0 = c.rdbuf0[:msglen] + } + _, err = io.ReadFull(c.cr, c.rdbuf0) + if err != nil { + return + } + + if debug { + l.Debugf("read %d bytes", len(c.rdbuf0)) + } + + msgBuf := c.rdbuf0 + if hdr.compression { + c.rdbuf1 = c.rdbuf1[:cap(c.rdbuf1)] + c.rdbuf1, err = lz4.Decode(c.rdbuf1, c.rdbuf0) + if err != nil { + return + } + msgBuf = c.rdbuf1 + if debug { + l.Debugf("decompressed to %d bytes", len(msgBuf)) + } + } + + if debug { + if len(msgBuf) > 1024 { + l.Debugf("message data:\n%s", hex.Dump(msgBuf[:1024])) + } else { + l.Debugf("message data:\n%s", hex.Dump(msgBuf)) + } + } + + switch hdr.msgType { + case messageTypeIndex, messageTypeIndexUpdate: + var idx IndexMessage + err = idx.UnmarshalXDR(msgBuf) + msg = idx + + case messageTypeRequest: + var req RequestMessage + err = req.UnmarshalXDR(msgBuf) + msg = req + + case messageTypeResponse: + var resp ResponseMessage + err = resp.UnmarshalXDR(msgBuf) + msg = resp + + case messageTypePing, messageTypePong: + msg = EmptyMessage{} + + case messageTypeClusterConfig: + var cc ClusterConfigMessage + err = cc.UnmarshalXDR(msgBuf) + msg = cc + + case messageTypeClose: + var cm CloseMessage + err = cm.UnmarshalXDR(msgBuf) + msg = cm + + default: + err = fmt.Errorf("protocol error: %s: unknown message type %#x", c.id, hdr.msgType) + } + + return +} + +func (c *rawConnection) handleIndex(im IndexMessage) { + if debug { + l.Debugf("Index(%v, %v, %d files)", c.id, im.Repository, len(im.Files)) + } + c.receiver.Index(c.id, im.Repository, im.Files) +} + +func (c *rawConnection) handleIndexUpdate(im IndexMessage) { + if debug { + l.Debugf("queueing IndexUpdate(%v, %v, %d files)", c.id, im.Repository, len(im.Files)) + } + c.receiver.IndexUpdate(c.id, im.Repository, im.Files) +} + +func (c *rawConnection) handleRequest(msgID int, req RequestMessage) { + data, _ := c.receiver.Request(c.id, req.Repository, req.Name, int64(req.Offset), int(req.Size)) + + c.send(msgID, messageTypeResponse, ResponseMessage{data}) +} + +func (c *rawConnection) handleResponse(msgID int, resp ResponseMessage) { + c.awaitingMut.Lock() + if rc := c.awaiting[msgID]; rc != nil { + c.awaiting[msgID] = nil + rc <- asyncResult{resp.Data, nil} + close(rc) + } + c.awaitingMut.Unlock() +} + +func (c *rawConnection) handlePong(msgID int) { + c.awaitingMut.Lock() + if rc := c.awaiting[msgID]; rc != nil { + c.awaiting[msgID] = nil + rc <- asyncResult{} + close(rc) + } + c.awaitingMut.Unlock() +} + +func (c *rawConnection) send(msgID int, msgType int, msg encodable) bool { + if msgID < 0 { + select { + case id := <-c.nextID: + msgID = id + case <-c.closed: + return false + } + } + + hdr := header{ + version: 0, + msgID: msgID, + msgType: msgType, + } + + select { + case c.outbox <- hdrMsg{hdr, msg}: + return true + case <-c.closed: + return false + } +} + +func (c *rawConnection) writerLoop() { + var msgBuf = make([]byte, 8) // buffer for wire format message, kept and reused + var uncBuf []byte // buffer for uncompressed message, kept and reused + for { + var tempBuf []byte + var err error + + select { + case hm := <-c.outbox: + if hm.msg != nil { + // Uncompressed message in uncBuf + uncBuf = hm.msg.AppendXDR(uncBuf[:0]) + + if len(uncBuf) >= c.compressionThreshold { + // Use compression for large messages + hm.hdr.compression = true + + // Make sure we have enough space for the compressed message plus header in msgBug + msgBuf = msgBuf[:cap(msgBuf)] + if maxLen := lz4.CompressBound(len(uncBuf)) + 8; maxLen > len(msgBuf) { + msgBuf = make([]byte, maxLen) + } + + // Compressed is written to msgBuf, we keep tb for the length only + tempBuf, err = lz4.Encode(msgBuf[8:], uncBuf) + binary.BigEndian.PutUint32(msgBuf[4:8], uint32(len(tempBuf))) + msgBuf = msgBuf[0 : len(tempBuf)+8] + + if debug { + l.Debugf("write compressed message; %v (len=%d)", hm.hdr, len(tempBuf)) + } + } else { + // No point in compressing very short messages + hm.hdr.compression = false + + msgBuf = msgBuf[:cap(msgBuf)] + if l := len(uncBuf) + 8; l > len(msgBuf) { + msgBuf = make([]byte, l) + } + + binary.BigEndian.PutUint32(msgBuf[4:8], uint32(len(uncBuf))) + msgBuf = msgBuf[0 : len(uncBuf)+8] + copy(msgBuf[8:], uncBuf) + + if debug { + l.Debugf("write uncompressed message; %v (len=%d)", hm.hdr, len(uncBuf)) + } + } + } else { + if debug { + l.Debugf("write empty message; %v", hm.hdr) + } + binary.BigEndian.PutUint32(msgBuf[4:8], 0) + msgBuf = msgBuf[:8] + } + + binary.BigEndian.PutUint32(msgBuf[0:4], encodeHeader(hm.hdr)) + + if err == nil { + var n int + n, err = c.cw.Write(msgBuf) + if debug { + l.Debugf("wrote %d bytes on the wire", n) + } + } + if err != nil { + c.close(err) + return + } + case <-c.closed: + return + } + } +} + +func (c *rawConnection) close(err error) { + c.once.Do(func() { + close(c.closed) + + c.awaitingMut.Lock() + for i, ch := range c.awaiting { + if ch != nil { + close(ch) + c.awaiting[i] = nil + } + } + c.awaitingMut.Unlock() + + go c.receiver.Close(c.id, err) + }) +} + +func (c *rawConnection) idGenerator() { + nextID := 0 + for { + nextID = (nextID + 1) & 0xfff + select { + case c.nextID <- nextID: + case <-c.closed: + return + } + } +} + +func (c *rawConnection) pingerLoop() { + var rc = make(chan bool, 1) + ticker := time.Tick(pingIdleTime / 2) + for { + select { + case <-ticker: + if d := time.Since(c.cr.Last()); d < pingIdleTime { + if debug { + l.Debugln(c.id, "ping skipped after rd", d) + } + continue + } + if d := time.Since(c.cw.Last()); d < pingIdleTime { + if debug { + l.Debugln(c.id, "ping skipped after wr", d) + } + continue + } + go func() { + if debug { + l.Debugln(c.id, "ping ->") + } + rc <- c.ping() + }() + select { + case ok := <-rc: + if debug { + l.Debugln(c.id, "<- pong") + } + if !ok { + c.close(fmt.Errorf("ping failure")) + } + case <-time.After(pingTimeout): + c.close(fmt.Errorf("ping timeout")) + case <-c.closed: + return + } + + case <-c.closed: + return + } + } +} + +type Statistics struct { + At time.Time + InBytesTotal uint64 + OutBytesTotal uint64 +} + +func (c *rawConnection) Statistics() Statistics { + return Statistics{ + At: time.Now(), + InBytesTotal: c.cr.Tot(), + OutBytesTotal: c.cw.Tot(), + } +} + +func IsDeleted(bits uint32) bool { + return bits&FlagDeleted != 0 +} + +func IsInvalid(bits uint32) bool { + return bits&FlagInvalid != 0 +} + +func IsDirectory(bits uint32) bool { + return bits&FlagDirectory != 0 +} + +func HasPermissionBits(bits uint32) bool { + return bits&FlagNoPermBits == 0 +} diff --git a/protocol_test.go b/protocol_test.go new file mode 100644 index 000000000..56a46f2ec --- /dev/null +++ b/protocol_test.go @@ -0,0 +1,383 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package protocol + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "reflect" + "testing" + "testing/quick" + + "github.com/calmh/xdr" +) + +var ( + c0ID = NewNodeID([]byte{1}) + c1ID = NewNodeID([]byte{2}) +) + +func TestHeaderFunctions(t *testing.T) { + f := func(ver, id, typ int) bool { + ver = int(uint(ver) % 16) + id = int(uint(id) % 4096) + typ = int(uint(typ) % 256) + h0 := header{version: ver, msgID: id, msgType: typ} + h1 := decodeHeader(encodeHeader(h0)) + return h0 == h1 + } + if err := quick.Check(f, nil); err != nil { + t.Error(err) + } +} + +func TestHeaderLayout(t *testing.T) { + var e, a uint32 + + // Version are the first four bits + e = 0xf0000000 + a = encodeHeader(header{version: 0xf}) + if a != e { + t.Errorf("Header layout incorrect; %08x != %08x", a, e) + } + + // Message ID are the following 12 bits + e = 0x0fff0000 + a = encodeHeader(header{msgID: 0xfff}) + if a != e { + t.Errorf("Header layout incorrect; %08x != %08x", a, e) + } + + // Type are the last 8 bits before reserved + e = 0x0000ff00 + a = encodeHeader(header{msgType: 0xff}) + if a != e { + t.Errorf("Header layout incorrect; %08x != %08x", a, e) + } +} + +func TestPing(t *testing.T) { + ar, aw := io.Pipe() + br, bw := io.Pipe() + + c0 := NewConnection(c0ID, ar, bw, nil, "name", true).(wireFormatConnection).next.(*rawConnection) + c1 := NewConnection(c1ID, br, aw, nil, "name", true).(wireFormatConnection).next.(*rawConnection) + + if ok := c0.ping(); !ok { + t.Error("c0 ping failed") + } + if ok := c1.ping(); !ok { + t.Error("c1 ping failed") + } +} + +func TestPingErr(t *testing.T) { + e := errors.New("something broke") + + for i := 0; i < 16; i++ { + for j := 0; j < 16; j++ { + m0 := newTestModel() + m1 := newTestModel() + + ar, aw := io.Pipe() + br, bw := io.Pipe() + eaw := &ErrPipe{PipeWriter: *aw, max: i, err: e} + ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e} + + c0 := NewConnection(c0ID, ar, ebw, m0, "name", true).(wireFormatConnection).next.(*rawConnection) + NewConnection(c1ID, br, eaw, m1, "name", true) + + res := c0.ping() + if (i < 8 || j < 8) && res { + t.Errorf("Unexpected ping success; i=%d, j=%d", i, j) + } else if (i >= 12 && j >= 12) && !res { + t.Errorf("Unexpected ping fail; i=%d, j=%d", i, j) + } + } + } +} + +// func TestRequestResponseErr(t *testing.T) { +// e := errors.New("something broke") + +// var pass bool +// for i := 0; i < 48; i++ { +// for j := 0; j < 38; j++ { +// m0 := newTestModel() +// m0.data = []byte("response data") +// m1 := newTestModel() + +// ar, aw := io.Pipe() +// br, bw := io.Pipe() +// eaw := &ErrPipe{PipeWriter: *aw, max: i, err: e} +// ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e} + +// NewConnection(c0ID, ar, ebw, m0, nil) +// c1 := NewConnection(c1ID, br, eaw, m1, nil).(wireFormatConnection).next.(*rawConnection) + +// d, err := c1.Request("default", "tn", 1234, 5678) +// if err == e || err == ErrClosed { +// t.Logf("Error at %d+%d bytes", i, j) +// if !m1.isClosed() { +// t.Fatal("c1 not closed") +// } +// if !m0.isClosed() { +// t.Fatal("c0 not closed") +// } +// continue +// } +// if err != nil { +// t.Fatal(err) +// } +// if string(d) != "response data" { +// t.Fatalf("Incorrect response data %q", string(d)) +// } +// if m0.repo != "default" { +// t.Fatalf("Incorrect repo %q", m0.repo) +// } +// if m0.name != "tn" { +// t.Fatalf("Incorrect name %q", m0.name) +// } +// if m0.offset != 1234 { +// t.Fatalf("Incorrect offset %d", m0.offset) +// } +// if m0.size != 5678 { +// t.Fatalf("Incorrect size %d", m0.size) +// } +// t.Logf("Pass at %d+%d bytes", i, j) +// pass = true +// } +// } +// if !pass { +// t.Fatal("Never passed") +// } +// } + +func TestVersionErr(t *testing.T) { + m0 := newTestModel() + m1 := newTestModel() + + ar, aw := io.Pipe() + br, bw := io.Pipe() + + c0 := NewConnection(c0ID, ar, bw, m0, "name", true).(wireFormatConnection).next.(*rawConnection) + NewConnection(c1ID, br, aw, m1, "name", true) + + w := xdr.NewWriter(c0.cw) + w.WriteUint32(encodeHeader(header{ + version: 2, + msgID: 0, + msgType: 0, + })) + w.WriteUint32(0) + + if !m1.isClosed() { + t.Error("Connection should close due to unknown version") + } +} + +func TestTypeErr(t *testing.T) { + m0 := newTestModel() + m1 := newTestModel() + + ar, aw := io.Pipe() + br, bw := io.Pipe() + + c0 := NewConnection(c0ID, ar, bw, m0, "name", true).(wireFormatConnection).next.(*rawConnection) + NewConnection(c1ID, br, aw, m1, "name", true) + + w := xdr.NewWriter(c0.cw) + w.WriteUint32(encodeHeader(header{ + version: 0, + msgID: 0, + msgType: 42, + })) + w.WriteUint32(0) + + if !m1.isClosed() { + t.Error("Connection should close due to unknown message type") + } +} + +func TestClose(t *testing.T) { + m0 := newTestModel() + m1 := newTestModel() + + ar, aw := io.Pipe() + br, bw := io.Pipe() + + c0 := NewConnection(c0ID, ar, bw, m0, "name", true).(wireFormatConnection).next.(*rawConnection) + NewConnection(c1ID, br, aw, m1, "name", true) + + c0.close(nil) + + <-c0.closed + if !m0.isClosed() { + t.Fatal("Connection should be closed") + } + + // None of these should panic, some should return an error + + if c0.ping() { + t.Error("Ping should not return true") + } + + c0.Index("default", nil) + c0.Index("default", nil) + + if _, err := c0.Request("default", "foo", 0, 0); err == nil { + t.Error("Request should return an error") + } +} + +func TestElementSizeExceededNested(t *testing.T) { + m := ClusterConfigMessage{ + Repositories: []Repository{ + {ID: "longstringlongstringlongstringinglongstringlongstringlonlongstringlongstringlon"}, + }, + } + _, err := m.EncodeXDR(ioutil.Discard) + if err == nil { + t.Errorf("ID length %d > max 64, but no error", len(m.Repositories[0].ID)) + } +} + +func TestMarshalIndexMessage(t *testing.T) { + var quickCfg = &quick.Config{MaxCountScale: 10} + if testing.Short() { + quickCfg = nil + } + + f := func(m1 IndexMessage) bool { + for _, f := range m1.Files { + for i := range f.Blocks { + f.Blocks[i].Offset = 0 + if len(f.Blocks[i].Hash) == 0 { + f.Blocks[i].Hash = nil + } + } + } + + return testMarshal(t, "index", &m1, &IndexMessage{}) + } + + if err := quick.Check(f, quickCfg); err != nil { + t.Error(err) + } +} + +func TestMarshalRequestMessage(t *testing.T) { + var quickCfg = &quick.Config{MaxCountScale: 10} + if testing.Short() { + quickCfg = nil + } + + f := func(m1 RequestMessage) bool { + return testMarshal(t, "request", &m1, &RequestMessage{}) + } + + if err := quick.Check(f, quickCfg); err != nil { + t.Error(err) + } +} + +func TestMarshalResponseMessage(t *testing.T) { + var quickCfg = &quick.Config{MaxCountScale: 10} + if testing.Short() { + quickCfg = nil + } + + f := func(m1 ResponseMessage) bool { + if len(m1.Data) == 0 { + m1.Data = nil + } + return testMarshal(t, "response", &m1, &ResponseMessage{}) + } + + if err := quick.Check(f, quickCfg); err != nil { + t.Error(err) + } +} + +func TestMarshalClusterConfigMessage(t *testing.T) { + var quickCfg = &quick.Config{MaxCountScale: 10} + if testing.Short() { + quickCfg = nil + } + + f := func(m1 ClusterConfigMessage) bool { + return testMarshal(t, "clusterconfig", &m1, &ClusterConfigMessage{}) + } + + if err := quick.Check(f, quickCfg); err != nil { + t.Error(err) + } +} + +func TestMarshalCloseMessage(t *testing.T) { + var quickCfg = &quick.Config{MaxCountScale: 10} + if testing.Short() { + quickCfg = nil + } + + f := func(m1 CloseMessage) bool { + return testMarshal(t, "close", &m1, &CloseMessage{}) + } + + if err := quick.Check(f, quickCfg); err != nil { + t.Error(err) + } +} + +type message interface { + EncodeXDR(io.Writer) (int, error) + DecodeXDR(io.Reader) error +} + +func testMarshal(t *testing.T, prefix string, m1, m2 message) bool { + var buf bytes.Buffer + + failed := func(bc []byte) { + bs, _ := json.MarshalIndent(m1, "", " ") + ioutil.WriteFile(prefix+"-1.txt", bs, 0644) + bs, _ = json.MarshalIndent(m2, "", " ") + ioutil.WriteFile(prefix+"-2.txt", bs, 0644) + if len(bc) > 0 { + f, _ := os.Create(prefix + "-data.txt") + fmt.Fprint(f, hex.Dump(bc)) + f.Close() + } + } + + _, err := m1.EncodeXDR(&buf) + if err == xdr.ErrElementSizeExceeded { + return true + } + if err != nil { + failed(nil) + t.Fatal(err) + } + + bc := make([]byte, len(buf.Bytes())) + copy(bc, buf.Bytes()) + + err = m2.DecodeXDR(&buf) + if err != nil { + failed(bc) + t.Fatal(err) + } + + ok := reflect.DeepEqual(m1, m2) + if !ok { + failed(bc) + } + return ok +} diff --git a/wireformat.go b/wireformat.go new file mode 100644 index 000000000..987c03eff --- /dev/null +++ b/wireformat.go @@ -0,0 +1,58 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package protocol + +import ( + "path/filepath" + + "code.google.com/p/go.text/unicode/norm" +) + +type wireFormatConnection struct { + next Connection +} + +func (c wireFormatConnection) ID() NodeID { + return c.next.ID() +} + +func (c wireFormatConnection) Name() string { + return c.next.Name() +} + +func (c wireFormatConnection) Index(repo string, fs []FileInfo) error { + var myFs = make([]FileInfo, len(fs)) + copy(myFs, fs) + + for i := range fs { + myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name)) + } + + return c.next.Index(repo, myFs) +} + +func (c wireFormatConnection) IndexUpdate(repo string, fs []FileInfo) error { + var myFs = make([]FileInfo, len(fs)) + copy(myFs, fs) + + for i := range fs { + myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name)) + } + + return c.next.IndexUpdate(repo, myFs) +} + +func (c wireFormatConnection) Request(repo, name string, offset int64, size int) ([]byte, error) { + name = norm.NFC.String(filepath.ToSlash(name)) + return c.next.Request(repo, name, offset, size) +} + +func (c wireFormatConnection) ClusterConfig(config ClusterConfigMessage) { + c.next.ClusterConfig(config) +} + +func (c wireFormatConnection) Statistics() Statistics { + return c.next.Statistics() +} From 4b488a2d28ea2bdff814ac9a4e6be3b26475dab1 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Sun, 28 Sep 2014 12:00:38 +0100 Subject: [PATCH 02/61] Rename Repository -> Folder, Node -> Device (fixes #739) --- common_test.go | 14 ++-- nodeid.go => deviceid.go | 36 +++++----- nodeid_test.go => deviceid_test.go | 20 +++--- message.go | 12 ++-- message_xdr.go | 108 ++++++++++++++--------------- nativemodel_darwin.go | 20 +++--- nativemodel_unix.go | 20 +++--- nativemodel_windows.go | 20 +++--- protocol.go | 60 ++++++++-------- protocol_test.go | 12 ++-- wireformat.go | 14 ++-- 11 files changed, 168 insertions(+), 168 deletions(-) rename nodeid.go => deviceid.go (74%) rename nodeid_test.go => deviceid_test.go (82%) diff --git a/common_test.go b/common_test.go index 9d387825c..8d62c8ee1 100644 --- a/common_test.go +++ b/common_test.go @@ -11,7 +11,7 @@ import ( type TestModel struct { data []byte - repo string + folder string name string offset int64 size int @@ -24,25 +24,25 @@ func newTestModel() *TestModel { } } -func (t *TestModel) Index(nodeID NodeID, repo string, files []FileInfo) { +func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo) { } -func (t *TestModel) IndexUpdate(nodeID NodeID, repo string, files []FileInfo) { +func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) { } -func (t *TestModel) Request(nodeID NodeID, repo, name string, offset int64, size int) ([]byte, error) { - t.repo = repo +func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, size int) ([]byte, error) { + t.folder = folder t.name = name t.offset = offset t.size = size return t.data, nil } -func (t *TestModel) Close(nodeID NodeID, err error) { +func (t *TestModel) Close(deviceID DeviceID, err error) { close(t.closedCh) } -func (t *TestModel) ClusterConfig(nodeID NodeID, config ClusterConfigMessage) { +func (t *TestModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { } func (t *TestModel) isClosed() bool { diff --git a/nodeid.go b/deviceid.go similarity index 74% rename from nodeid.go rename to deviceid.go index 9079781b9..f4427dc6f 100644 --- a/nodeid.go +++ b/deviceid.go @@ -16,36 +16,36 @@ import ( "github.com/syncthing/syncthing/internal/luhn" ) -type NodeID [32]byte +type DeviceID [32]byte -var LocalNodeID = NodeID{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} +var LocalDeviceID = DeviceID{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} -// NewNodeID generates a new node ID from the raw bytes of a certificate -func NewNodeID(rawCert []byte) NodeID { - var n NodeID +// NewDeviceID generates a new device ID from the raw bytes of a certificate +func NewDeviceID(rawCert []byte) DeviceID { + var n DeviceID hf := sha256.New() hf.Write(rawCert) hf.Sum(n[:0]) return n } -func NodeIDFromString(s string) (NodeID, error) { - var n NodeID +func DeviceIDFromString(s string) (DeviceID, error) { + var n DeviceID err := n.UnmarshalText([]byte(s)) return n, err } -func NodeIDFromBytes(bs []byte) NodeID { - var n NodeID +func DeviceIDFromBytes(bs []byte) DeviceID { + var n DeviceID if len(bs) != len(n) { - panic("incorrect length of byte slice representing node ID") + panic("incorrect length of byte slice representing device ID") } copy(n[:], bs) return n } -// String returns the canonical string representation of the node ID -func (n NodeID) String() string { +// String returns the canonical string representation of the device ID +func (n DeviceID) String() string { id := base32.StdEncoding.EncodeToString(n[:]) id = strings.Trim(id, "=") id, err := luhnify(id) @@ -57,23 +57,23 @@ func (n NodeID) String() string { return id } -func (n NodeID) GoString() string { +func (n DeviceID) GoString() string { return n.String() } -func (n NodeID) Compare(other NodeID) int { +func (n DeviceID) Compare(other DeviceID) int { return bytes.Compare(n[:], other[:]) } -func (n NodeID) Equals(other NodeID) bool { +func (n DeviceID) Equals(other DeviceID) bool { return bytes.Compare(n[:], other[:]) == 0 } -func (n *NodeID) MarshalText() ([]byte, error) { +func (n *DeviceID) MarshalText() ([]byte, error) { return []byte(n.String()), nil } -func (n *NodeID) UnmarshalText(bs []byte) error { +func (n *DeviceID) UnmarshalText(bs []byte) error { id := string(bs) id = strings.Trim(id, "=") id = strings.ToUpper(id) @@ -98,7 +98,7 @@ func (n *NodeID) UnmarshalText(bs []byte) error { copy(n[:], dec) return nil default: - return errors.New("node ID invalid: incorrect length") + return errors.New("device ID invalid: incorrect length") } } diff --git a/nodeid_test.go b/deviceid_test.go similarity index 82% rename from nodeid_test.go rename to deviceid_test.go index 5b861b6de..069e4b988 100644 --- a/nodeid_test.go +++ b/deviceid_test.go @@ -20,14 +20,14 @@ var formatCases = []string{ "p561017mzjnu2yiqgdreydm2mgtimgl3bxnpq6w5bmt88z4tjxzwicq2", } -func TestFormatNodeID(t *testing.T) { +func TestFormatDeviceID(t *testing.T) { for i, tc := range formatCases { - var id NodeID + var id DeviceID err := id.UnmarshalText([]byte(tc)) if err != nil { t.Errorf("#%d UnmarshalText(%q); %v", i, tc, err) } else if f := id.String(); f != formatted { - t.Errorf("#%d FormatNodeID(%q)\n\t%q !=\n\t%q", i, tc, f, formatted) + t.Errorf("#%d FormatDeviceID(%q)\n\t%q !=\n\t%q", i, tc, f, formatted) } } } @@ -46,20 +46,20 @@ var validateCases = []struct { {"p56ioi7mzjnu2iqgdreydm2mgtmgl3bxnpq6w5btbbz4tjxzwicqCCCC", false}, } -func TestValidateNodeID(t *testing.T) { +func TestValidateDeviceID(t *testing.T) { for _, tc := range validateCases { - var id NodeID + var id DeviceID err := id.UnmarshalText([]byte(tc.s)) if (err == nil && !tc.ok) || (err != nil && tc.ok) { - t.Errorf("ValidateNodeID(%q); %v != %v", tc.s, err, tc.ok) + t.Errorf("ValidateDeviceID(%q); %v != %v", tc.s, err, tc.ok) } } } -func TestMarshallingNodeID(t *testing.T) { - n0 := NodeID{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 10, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32} - n1 := NodeID{} - n2 := NodeID{} +func TestMarshallingDeviceID(t *testing.T) { + n0 := DeviceID{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 10, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32} + n1 := DeviceID{} + n2 := DeviceID{} bs, _ := n0.MarshalText() n1.UnmarshalText(bs) diff --git a/message.go b/message.go index 779817a7b..09cb323c8 100644 --- a/message.go +++ b/message.go @@ -7,7 +7,7 @@ package protocol import "fmt" type IndexMessage struct { - Repository string // max:64 + Folder string // max:64 Files []FileInfo } @@ -90,7 +90,7 @@ func (b BlockInfo) String() string { } type RequestMessage struct { - Repository string // max:64 + Folder string // max:64 Name string // max:8192 Offset uint64 Size uint32 @@ -103,7 +103,7 @@ type ResponseMessage struct { type ClusterConfigMessage struct { ClientName string // max:64 ClientVersion string // max:64 - Repositories []Repository // max:64 + Folders []Folder // max:64 Options []Option // max:64 } @@ -116,12 +116,12 @@ func (o *ClusterConfigMessage) GetOption(key string) string { return "" } -type Repository struct { +type Folder struct { ID string // max:64 - Nodes []Node // max:64 + Devices []Device // max:64 } -type Node struct { +type Device struct { ID []byte // max:32 Flags uint32 MaxLocalVersion uint64 diff --git a/message_xdr.go b/message_xdr.go index 11a11da34..c7f16a173 100644 --- a/message_xdr.go +++ b/message_xdr.go @@ -18,10 +18,10 @@ IndexMessage Structure: 0 1 2 3 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Length of Repository | +| Length of Folder | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ / / -\ Repository (variable length) \ +\ Folder (variable length) \ / / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | Number of Files | @@ -33,7 +33,7 @@ IndexMessage Structure: struct IndexMessage { - string Repository<64>; + string Folder<64>; FileInfo Files<>; } @@ -56,10 +56,10 @@ func (o IndexMessage) AppendXDR(bs []byte) []byte { } func (o IndexMessage) encodeXDR(xw *xdr.Writer) (int, error) { - if len(o.Repository) > 64 { + if len(o.Folder) > 64 { return xw.Tot(), xdr.ErrElementSizeExceeded } - xw.WriteString(o.Repository) + xw.WriteString(o.Folder) xw.WriteUint32(uint32(len(o.Files))) for i := range o.Files { _, err := o.Files[i].encodeXDR(xw) @@ -82,7 +82,7 @@ func (o *IndexMessage) UnmarshalXDR(bs []byte) error { } func (o *IndexMessage) decodeXDR(xr *xdr.Reader) error { - o.Repository = xr.ReadStringMax(64) + o.Folder = xr.ReadStringMax(64) _FilesSize := int(xr.ReadUint32()) o.Files = make([]FileInfo, _FilesSize) for i := range o.Files { @@ -362,10 +362,10 @@ RequestMessage Structure: 0 1 2 3 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Length of Repository | +| Length of Folder | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ / / -\ Repository (variable length) \ +\ Folder (variable length) \ / / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | Length of Name | @@ -383,7 +383,7 @@ RequestMessage Structure: struct RequestMessage { - string Repository<64>; + string Folder<64>; string Name<8192>; unsigned hyper Offset; unsigned int Size; @@ -408,10 +408,10 @@ func (o RequestMessage) AppendXDR(bs []byte) []byte { } func (o RequestMessage) encodeXDR(xw *xdr.Writer) (int, error) { - if len(o.Repository) > 64 { + if len(o.Folder) > 64 { return xw.Tot(), xdr.ErrElementSizeExceeded } - xw.WriteString(o.Repository) + xw.WriteString(o.Folder) if len(o.Name) > 8192 { return xw.Tot(), xdr.ErrElementSizeExceeded } @@ -433,7 +433,7 @@ func (o *RequestMessage) UnmarshalXDR(bs []byte) error { } func (o *RequestMessage) decodeXDR(xr *xdr.Reader) error { - o.Repository = xr.ReadStringMax(64) + o.Folder = xr.ReadStringMax(64) o.Name = xr.ReadStringMax(8192) o.Offset = xr.ReadUint64() o.Size = xr.ReadUint32() @@ -517,10 +517,10 @@ ClusterConfigMessage Structure: \ Client Version (variable length) \ / / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Number of Repositories | +| Number of Folders | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ / / -\ Zero or more Repository Structures \ +\ Zero or more Folder Structures \ / / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | Number of Options | @@ -534,7 +534,7 @@ ClusterConfigMessage Structure: struct ClusterConfigMessage { string ClientName<64>; string ClientVersion<64>; - Repository Repositories<64>; + Folder Folders<64>; Option Options<64>; } @@ -565,12 +565,12 @@ func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) { return xw.Tot(), xdr.ErrElementSizeExceeded } xw.WriteString(o.ClientVersion) - if len(o.Repositories) > 64 { + if len(o.Folders) > 64 { return xw.Tot(), xdr.ErrElementSizeExceeded } - xw.WriteUint32(uint32(len(o.Repositories))) - for i := range o.Repositories { - _, err := o.Repositories[i].encodeXDR(xw) + xw.WriteUint32(uint32(len(o.Folders))) + for i := range o.Folders { + _, err := o.Folders[i].encodeXDR(xw) if err != nil { return xw.Tot(), err } @@ -602,13 +602,13 @@ func (o *ClusterConfigMessage) UnmarshalXDR(bs []byte) error { func (o *ClusterConfigMessage) decodeXDR(xr *xdr.Reader) error { o.ClientName = xr.ReadStringMax(64) o.ClientVersion = xr.ReadStringMax(64) - _RepositoriesSize := int(xr.ReadUint32()) - if _RepositoriesSize > 64 { + _FoldersSize := int(xr.ReadUint32()) + if _FoldersSize > 64 { return xdr.ErrElementSizeExceeded } - o.Repositories = make([]Repository, _RepositoriesSize) - for i := range o.Repositories { - (&o.Repositories[i]).decodeXDR(xr) + o.Folders = make([]Folder, _FoldersSize) + for i := range o.Folders { + (&o.Folders[i]).decodeXDR(xr) } _OptionsSize := int(xr.ReadUint32()) if _OptionsSize > 64 { @@ -623,7 +623,7 @@ func (o *ClusterConfigMessage) decodeXDR(xr *xdr.Reader) error { /* -Repository Structure: +Folder Structure: 0 1 2 3 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 @@ -634,48 +634,48 @@ Repository Structure: \ ID (variable length) \ / / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Number of Nodes | +| Number of Devices | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ / / -\ Zero or more Node Structures \ +\ Zero or more Device Structures \ / / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -struct Repository { +struct Folder { string ID<64>; - Node Nodes<64>; + Device Devices<64>; } */ -func (o Repository) EncodeXDR(w io.Writer) (int, error) { +func (o Folder) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) return o.encodeXDR(xw) } -func (o Repository) MarshalXDR() []byte { +func (o Folder) MarshalXDR() []byte { return o.AppendXDR(make([]byte, 0, 128)) } -func (o Repository) AppendXDR(bs []byte) []byte { +func (o Folder) AppendXDR(bs []byte) []byte { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) o.encodeXDR(xw) return []byte(aw) } -func (o Repository) encodeXDR(xw *xdr.Writer) (int, error) { +func (o Folder) encodeXDR(xw *xdr.Writer) (int, error) { if len(o.ID) > 64 { return xw.Tot(), xdr.ErrElementSizeExceeded } xw.WriteString(o.ID) - if len(o.Nodes) > 64 { + if len(o.Devices) > 64 { return xw.Tot(), xdr.ErrElementSizeExceeded } - xw.WriteUint32(uint32(len(o.Nodes))) - for i := range o.Nodes { - _, err := o.Nodes[i].encodeXDR(xw) + xw.WriteUint32(uint32(len(o.Devices))) + for i := range o.Devices { + _, err := o.Devices[i].encodeXDR(xw) if err != nil { return xw.Tot(), err } @@ -683,33 +683,33 @@ func (o Repository) encodeXDR(xw *xdr.Writer) (int, error) { return xw.Tot(), xw.Error() } -func (o *Repository) DecodeXDR(r io.Reader) error { +func (o *Folder) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) return o.decodeXDR(xr) } -func (o *Repository) UnmarshalXDR(bs []byte) error { +func (o *Folder) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) return o.decodeXDR(xr) } -func (o *Repository) decodeXDR(xr *xdr.Reader) error { +func (o *Folder) decodeXDR(xr *xdr.Reader) error { o.ID = xr.ReadStringMax(64) - _NodesSize := int(xr.ReadUint32()) - if _NodesSize > 64 { + _DevicesSize := int(xr.ReadUint32()) + if _DevicesSize > 64 { return xdr.ErrElementSizeExceeded } - o.Nodes = make([]Node, _NodesSize) - for i := range o.Nodes { - (&o.Nodes[i]).decodeXDR(xr) + o.Devices = make([]Device, _DevicesSize) + for i := range o.Devices { + (&o.Devices[i]).decodeXDR(xr) } return xr.Error() } /* -Node Structure: +Device Structure: 0 1 2 3 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 @@ -728,7 +728,7 @@ Node Structure: +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -struct Node { +struct Device { opaque ID<32>; unsigned int Flags; unsigned hyper MaxLocalVersion; @@ -736,23 +736,23 @@ struct Node { */ -func (o Node) EncodeXDR(w io.Writer) (int, error) { +func (o Device) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) return o.encodeXDR(xw) } -func (o Node) MarshalXDR() []byte { +func (o Device) MarshalXDR() []byte { return o.AppendXDR(make([]byte, 0, 128)) } -func (o Node) AppendXDR(bs []byte) []byte { +func (o Device) AppendXDR(bs []byte) []byte { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) o.encodeXDR(xw) return []byte(aw) } -func (o Node) encodeXDR(xw *xdr.Writer) (int, error) { +func (o Device) encodeXDR(xw *xdr.Writer) (int, error) { if len(o.ID) > 32 { return xw.Tot(), xdr.ErrElementSizeExceeded } @@ -762,18 +762,18 @@ func (o Node) encodeXDR(xw *xdr.Writer) (int, error) { return xw.Tot(), xw.Error() } -func (o *Node) DecodeXDR(r io.Reader) error { +func (o *Device) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) return o.decodeXDR(xr) } -func (o *Node) UnmarshalXDR(bs []byte) error { +func (o *Device) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) return o.decodeXDR(xr) } -func (o *Node) decodeXDR(xr *xdr.Reader) error { +func (o *Device) decodeXDR(xr *xdr.Reader) error { o.ID = xr.ReadBytesMax(32) o.Flags = xr.ReadUint32() o.MaxLocalVersion = xr.ReadUint64() diff --git a/nativemodel_darwin.go b/nativemodel_darwin.go index 9ac402fe6..8e8c7a421 100644 --- a/nativemodel_darwin.go +++ b/nativemodel_darwin.go @@ -14,29 +14,29 @@ type nativeModel struct { next Model } -func (m nativeModel) Index(nodeID NodeID, repo string, files []FileInfo) { +func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) { for i := range files { files[i].Name = norm.NFD.String(files[i].Name) } - m.next.Index(nodeID, repo, files) + m.next.Index(deviceID, folder, files) } -func (m nativeModel) IndexUpdate(nodeID NodeID, repo string, files []FileInfo) { +func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) { for i := range files { files[i].Name = norm.NFD.String(files[i].Name) } - m.next.IndexUpdate(nodeID, repo, files) + m.next.IndexUpdate(deviceID, folder, files) } -func (m nativeModel) Request(nodeID NodeID, repo string, name string, offset int64, size int) ([]byte, error) { +func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error) { name = norm.NFD.String(name) - return m.next.Request(nodeID, repo, name, offset, size) + return m.next.Request(deviceID, folder, name, offset, size) } -func (m nativeModel) ClusterConfig(nodeID NodeID, config ClusterConfigMessage) { - m.next.ClusterConfig(nodeID, config) +func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { + m.next.ClusterConfig(deviceID, config) } -func (m nativeModel) Close(nodeID NodeID, err error) { - m.next.Close(nodeID, err) +func (m nativeModel) Close(deviceID DeviceID, err error) { + m.next.Close(deviceID, err) } diff --git a/nativemodel_unix.go b/nativemodel_unix.go index 23fbe0b6b..77d335fae 100644 --- a/nativemodel_unix.go +++ b/nativemodel_unix.go @@ -12,22 +12,22 @@ type nativeModel struct { next Model } -func (m nativeModel) Index(nodeID NodeID, repo string, files []FileInfo) { - m.next.Index(nodeID, repo, files) +func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) { + m.next.Index(deviceID, folder, files) } -func (m nativeModel) IndexUpdate(nodeID NodeID, repo string, files []FileInfo) { - m.next.IndexUpdate(nodeID, repo, files) +func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) { + m.next.IndexUpdate(deviceID, folder, files) } -func (m nativeModel) Request(nodeID NodeID, repo string, name string, offset int64, size int) ([]byte, error) { - return m.next.Request(nodeID, repo, name, offset, size) +func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error) { + return m.next.Request(deviceID, folder, name, offset, size) } -func (m nativeModel) ClusterConfig(nodeID NodeID, config ClusterConfigMessage) { - m.next.ClusterConfig(nodeID, config) +func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { + m.next.ClusterConfig(deviceID, config) } -func (m nativeModel) Close(nodeID NodeID, err error) { - m.next.Close(nodeID, err) +func (m nativeModel) Close(deviceID DeviceID, err error) { + m.next.Close(deviceID, err) } diff --git a/nativemodel_windows.go b/nativemodel_windows.go index 9841d63f7..252860ea5 100644 --- a/nativemodel_windows.go +++ b/nativemodel_windows.go @@ -26,7 +26,7 @@ type nativeModel struct { next Model } -func (m nativeModel) Index(nodeID NodeID, repo string, files []FileInfo) { +func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) { for i, f := range files { if strings.ContainsAny(f.Name, disallowedCharacters) { if f.IsDeleted() { @@ -39,10 +39,10 @@ func (m nativeModel) Index(nodeID NodeID, repo string, files []FileInfo) { } files[i].Name = filepath.FromSlash(f.Name) } - m.next.Index(nodeID, repo, files) + m.next.Index(deviceID, folder, files) } -func (m nativeModel) IndexUpdate(nodeID NodeID, repo string, files []FileInfo) { +func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) { for i, f := range files { if strings.ContainsAny(f.Name, disallowedCharacters) { if f.IsDeleted() { @@ -55,18 +55,18 @@ func (m nativeModel) IndexUpdate(nodeID NodeID, repo string, files []FileInfo) { } files[i].Name = filepath.FromSlash(files[i].Name) } - m.next.IndexUpdate(nodeID, repo, files) + m.next.IndexUpdate(deviceID, folder, files) } -func (m nativeModel) Request(nodeID NodeID, repo string, name string, offset int64, size int) ([]byte, error) { +func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error) { name = filepath.FromSlash(name) - return m.next.Request(nodeID, repo, name, offset, size) + return m.next.Request(deviceID, folder, name, offset, size) } -func (m nativeModel) ClusterConfig(nodeID NodeID, config ClusterConfigMessage) { - m.next.ClusterConfig(nodeID, config) +func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { + m.next.ClusterConfig(deviceID, config) } -func (m nativeModel) Close(nodeID NodeID, err error) { - m.next.Close(nodeID, err) +func (m nativeModel) Close(deviceID DeviceID, err error) { + m.next.Close(deviceID, err) } diff --git a/protocol.go b/protocol.go index 86fd6199f..26d75de4f 100644 --- a/protocol.go +++ b/protocol.go @@ -57,30 +57,30 @@ var ( ) type Model interface { - // An index was received from the peer node - Index(nodeID NodeID, repo string, files []FileInfo) - // An index update was received from the peer node - IndexUpdate(nodeID NodeID, repo string, files []FileInfo) - // A request was made by the peer node - Request(nodeID NodeID, repo string, name string, offset int64, size int) ([]byte, error) + // An index was received from the peer device + Index(deviceID DeviceID, folder string, files []FileInfo) + // An index update was received from the peer device + IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) + // A request was made by the peer device + Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error) // A cluster configuration message was received - ClusterConfig(nodeID NodeID, config ClusterConfigMessage) - // The peer node closed the connection - Close(nodeID NodeID, err error) + ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) + // The peer device closed the connection + Close(deviceID DeviceID, err error) } type Connection interface { - ID() NodeID + ID() DeviceID Name() string - Index(repo string, files []FileInfo) error - IndexUpdate(repo string, files []FileInfo) error - Request(repo string, name string, offset int64, size int) ([]byte, error) + Index(folder string, files []FileInfo) error + IndexUpdate(folder string, files []FileInfo) error + Request(folder string, name string, offset int64, size int) ([]byte, error) ClusterConfig(config ClusterConfigMessage) Statistics() Statistics } type rawConnection struct { - id NodeID + id DeviceID name string receiver Model state int @@ -123,7 +123,7 @@ const ( pingIdleTime = 60 * time.Second ) -func NewConnection(nodeID NodeID, reader io.Reader, writer io.Writer, receiver Model, name string, compress bool) Connection { +func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiver Model, name string, compress bool) Connection { cr := &countingReader{Reader: reader} cw := &countingWriter{Writer: writer} @@ -132,7 +132,7 @@ func NewConnection(nodeID NodeID, reader io.Reader, writer io.Writer, receiver M compThres = 128 // compress messages that are 128 bytes long or larger } c := rawConnection{ - id: nodeID, + id: deviceID, name: name, receiver: nativeModel{receiver}, state: stateInitial, @@ -152,7 +152,7 @@ func NewConnection(nodeID NodeID, reader io.Reader, writer io.Writer, receiver M return wireFormatConnection{&c} } -func (c *rawConnection) ID() NodeID { +func (c *rawConnection) ID() DeviceID { return c.id } @@ -160,34 +160,34 @@ func (c *rawConnection) Name() string { return c.name } -// Index writes the list of file information to the connected peer node -func (c *rawConnection) Index(repo string, idx []FileInfo) error { +// Index writes the list of file information to the connected peer device +func (c *rawConnection) Index(folder string, idx []FileInfo) error { select { case <-c.closed: return ErrClosed default: } c.idxMut.Lock() - c.send(-1, messageTypeIndex, IndexMessage{repo, idx}) + c.send(-1, messageTypeIndex, IndexMessage{folder, idx}) c.idxMut.Unlock() return nil } -// IndexUpdate writes the list of file information to the connected peer node as an update -func (c *rawConnection) IndexUpdate(repo string, idx []FileInfo) error { +// IndexUpdate writes the list of file information to the connected peer device as an update +func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo) error { select { case <-c.closed: return ErrClosed default: } c.idxMut.Lock() - c.send(-1, messageTypeIndexUpdate, IndexMessage{repo, idx}) + c.send(-1, messageTypeIndexUpdate, IndexMessage{folder, idx}) c.idxMut.Unlock() return nil } // Request returns the bytes for the specified block after fetching them from the connected peer. -func (c *rawConnection) Request(repo string, name string, offset int64, size int) ([]byte, error) { +func (c *rawConnection) Request(folder string, name string, offset int64, size int) ([]byte, error) { var id int select { case id = <-c.nextID: @@ -203,7 +203,7 @@ func (c *rawConnection) Request(repo string, name string, offset int64, size int c.awaiting[id] = rc c.awaitingMut.Unlock() - ok := c.send(id, messageTypeRequest, RequestMessage{repo, name, uint64(offset), uint32(size)}) + ok := c.send(id, messageTypeRequest, RequestMessage{folder, name, uint64(offset), uint32(size)}) if !ok { return nil, ErrClosed } @@ -399,20 +399,20 @@ func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) { func (c *rawConnection) handleIndex(im IndexMessage) { if debug { - l.Debugf("Index(%v, %v, %d files)", c.id, im.Repository, len(im.Files)) + l.Debugf("Index(%v, %v, %d files)", c.id, im.Folder, len(im.Files)) } - c.receiver.Index(c.id, im.Repository, im.Files) + c.receiver.Index(c.id, im.Folder, im.Files) } func (c *rawConnection) handleIndexUpdate(im IndexMessage) { if debug { - l.Debugf("queueing IndexUpdate(%v, %v, %d files)", c.id, im.Repository, len(im.Files)) + l.Debugf("queueing IndexUpdate(%v, %v, %d files)", c.id, im.Folder, len(im.Files)) } - c.receiver.IndexUpdate(c.id, im.Repository, im.Files) + c.receiver.IndexUpdate(c.id, im.Folder, im.Files) } func (c *rawConnection) handleRequest(msgID int, req RequestMessage) { - data, _ := c.receiver.Request(c.id, req.Repository, req.Name, int64(req.Offset), int(req.Size)) + data, _ := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), int(req.Size)) c.send(msgID, messageTypeResponse, ResponseMessage{data}) } diff --git a/protocol_test.go b/protocol_test.go index 56a46f2ec..ee8587586 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -21,8 +21,8 @@ import ( ) var ( - c0ID = NewNodeID([]byte{1}) - c1ID = NewNodeID([]byte{2}) + c0ID = NewDeviceID([]byte{1}) + c1ID = NewDeviceID([]byte{2}) ) func TestHeaderFunctions(t *testing.T) { @@ -140,8 +140,8 @@ func TestPingErr(t *testing.T) { // if string(d) != "response data" { // t.Fatalf("Incorrect response data %q", string(d)) // } -// if m0.repo != "default" { -// t.Fatalf("Incorrect repo %q", m0.repo) +// if m0.folder != "default" { +// t.Fatalf("Incorrect folder %q", m0.folder) // } // if m0.name != "tn" { // t.Fatalf("Incorrect name %q", m0.name) @@ -240,13 +240,13 @@ func TestClose(t *testing.T) { func TestElementSizeExceededNested(t *testing.T) { m := ClusterConfigMessage{ - Repositories: []Repository{ + Folders: []Folder{ {ID: "longstringlongstringlongstringinglongstringlongstringlonlongstringlongstringlon"}, }, } _, err := m.EncodeXDR(ioutil.Discard) if err == nil { - t.Errorf("ID length %d > max 64, but no error", len(m.Repositories[0].ID)) + t.Errorf("ID length %d > max 64, but no error", len(m.Folders[0].ID)) } } diff --git a/wireformat.go b/wireformat.go index 987c03eff..ebd714cd2 100644 --- a/wireformat.go +++ b/wireformat.go @@ -14,7 +14,7 @@ type wireFormatConnection struct { next Connection } -func (c wireFormatConnection) ID() NodeID { +func (c wireFormatConnection) ID() DeviceID { return c.next.ID() } @@ -22,7 +22,7 @@ func (c wireFormatConnection) Name() string { return c.next.Name() } -func (c wireFormatConnection) Index(repo string, fs []FileInfo) error { +func (c wireFormatConnection) Index(folder string, fs []FileInfo) error { var myFs = make([]FileInfo, len(fs)) copy(myFs, fs) @@ -30,10 +30,10 @@ func (c wireFormatConnection) Index(repo string, fs []FileInfo) error { myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name)) } - return c.next.Index(repo, myFs) + return c.next.Index(folder, myFs) } -func (c wireFormatConnection) IndexUpdate(repo string, fs []FileInfo) error { +func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo) error { var myFs = make([]FileInfo, len(fs)) copy(myFs, fs) @@ -41,12 +41,12 @@ func (c wireFormatConnection) IndexUpdate(repo string, fs []FileInfo) error { myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name)) } - return c.next.IndexUpdate(repo, myFs) + return c.next.IndexUpdate(folder, myFs) } -func (c wireFormatConnection) Request(repo, name string, offset int64, size int) ([]byte, error) { +func (c wireFormatConnection) Request(folder, name string, offset int64, size int) ([]byte, error) { name = norm.NFC.String(filepath.ToSlash(name)) - return c.next.Request(repo, name, offset, size) + return c.next.Request(folder, name, offset, size) } func (c wireFormatConnection) ClusterConfig(config ClusterConfigMessage) { From 1bc5632771b9daec3c521ab3353256099da02772 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Sun, 28 Sep 2014 12:05:25 +0100 Subject: [PATCH 03/61] Run go fmt -w --- common_test.go | 2 +- message.go | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/common_test.go b/common_test.go index 8d62c8ee1..e337c7e5b 100644 --- a/common_test.go +++ b/common_test.go @@ -11,7 +11,7 @@ import ( type TestModel struct { data []byte - folder string + folder string name string offset int64 size int diff --git a/message.go b/message.go index 09cb323c8..a99f2d514 100644 --- a/message.go +++ b/message.go @@ -8,7 +8,7 @@ import "fmt" type IndexMessage struct { Folder string // max:64 - Files []FileInfo + Files []FileInfo } type FileInfo struct { @@ -91,9 +91,9 @@ func (b BlockInfo) String() string { type RequestMessage struct { Folder string // max:64 - Name string // max:8192 - Offset uint64 - Size uint32 + Name string // max:8192 + Offset uint64 + Size uint32 } type ResponseMessage struct { @@ -101,10 +101,10 @@ type ResponseMessage struct { } type ClusterConfigMessage struct { - ClientName string // max:64 - ClientVersion string // max:64 - Folders []Folder // max:64 - Options []Option // max:64 + ClientName string // max:64 + ClientVersion string // max:64 + Folders []Folder // max:64 + Options []Option // max:64 } func (o *ClusterConfigMessage) GetOption(key string) string { @@ -117,7 +117,7 @@ func (o *ClusterConfigMessage) GetOption(key string) string { } type Folder struct { - ID string // max:64 + ID string // max:64 Devices []Device // max:64 } From 43289103cbb14f918f64f301099d730980bc641c Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Mon, 29 Sep 2014 21:43:32 +0200 Subject: [PATCH 04/61] Relicense to GPL --- common_test.go | 15 +++++++++++++-- counting.go | 15 +++++++++++++-- debug.go | 15 +++++++++++++-- deviceid.go | 15 +++++++++++++-- deviceid_test.go | 15 +++++++++++++-- doc.go | 15 +++++++++++++-- header.go | 15 +++++++++++++-- message.go | 15 +++++++++++++-- nativemodel_darwin.go | 15 +++++++++++++-- nativemodel_unix.go | 15 +++++++++++++-- nativemodel_windows.go | 15 +++++++++++++-- protocol.go | 15 +++++++++++++-- protocol_test.go | 15 +++++++++++++-- wireformat.go | 15 +++++++++++++-- 14 files changed, 182 insertions(+), 28 deletions(-) diff --git a/common_test.go b/common_test.go index e337c7e5b..29b910bab 100644 --- a/common_test.go +++ b/common_test.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . package protocol diff --git a/counting.go b/counting.go index 512774fba..4f95c9f9a 100644 --- a/counting.go +++ b/counting.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . package protocol diff --git a/debug.go b/debug.go index 6c586b90e..930aaf344 100644 --- a/debug.go +++ b/debug.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . package protocol diff --git a/deviceid.go b/deviceid.go index f4427dc6f..4b034f3f1 100644 --- a/deviceid.go +++ b/deviceid.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . package protocol diff --git a/deviceid_test.go b/deviceid_test.go index 069e4b988..14e2dfa34 100644 --- a/deviceid_test.go +++ b/deviceid_test.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . package protocol diff --git a/doc.go b/doc.go index 8c6b524e6..95b5e102f 100644 --- a/doc.go +++ b/doc.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . // Package protocol implements the Block Exchange Protocol. package protocol diff --git a/header.go b/header.go index 6fd2ebb8f..96ccff66f 100644 --- a/header.go +++ b/header.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . package protocol diff --git a/message.go b/message.go index a99f2d514..feeba69df 100644 --- a/message.go +++ b/message.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . package protocol diff --git a/nativemodel_darwin.go b/nativemodel_darwin.go index 8e8c7a421..7fb5c964e 100644 --- a/nativemodel_darwin.go +++ b/nativemodel_darwin.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . // +build darwin diff --git a/nativemodel_unix.go b/nativemodel_unix.go index 77d335fae..61ad9c438 100644 --- a/nativemodel_unix.go +++ b/nativemodel_unix.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . // +build !windows,!darwin diff --git a/nativemodel_windows.go b/nativemodel_windows.go index 252860ea5..57d3f3446 100644 --- a/nativemodel_windows.go +++ b/nativemodel_windows.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . // +build windows diff --git a/protocol.go b/protocol.go index 26d75de4f..19cdfbb16 100644 --- a/protocol.go +++ b/protocol.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . package protocol diff --git a/protocol_test.go b/protocol_test.go index ee8587586..cc6f9472a 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . package protocol diff --git a/wireformat.go b/wireformat.go index ebd714cd2..f06b9454e 100644 --- a/wireformat.go +++ b/wireformat.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . package protocol From 1ef8378a3001dc64f0d174d39b1c745d6c897b1c Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Thu, 16 Oct 2014 09:26:20 +0200 Subject: [PATCH 05/61] FileInfoTruncated.String() for stindex' benefit --- message.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/message.go b/message.go index feeba69df..800787b91 100644 --- a/message.go +++ b/message.go @@ -64,6 +64,11 @@ type FileInfoTruncated struct { NumBlocks uint32 } +func (f FileInfoTruncated) String() string { + return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%d, Size:%d, NumBlocks:%d}", + f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.NumBlocks) +} + // Returns a statistical guess on the size, not the exact figure func (f FileInfoTruncated) Size() int64 { if IsDeleted(f.Flags) || IsDirectory(f.Flags) { From c618eba9a9c6466db70e2ce42247723b2ea6c778 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Mon, 6 Oct 2014 21:57:33 +0100 Subject: [PATCH 06/61] Implement BlockMap --- message.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/message.go b/message.go index 800787b91..b47145055 100644 --- a/message.go +++ b/message.go @@ -54,6 +54,10 @@ func (f FileInfo) IsInvalid() bool { return IsInvalid(f.Flags) } +func (f FileInfo) IsDirectory() bool { + return IsDirectory(f.Flags) +} + // Used for unmarshalling a FileInfo structure but skipping the actual block list type FileInfoTruncated struct { Name string // max:8192 From ec9d68960fe196cd53abd83243abe24bf57cb5b3 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Mon, 20 Oct 2014 21:45:26 +0100 Subject: [PATCH 07/61] Remove 64 device limit --- message.go | 4 ++-- message_xdr.go | 8 +------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/message.go b/message.go index b47145055..445ac7a75 100644 --- a/message.go +++ b/message.go @@ -137,8 +137,8 @@ func (o *ClusterConfigMessage) GetOption(key string) string { } type Folder struct { - ID string // max:64 - Devices []Device // max:64 + ID string // max:64 + Devices []Device } type Device struct { diff --git a/message_xdr.go b/message_xdr.go index c7f16a173..324125ea4 100644 --- a/message_xdr.go +++ b/message_xdr.go @@ -644,7 +644,7 @@ Folder Structure: struct Folder { string ID<64>; - Device Devices<64>; + Device Devices<>; } */ @@ -670,9 +670,6 @@ func (o Folder) encodeXDR(xw *xdr.Writer) (int, error) { return xw.Tot(), xdr.ErrElementSizeExceeded } xw.WriteString(o.ID) - if len(o.Devices) > 64 { - return xw.Tot(), xdr.ErrElementSizeExceeded - } xw.WriteUint32(uint32(len(o.Devices))) for i := range o.Devices { _, err := o.Devices[i].encodeXDR(xw) @@ -697,9 +694,6 @@ func (o *Folder) UnmarshalXDR(bs []byte) error { func (o *Folder) decodeXDR(xr *xdr.Reader) error { o.ID = xr.ReadStringMax(64) _DevicesSize := int(xr.ReadUint32()) - if _DevicesSize > 64 { - return xdr.ErrElementSizeExceeded - } o.Devices = make([]Device, _DevicesSize) for i := range o.Devices { (&o.Devices[i]).decodeXDR(xr) From 65eb528e2d459de38d71afae5b3dc5d49ce4015b Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 21 Oct 2014 08:40:05 +0200 Subject: [PATCH 08/61] Update xdr; handle marshalling errors --- message_xdr.go | 256 ++++++++++++++++++++++++++++++++--------------- protocol.go | 8 +- protocol_test.go | 3 +- 3 files changed, 184 insertions(+), 83 deletions(-) diff --git a/message_xdr.go b/message_xdr.go index 324125ea4..948e63c32 100644 --- a/message_xdr.go +++ b/message_xdr.go @@ -44,20 +44,28 @@ func (o IndexMessage) EncodeXDR(w io.Writer) (int, error) { return o.encodeXDR(xw) } -func (o IndexMessage) MarshalXDR() []byte { +func (o IndexMessage) MarshalXDR() ([]byte, error) { return o.AppendXDR(make([]byte, 0, 128)) } -func (o IndexMessage) AppendXDR(bs []byte) []byte { +func (o IndexMessage) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o IndexMessage) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - o.encodeXDR(xw) - return []byte(aw) + _, err := o.encodeXDR(xw) + return []byte(aw), err } func (o IndexMessage) encodeXDR(xw *xdr.Writer) (int, error) { - if len(o.Folder) > 64 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.Folder); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("Folder", l, 64) } xw.WriteString(o.Folder) xw.WriteUint32(uint32(len(o.Files))) @@ -142,20 +150,28 @@ func (o FileInfo) EncodeXDR(w io.Writer) (int, error) { return o.encodeXDR(xw) } -func (o FileInfo) MarshalXDR() []byte { +func (o FileInfo) MarshalXDR() ([]byte, error) { return o.AppendXDR(make([]byte, 0, 128)) } -func (o FileInfo) AppendXDR(bs []byte) []byte { +func (o FileInfo) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o FileInfo) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - o.encodeXDR(xw) - return []byte(aw) + _, err := o.encodeXDR(xw) + return []byte(aw), err } func (o FileInfo) encodeXDR(xw *xdr.Writer) (int, error) { - if len(o.Name) > 8192 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.Name); l > 8192 { + return xw.Tot(), xdr.ElementSizeExceeded("Name", l, 8192) } xw.WriteString(o.Name) xw.WriteUint32(o.Flags) @@ -244,20 +260,28 @@ func (o FileInfoTruncated) EncodeXDR(w io.Writer) (int, error) { return o.encodeXDR(xw) } -func (o FileInfoTruncated) MarshalXDR() []byte { +func (o FileInfoTruncated) MarshalXDR() ([]byte, error) { return o.AppendXDR(make([]byte, 0, 128)) } -func (o FileInfoTruncated) AppendXDR(bs []byte) []byte { +func (o FileInfoTruncated) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o FileInfoTruncated) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - o.encodeXDR(xw) - return []byte(aw) + _, err := o.encodeXDR(xw) + return []byte(aw), err } func (o FileInfoTruncated) encodeXDR(xw *xdr.Writer) (int, error) { - if len(o.Name) > 8192 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.Name); l > 8192 { + return xw.Tot(), xdr.ElementSizeExceeded("Name", l, 8192) } xw.WriteString(o.Name) xw.WriteUint32(o.Flags) @@ -318,21 +342,29 @@ func (o BlockInfo) EncodeXDR(w io.Writer) (int, error) { return o.encodeXDR(xw) } -func (o BlockInfo) MarshalXDR() []byte { +func (o BlockInfo) MarshalXDR() ([]byte, error) { return o.AppendXDR(make([]byte, 0, 128)) } -func (o BlockInfo) AppendXDR(bs []byte) []byte { +func (o BlockInfo) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o BlockInfo) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - o.encodeXDR(xw) - return []byte(aw) + _, err := o.encodeXDR(xw) + return []byte(aw), err } func (o BlockInfo) encodeXDR(xw *xdr.Writer) (int, error) { xw.WriteUint32(o.Size) - if len(o.Hash) > 64 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.Hash); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("Hash", l, 64) } xw.WriteBytes(o.Hash) return xw.Tot(), xw.Error() @@ -396,24 +428,32 @@ func (o RequestMessage) EncodeXDR(w io.Writer) (int, error) { return o.encodeXDR(xw) } -func (o RequestMessage) MarshalXDR() []byte { +func (o RequestMessage) MarshalXDR() ([]byte, error) { return o.AppendXDR(make([]byte, 0, 128)) } -func (o RequestMessage) AppendXDR(bs []byte) []byte { +func (o RequestMessage) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o RequestMessage) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - o.encodeXDR(xw) - return []byte(aw) + _, err := o.encodeXDR(xw) + return []byte(aw), err } func (o RequestMessage) encodeXDR(xw *xdr.Writer) (int, error) { - if len(o.Folder) > 64 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.Folder); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("Folder", l, 64) } xw.WriteString(o.Folder) - if len(o.Name) > 8192 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.Name); l > 8192 { + return xw.Tot(), xdr.ElementSizeExceeded("Name", l, 8192) } xw.WriteString(o.Name) xw.WriteUint64(o.Offset) @@ -466,15 +506,23 @@ func (o ResponseMessage) EncodeXDR(w io.Writer) (int, error) { return o.encodeXDR(xw) } -func (o ResponseMessage) MarshalXDR() []byte { +func (o ResponseMessage) MarshalXDR() ([]byte, error) { return o.AppendXDR(make([]byte, 0, 128)) } -func (o ResponseMessage) AppendXDR(bs []byte) []byte { +func (o ResponseMessage) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o ResponseMessage) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - o.encodeXDR(xw) - return []byte(aw) + _, err := o.encodeXDR(xw) + return []byte(aw), err } func (o ResponseMessage) encodeXDR(xw *xdr.Writer) (int, error) { @@ -545,28 +593,36 @@ func (o ClusterConfigMessage) EncodeXDR(w io.Writer) (int, error) { return o.encodeXDR(xw) } -func (o ClusterConfigMessage) MarshalXDR() []byte { +func (o ClusterConfigMessage) MarshalXDR() ([]byte, error) { return o.AppendXDR(make([]byte, 0, 128)) } -func (o ClusterConfigMessage) AppendXDR(bs []byte) []byte { +func (o ClusterConfigMessage) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o ClusterConfigMessage) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - o.encodeXDR(xw) - return []byte(aw) + _, err := o.encodeXDR(xw) + return []byte(aw), err } func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) { - if len(o.ClientName) > 64 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.ClientName); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("ClientName", l, 64) } xw.WriteString(o.ClientName) - if len(o.ClientVersion) > 64 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.ClientVersion); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("ClientVersion", l, 64) } xw.WriteString(o.ClientVersion) - if len(o.Folders) > 64 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.Folders); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("Folders", l, 64) } xw.WriteUint32(uint32(len(o.Folders))) for i := range o.Folders { @@ -575,8 +631,8 @@ func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) { return xw.Tot(), err } } - if len(o.Options) > 64 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.Options); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("Options", l, 64) } xw.WriteUint32(uint32(len(o.Options))) for i := range o.Options { @@ -604,7 +660,7 @@ func (o *ClusterConfigMessage) decodeXDR(xr *xdr.Reader) error { o.ClientVersion = xr.ReadStringMax(64) _FoldersSize := int(xr.ReadUint32()) if _FoldersSize > 64 { - return xdr.ErrElementSizeExceeded + return xdr.ElementSizeExceeded("Folders", _FoldersSize, 64) } o.Folders = make([]Folder, _FoldersSize) for i := range o.Folders { @@ -612,7 +668,7 @@ func (o *ClusterConfigMessage) decodeXDR(xr *xdr.Reader) error { } _OptionsSize := int(xr.ReadUint32()) if _OptionsSize > 64 { - return xdr.ErrElementSizeExceeded + return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) } o.Options = make([]Option, _OptionsSize) for i := range o.Options { @@ -654,20 +710,28 @@ func (o Folder) EncodeXDR(w io.Writer) (int, error) { return o.encodeXDR(xw) } -func (o Folder) MarshalXDR() []byte { +func (o Folder) MarshalXDR() ([]byte, error) { return o.AppendXDR(make([]byte, 0, 128)) } -func (o Folder) AppendXDR(bs []byte) []byte { +func (o Folder) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o Folder) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - o.encodeXDR(xw) - return []byte(aw) + _, err := o.encodeXDR(xw) + return []byte(aw), err } func (o Folder) encodeXDR(xw *xdr.Writer) (int, error) { - if len(o.ID) > 64 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.ID); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("ID", l, 64) } xw.WriteString(o.ID) xw.WriteUint32(uint32(len(o.Devices))) @@ -735,20 +799,28 @@ func (o Device) EncodeXDR(w io.Writer) (int, error) { return o.encodeXDR(xw) } -func (o Device) MarshalXDR() []byte { +func (o Device) MarshalXDR() ([]byte, error) { return o.AppendXDR(make([]byte, 0, 128)) } -func (o Device) AppendXDR(bs []byte) []byte { +func (o Device) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o Device) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - o.encodeXDR(xw) - return []byte(aw) + _, err := o.encodeXDR(xw) + return []byte(aw), err } func (o Device) encodeXDR(xw *xdr.Writer) (int, error) { - if len(o.ID) > 32 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.ID); l > 32 { + return xw.Tot(), xdr.ElementSizeExceeded("ID", l, 32) } xw.WriteBytes(o.ID) xw.WriteUint32(o.Flags) @@ -807,24 +879,32 @@ func (o Option) EncodeXDR(w io.Writer) (int, error) { return o.encodeXDR(xw) } -func (o Option) MarshalXDR() []byte { +func (o Option) MarshalXDR() ([]byte, error) { return o.AppendXDR(make([]byte, 0, 128)) } -func (o Option) AppendXDR(bs []byte) []byte { +func (o Option) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o Option) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - o.encodeXDR(xw) - return []byte(aw) + _, err := o.encodeXDR(xw) + return []byte(aw), err } func (o Option) encodeXDR(xw *xdr.Writer) (int, error) { - if len(o.Key) > 64 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.Key); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("Key", l, 64) } xw.WriteString(o.Key) - if len(o.Value) > 1024 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.Value); l > 1024 { + return xw.Tot(), xdr.ElementSizeExceeded("Value", l, 1024) } xw.WriteString(o.Value) return xw.Tot(), xw.Error() @@ -873,20 +953,28 @@ func (o CloseMessage) EncodeXDR(w io.Writer) (int, error) { return o.encodeXDR(xw) } -func (o CloseMessage) MarshalXDR() []byte { +func (o CloseMessage) MarshalXDR() ([]byte, error) { return o.AppendXDR(make([]byte, 0, 128)) } -func (o CloseMessage) AppendXDR(bs []byte) []byte { +func (o CloseMessage) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o CloseMessage) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - o.encodeXDR(xw) - return []byte(aw) + _, err := o.encodeXDR(xw) + return []byte(aw), err } func (o CloseMessage) encodeXDR(xw *xdr.Writer) (int, error) { - if len(o.Reason) > 1024 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.Reason); l > 1024 { + return xw.Tot(), xdr.ElementSizeExceeded("Reason", l, 1024) } xw.WriteString(o.Reason) return xw.Tot(), xw.Error() @@ -927,15 +1015,23 @@ func (o EmptyMessage) EncodeXDR(w io.Writer) (int, error) { return o.encodeXDR(xw) } -func (o EmptyMessage) MarshalXDR() []byte { +func (o EmptyMessage) MarshalXDR() ([]byte, error) { return o.AppendXDR(make([]byte, 0, 128)) } -func (o EmptyMessage) AppendXDR(bs []byte) []byte { +func (o EmptyMessage) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o EmptyMessage) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - o.encodeXDR(xw) - return []byte(aw) + _, err := o.encodeXDR(xw) + return []byte(aw), err } func (o EmptyMessage) encodeXDR(xw *xdr.Writer) (int, error) { diff --git a/protocol.go b/protocol.go index 19cdfbb16..08ef226f0 100644 --- a/protocol.go +++ b/protocol.go @@ -126,7 +126,7 @@ type hdrMsg struct { } type encodable interface { - AppendXDR([]byte) []byte + AppendXDR([]byte) ([]byte, error) } const ( @@ -483,7 +483,11 @@ func (c *rawConnection) writerLoop() { case hm := <-c.outbox: if hm.msg != nil { // Uncompressed message in uncBuf - uncBuf = hm.msg.AppendXDR(uncBuf[:0]) + uncBuf, err = hm.msg.AppendXDR(uncBuf[:0]) + if err != nil { + c.close(err) + return + } if len(uncBuf) >= c.compressionThreshold { // Use compression for large messages diff --git a/protocol_test.go b/protocol_test.go index cc6f9472a..a7bb1416c 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -25,6 +25,7 @@ import ( "io/ioutil" "os" "reflect" + "strings" "testing" "testing/quick" @@ -369,7 +370,7 @@ func testMarshal(t *testing.T, prefix string, m1, m2 message) bool { } _, err := m1.EncodeXDR(&buf) - if err == xdr.ErrElementSizeExceeded { + if err != nil && strings.Contains(err.Error(), "exceeds size") { return true } if err != nil { From 28610a9a426fac3ae88b812897b1c1a540112071 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Sun, 26 Oct 2014 13:15:14 +0100 Subject: [PATCH 09/61] Break out logger as a reusable component --- debug.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug.go b/debug.go index 930aaf344..c46f4a984 100644 --- a/debug.go +++ b/debug.go @@ -19,7 +19,7 @@ import ( "os" "strings" - "github.com/syncthing/syncthing/internal/logger" + "github.com/calmh/logger" ) var ( From ad29093ac12cf822f1b55f8834b820dcd015a9e3 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Sun, 16 Nov 2014 21:13:20 +0100 Subject: [PATCH 10/61] Use more inclusive copyright header --- common_test.go | 2 +- counting.go | 2 +- debug.go | 2 +- deviceid.go | 2 +- deviceid_test.go | 2 +- doc.go | 2 +- header.go | 2 +- message.go | 2 +- nativemodel_darwin.go | 2 +- nativemodel_unix.go | 2 +- nativemodel_windows.go | 2 +- protocol.go | 2 +- protocol_test.go | 2 +- wireformat.go | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/common_test.go b/common_test.go index 29b910bab..e38da00a8 100644 --- a/common_test.go +++ b/common_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free diff --git a/counting.go b/counting.go index 4f95c9f9a..ac4aeaa9a 100644 --- a/counting.go +++ b/counting.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free diff --git a/debug.go b/debug.go index c46f4a984..4664b0e23 100644 --- a/debug.go +++ b/debug.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free diff --git a/deviceid.go b/deviceid.go index 4b034f3f1..93753d7fa 100644 --- a/deviceid.go +++ b/deviceid.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free diff --git a/deviceid_test.go b/deviceid_test.go index 14e2dfa34..da398c6fb 100644 --- a/deviceid_test.go +++ b/deviceid_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free diff --git a/doc.go b/doc.go index 95b5e102f..4f73e6c86 100644 --- a/doc.go +++ b/doc.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free diff --git a/header.go b/header.go index 96ccff66f..8da092e5c 100644 --- a/header.go +++ b/header.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free diff --git a/message.go b/message.go index 445ac7a75..eff837fe1 100644 --- a/message.go +++ b/message.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free diff --git a/nativemodel_darwin.go b/nativemodel_darwin.go index 7fb5c964e..8b2b842a4 100644 --- a/nativemodel_darwin.go +++ b/nativemodel_darwin.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free diff --git a/nativemodel_unix.go b/nativemodel_unix.go index 61ad9c438..bf6499e63 100644 --- a/nativemodel_unix.go +++ b/nativemodel_unix.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free diff --git a/nativemodel_windows.go b/nativemodel_windows.go index 57d3f3446..d2c079bdc 100644 --- a/nativemodel_windows.go +++ b/nativemodel_windows.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free diff --git a/protocol.go b/protocol.go index 08ef226f0..ae7f480b3 100644 --- a/protocol.go +++ b/protocol.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free diff --git a/protocol_test.go b/protocol_test.go index a7bb1416c..9da9422c1 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free diff --git a/wireformat.go b/wireformat.go index f06b9454e..b7f4e367d 100644 --- a/wireformat.go +++ b/wireformat.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free From e0da2764c930ecde44dcf2d66283c523830da0ae Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Tue, 4 Nov 2014 23:22:15 +0000 Subject: [PATCH 11/61] Code smell --- message.go | 28 +++++++++++++++++++++------- protocol.go | 16 ---------------- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/message.go b/message.go index eff837fe1..5d35f17cb 100644 --- a/message.go +++ b/message.go @@ -37,7 +37,7 @@ func (f FileInfo) String() string { } func (f FileInfo) Size() (bytes int64) { - if IsDeleted(f.Flags) || IsDirectory(f.Flags) { + if f.IsDeleted() || f.IsDirectory() { return 128 } for _, b := range f.Blocks { @@ -47,15 +47,19 @@ func (f FileInfo) Size() (bytes int64) { } func (f FileInfo) IsDeleted() bool { - return IsDeleted(f.Flags) + return f.Flags&FlagDeleted != 0 } func (f FileInfo) IsInvalid() bool { - return IsInvalid(f.Flags) + return f.Flags&FlagInvalid != 0 } func (f FileInfo) IsDirectory() bool { - return IsDirectory(f.Flags) + return f.Flags&FlagDirectory != 0 +} + +func (f FileInfo) HasPermissionBits() bool { + return f.Flags&FlagNoPermBits == 0 } // Used for unmarshalling a FileInfo structure but skipping the actual block list @@ -75,7 +79,7 @@ func (f FileInfoTruncated) String() string { // Returns a statistical guess on the size, not the exact figure func (f FileInfoTruncated) Size() int64 { - if IsDeleted(f.Flags) || IsDirectory(f.Flags) { + if f.IsDeleted() || f.IsDirectory() { return 128 } if f.NumBlocks < 2 { @@ -86,17 +90,27 @@ func (f FileInfoTruncated) Size() int64 { } func (f FileInfoTruncated) IsDeleted() bool { - return IsDeleted(f.Flags) + return f.Flags&FlagDeleted != 0 } func (f FileInfoTruncated) IsInvalid() bool { - return IsInvalid(f.Flags) + return f.Flags&FlagInvalid != 0 +} + +func (f FileInfoTruncated) IsDirectory() bool { + return f.Flags&FlagDirectory != 0 +} + +func (f FileInfoTruncated) HasPermissionBits() bool { + return f.Flags&FlagNoPermBits == 0 } type FileIntf interface { Size() int64 IsDeleted() bool IsInvalid() bool + IsDirectory() bool + HasPermissionBits() bool } type BlockInfo struct { diff --git a/protocol.go b/protocol.go index ae7f480b3..7de53eab9 100644 --- a/protocol.go +++ b/protocol.go @@ -637,19 +637,3 @@ func (c *rawConnection) Statistics() Statistics { OutBytesTotal: c.cw.Tot(), } } - -func IsDeleted(bits uint32) bool { - return bits&FlagDeleted != 0 -} - -func IsInvalid(bits uint32) bool { - return bits&FlagInvalid != 0 -} - -func IsDirectory(bits uint32) bool { - return bits&FlagDirectory != 0 -} - -func HasPermissionBits(bits uint32) bool { - return bits&FlagNoPermBits == 0 -} From ddc56c8a0ddf0afa5cd9b8798f39cf1bacd6b821 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Fri, 7 Nov 2014 23:06:04 +0000 Subject: [PATCH 12/61] Add symlink support at the protocol level --- message.go | 9 +++++++++ protocol.go | 12 ++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/message.go b/message.go index 5d35f17cb..355050cab 100644 --- a/message.go +++ b/message.go @@ -58,6 +58,10 @@ func (f FileInfo) IsDirectory() bool { return f.Flags&FlagDirectory != 0 } +func (f FileInfo) IsSymlink() bool { + return f.Flags&FlagSymlink != 0 +} + func (f FileInfo) HasPermissionBits() bool { return f.Flags&FlagNoPermBits == 0 } @@ -101,6 +105,10 @@ func (f FileInfoTruncated) IsDirectory() bool { return f.Flags&FlagDirectory != 0 } +func (f FileInfoTruncated) IsSymlink() bool { + return f.Flags&FlagSymlink != 0 +} + func (f FileInfoTruncated) HasPermissionBits() bool { return f.Flags&FlagNoPermBits == 0 } @@ -110,6 +118,7 @@ type FileIntf interface { IsDeleted() bool IsInvalid() bool IsDirectory() bool + IsSymlink() bool HasPermissionBits() bool } diff --git a/protocol.go b/protocol.go index 7de53eab9..fb51a1c01 100644 --- a/protocol.go +++ b/protocol.go @@ -49,10 +49,14 @@ const ( ) const ( - FlagDeleted uint32 = 1 << 12 - FlagInvalid = 1 << 13 - FlagDirectory = 1 << 14 - FlagNoPermBits = 1 << 15 + FlagDeleted uint32 = 1 << 12 + FlagInvalid = 1 << 13 + FlagDirectory = 1 << 14 + FlagNoPermBits = 1 << 15 + FlagSymlink = 1 << 16 + FlagSymlinkMissingTarget = 1 << 17 + + SymlinkTypeMask = FlagDirectory | FlagSymlinkMissingTarget ) const ( From 3af96e50bd09a36167202e0f5e9fe8d310c70a3b Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Sun, 23 Nov 2014 00:52:48 +0000 Subject: [PATCH 13/61] Use custom structure for /need calls (fixes #1001) Also, remove trimming by number of blocks as this no longer affects the size of the response. --- message.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/message.go b/message.go index 355050cab..9a96f80c5 100644 --- a/message.go +++ b/message.go @@ -81,16 +81,19 @@ func (f FileInfoTruncated) String() string { f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.NumBlocks) } +func BlocksToSize(num uint32) int64 { + if num < 2 { + return BlockSize / 2 + } + return int64(num-1)*BlockSize + BlockSize/2 +} + // Returns a statistical guess on the size, not the exact figure func (f FileInfoTruncated) Size() int64 { if f.IsDeleted() || f.IsDirectory() { return 128 } - if f.NumBlocks < 2 { - return BlockSize / 2 - } else { - return int64(f.NumBlocks-1)*BlockSize + BlockSize/2 - } + return BlocksToSize(f.NumBlocks) } func (f FileInfoTruncated) IsDeleted() bool { From dc71ec734d7f4ddbc83043920134075bf8b2d3f0 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Sun, 30 Nov 2014 00:17:00 +0100 Subject: [PATCH 14/61] Dependency update, new golang.org/x package names --- nativemodel_darwin.go | 2 +- wireformat.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nativemodel_darwin.go b/nativemodel_darwin.go index 8b2b842a4..ba30d1ae1 100644 --- a/nativemodel_darwin.go +++ b/nativemodel_darwin.go @@ -19,7 +19,7 @@ package protocol // Darwin uses NFD normalization -import "code.google.com/p/go.text/unicode/norm" +import "golang.org/x/text/unicode/norm" type nativeModel struct { next Model diff --git a/wireformat.go b/wireformat.go index b7f4e367d..84da71423 100644 --- a/wireformat.go +++ b/wireformat.go @@ -18,7 +18,7 @@ package protocol import ( "path/filepath" - "code.google.com/p/go.text/unicode/norm" + "golang.org/x/text/unicode/norm" ) type wireFormatConnection struct { From 190c61ba2f07b4b698f8281c3bef9118ca785d52 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Sat, 6 Dec 2014 14:23:10 +0100 Subject: [PATCH 15/61] Use Go 1.4 'generate' to create XDR codec --- message.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/message.go b/message.go index 9a96f80c5..8cc191d8c 100644 --- a/message.go +++ b/message.go @@ -13,6 +13,9 @@ // You should have received a copy of the GNU General Public License along // with this program. If not, see . +//go:generate -command genxdr go run ../../Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go +//go:generate genxdr -o message_xdr.go message.go + package protocol import "fmt" From 09b534b8a32392776c7eed5c41f3f0cc1356b56e Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Mon, 1 Dec 2014 19:23:06 +0000 Subject: [PATCH 16/61] Add job queue (fixes #629) Request to terminate currently ongoing downloads and jump to the bumped file incoming in 3, 2, 1. Also, has a slightly strange effect where we pop a job off the queue, but the copyChannel is still busy and blocks, though it gets moved to the progress slice in the jobqueue, and looks like it's in progress which it isn't as it's waiting to be picked up from the copyChan. As a result, the progress emitter doesn't register on the task, and hence the file doesn't have a progress bar, but cannot be replaced by a bump. I guess I can fix progress bar issue by moving the progressEmiter.Register just before passing the file to the copyChan, but then we are back to the initial problem of a file with a progress bar, but no progress happening as it's stuck on write to copyChan I checked if there is a way to check for channel writeability (before popping) but got struck by lightning just for bringing the idea up in #go-nuts. My ideal scenario would be to check if copyChan is writeable, pop job from the queue and shove it down handleFile. This way jobs would stay in the queue while they cannot be handled, meaning that the `Bump` could bring your file up higher. --- message.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/message.go b/message.go index 8cc191d8c..ae04a9da8 100644 --- a/message.go +++ b/message.go @@ -69,6 +69,17 @@ func (f FileInfo) HasPermissionBits() bool { return f.Flags&FlagNoPermBits == 0 } +func (f FileInfo) ToTruncated() FileInfoTruncated { + return FileInfoTruncated{ + Name: f.Name, + Flags: f.Flags, + Modified: f.Modified, + Version: f.Version, + LocalVersion: f.LocalVersion, + NumBlocks: uint32(len(f.Blocks)), + } +} + // Used for unmarshalling a FileInfo structure but skipping the actual block list type FileInfoTruncated struct { Name string // max:8192 From a3ea9427d1b9167490d910add0af6ed106c882a7 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Thu, 8 Jan 2015 14:21:58 +0100 Subject: [PATCH 17/61] Ensure backwards compatibility before modifying protocol This change makes sure that things work smoothly when "we" are a newer version than our peer and have more fields in our messages than they do. Missing fields will be left at zero/nil. (The other side will ignore our extra fields, for the same effect.) --- protocol.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/protocol.go b/protocol.go index fb51a1c01..a6a4b8b4f 100644 --- a/protocol.go +++ b/protocol.go @@ -133,6 +133,10 @@ type encodable interface { AppendXDR([]byte) ([]byte, error) } +type isEofer interface { + IsEOF() bool +} + const ( pingTimeout = 30 * time.Second pingIdleTime = 60 * time.Second @@ -376,20 +380,36 @@ func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) { } } + // We check each returned error for the XDRError.IsEOF() method. + // IsEOF()==true here means that the message contained fewer fields than + // expected. It does not signify an EOF on the socket, because we've + // successfully read a size value and that many bytes already. New fields + // we expected but the other peer didn't send should be interpreted as + // zero/nil, and if that's not valid we'll verify it somewhere else. + switch hdr.msgType { case messageTypeIndex, messageTypeIndexUpdate: var idx IndexMessage err = idx.UnmarshalXDR(msgBuf) + if xdrErr, ok := err.(isEofer); ok && xdrErr.IsEOF() { + err = nil + } msg = idx case messageTypeRequest: var req RequestMessage err = req.UnmarshalXDR(msgBuf) + if xdrErr, ok := err.(isEofer); ok && xdrErr.IsEOF() { + err = nil + } msg = req case messageTypeResponse: var resp ResponseMessage err = resp.UnmarshalXDR(msgBuf) + if xdrErr, ok := err.(isEofer); ok && xdrErr.IsEOF() { + err = nil + } msg = resp case messageTypePing, messageTypePong: @@ -398,11 +418,17 @@ func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) { case messageTypeClusterConfig: var cc ClusterConfigMessage err = cc.UnmarshalXDR(msgBuf) + if xdrErr, ok := err.(isEofer); ok && xdrErr.IsEOF() { + err = nil + } msg = cc case messageTypeClose: var cm CloseMessage err = cm.UnmarshalXDR(msgBuf) + if xdrErr, ok := err.(isEofer); ok && xdrErr.IsEOF() { + err = nil + } msg = cm default: From c111ed4b2046d307c2767cff61dab77aba6fca0e Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Thu, 8 Jan 2015 10:28:39 +0100 Subject: [PATCH 18/61] Add fields for future extensibility This adds a number of fields to the end of existing messages. This is a backwards compatible change. --- message.go | 21 ++++++++----- message_xdr.go | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++ protocol.go | 21 ++++++++++--- 3 files changed, 113 insertions(+), 11 deletions(-) diff --git a/message.go b/message.go index ae04a9da8..e0c66b9dc 100644 --- a/message.go +++ b/message.go @@ -21,8 +21,10 @@ package protocol import "fmt" type IndexMessage struct { - Folder string // max:64 - Files []FileInfo + Folder string // max:64 + Files []FileInfo + Flags uint32 + Options []Option // max:64 } type FileInfo struct { @@ -150,14 +152,18 @@ func (b BlockInfo) String() string { } type RequestMessage struct { - Folder string // max:64 - Name string // max:8192 - Offset uint64 - Size uint32 + Folder string // max:64 + Name string // max:8192 + Offset uint64 + Size uint32 + Hash []byte // max:64 + Flags uint32 + Options []Option // max:64 } type ResponseMessage struct { - Data []byte + Data []byte + Error uint32 } type ClusterConfigMessage struct { @@ -194,6 +200,7 @@ type Option struct { type CloseMessage struct { Reason string // max:1024 + Code uint32 } type EmptyMessage struct{} diff --git a/message_xdr.go b/message_xdr.go index 948e63c32..da13111c2 100644 --- a/message_xdr.go +++ b/message_xdr.go @@ -30,11 +30,21 @@ IndexMessage Structure: \ Zero or more FileInfo Structures \ / / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Flags | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Number of Options | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Zero or more Option Structures \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ struct IndexMessage { string Folder<64>; FileInfo Files<>; + unsigned int Flags; + Option Options<64>; } */ @@ -75,6 +85,17 @@ func (o IndexMessage) encodeXDR(xw *xdr.Writer) (int, error) { return xw.Tot(), err } } + xw.WriteUint32(o.Flags) + if l := len(o.Options); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("Options", l, 64) + } + xw.WriteUint32(uint32(len(o.Options))) + for i := range o.Options { + _, err := o.Options[i].encodeXDR(xw) + if err != nil { + return xw.Tot(), err + } + } return xw.Tot(), xw.Error() } @@ -96,6 +117,15 @@ func (o *IndexMessage) decodeXDR(xr *xdr.Reader) error { for i := range o.Files { (&o.Files[i]).decodeXDR(xr) } + o.Flags = xr.ReadUint32() + _OptionsSize := int(xr.ReadUint32()) + if _OptionsSize > 64 { + return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) + } + o.Options = make([]Option, _OptionsSize) + for i := range o.Options { + (&o.Options[i]).decodeXDR(xr) + } return xr.Error() } @@ -412,6 +442,20 @@ RequestMessage Structure: +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | Size | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Hash | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Hash (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Flags | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Number of Options | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Zero or more Option Structures \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ struct RequestMessage { @@ -419,6 +463,9 @@ struct RequestMessage { string Name<8192>; unsigned hyper Offset; unsigned int Size; + opaque Hash<64>; + unsigned int Flags; + Option Options<64>; } */ @@ -458,6 +505,21 @@ func (o RequestMessage) encodeXDR(xw *xdr.Writer) (int, error) { xw.WriteString(o.Name) xw.WriteUint64(o.Offset) xw.WriteUint32(o.Size) + if l := len(o.Hash); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("Hash", l, 64) + } + xw.WriteBytes(o.Hash) + xw.WriteUint32(o.Flags) + if l := len(o.Options); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("Options", l, 64) + } + xw.WriteUint32(uint32(len(o.Options))) + for i := range o.Options { + _, err := o.Options[i].encodeXDR(xw) + if err != nil { + return xw.Tot(), err + } + } return xw.Tot(), xw.Error() } @@ -477,6 +539,16 @@ func (o *RequestMessage) decodeXDR(xr *xdr.Reader) error { o.Name = xr.ReadStringMax(8192) o.Offset = xr.ReadUint64() o.Size = xr.ReadUint32() + o.Hash = xr.ReadBytesMax(64) + o.Flags = xr.ReadUint32() + _OptionsSize := int(xr.ReadUint32()) + if _OptionsSize > 64 { + return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) + } + o.Options = make([]Option, _OptionsSize) + for i := range o.Options { + (&o.Options[i]).decodeXDR(xr) + } return xr.Error() } @@ -493,10 +565,13 @@ ResponseMessage Structure: \ Data (variable length) \ / / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Error | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ struct ResponseMessage { opaque Data<>; + unsigned int Error; } */ @@ -527,6 +602,7 @@ func (o ResponseMessage) AppendXDR(bs []byte) ([]byte, error) { func (o ResponseMessage) encodeXDR(xw *xdr.Writer) (int, error) { xw.WriteBytes(o.Data) + xw.WriteUint32(o.Error) return xw.Tot(), xw.Error() } @@ -543,6 +619,7 @@ func (o *ResponseMessage) UnmarshalXDR(bs []byte) error { func (o *ResponseMessage) decodeXDR(xr *xdr.Reader) error { o.Data = xr.ReadBytes() + o.Error = xr.ReadUint32() return xr.Error() } @@ -940,10 +1017,13 @@ CloseMessage Structure: \ Reason (variable length) \ / / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Code | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ struct CloseMessage { string Reason<1024>; + unsigned int Code; } */ @@ -977,6 +1057,7 @@ func (o CloseMessage) encodeXDR(xw *xdr.Writer) (int, error) { return xw.Tot(), xdr.ElementSizeExceeded("Reason", l, 1024) } xw.WriteString(o.Reason) + xw.WriteUint32(o.Code) return xw.Tot(), xw.Error() } @@ -993,6 +1074,7 @@ func (o *CloseMessage) UnmarshalXDR(bs []byte) error { func (o *CloseMessage) decodeXDR(xr *xdr.Reader) error { o.Reason = xr.ReadStringMax(1024) + o.Code = xr.ReadUint32() return xr.Error() } diff --git a/protocol.go b/protocol.go index a6a4b8b4f..e65d5e72e 100644 --- a/protocol.go +++ b/protocol.go @@ -187,7 +187,10 @@ func (c *rawConnection) Index(folder string, idx []FileInfo) error { default: } c.idxMut.Lock() - c.send(-1, messageTypeIndex, IndexMessage{folder, idx}) + c.send(-1, messageTypeIndex, IndexMessage{ + Folder: folder, + Files: idx, + }) c.idxMut.Unlock() return nil } @@ -200,7 +203,10 @@ func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo) error { default: } c.idxMut.Lock() - c.send(-1, messageTypeIndexUpdate, IndexMessage{folder, idx}) + c.send(-1, messageTypeIndexUpdate, IndexMessage{ + Folder: folder, + Files: idx, + }) c.idxMut.Unlock() return nil } @@ -222,7 +228,12 @@ func (c *rawConnection) Request(folder string, name string, offset int64, size i c.awaiting[id] = rc c.awaitingMut.Unlock() - ok := c.send(id, messageTypeRequest, RequestMessage{folder, name, uint64(offset), uint32(size)}) + ok := c.send(id, messageTypeRequest, RequestMessage{ + Folder: folder, + Name: name, + Offset: uint64(offset), + Size: uint32(size), + }) if !ok { return nil, ErrClosed } @@ -455,7 +466,9 @@ func (c *rawConnection) handleIndexUpdate(im IndexMessage) { func (c *rawConnection) handleRequest(msgID int, req RequestMessage) { data, _ := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), int(req.Size)) - c.send(msgID, messageTypeResponse, ResponseMessage{data}) + c.send(msgID, messageTypeResponse, ResponseMessage{ + Data: data, + }) } func (c *rawConnection) handleResponse(msgID int, resp ResponseMessage) { From 8c32955da13ca35484069ac2587d9ddee56f3b90 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Thu, 8 Jan 2015 22:11:10 +0100 Subject: [PATCH 19/61] Actually close connection based on unknown protocol version --- protocol.go | 5 +++++ protocol_test.go | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/protocol.go b/protocol.go index e65d5e72e..a55256799 100644 --- a/protocol.go +++ b/protocol.go @@ -356,6 +356,11 @@ func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) { l.Debugf("read header %v (msglen=%d)", hdr, msglen) } + if hdr.version != 0 { + err = fmt.Errorf("unknown protocol version 0x%x", hdr.version) + return + } + if cap(c.rdbuf0) < msglen { c.rdbuf0 = make([]byte, msglen) } else { diff --git a/protocol_test.go b/protocol_test.go index 9da9422c1..75e0086ee 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -189,7 +189,7 @@ func TestVersionErr(t *testing.T) { msgID: 0, msgType: 0, })) - w.WriteUint32(0) + w.WriteUint32(0) // Avoids reader closing due to EOF if !m1.isClosed() { t.Error("Connection should close due to unknown version") @@ -212,7 +212,7 @@ func TestTypeErr(t *testing.T) { msgID: 0, msgType: 42, })) - w.WriteUint32(0) + w.WriteUint32(0) // Avoids reader closing due to EOF if !m1.isClosed() { t.Error("Connection should close due to unknown message type") From 36708a5067f340cdb7383c6ea0c80c4defcdbafd Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Fri, 9 Jan 2015 08:18:42 +0100 Subject: [PATCH 20/61] Move FileIntf to files package, expose Iterator type This is where FileIntf is used, so it should be defined here (it's not a protocol thing, really). --- message.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/message.go b/message.go index e0c66b9dc..3129f3de0 100644 --- a/message.go +++ b/message.go @@ -132,15 +132,6 @@ func (f FileInfoTruncated) HasPermissionBits() bool { return f.Flags&FlagNoPermBits == 0 } -type FileIntf interface { - Size() int64 - IsDeleted() bool - IsInvalid() bool - IsDirectory() bool - IsSymlink() bool - HasPermissionBits() bool -} - type BlockInfo struct { Offset int64 // noencode (cache only) Size uint32 From d9ed8e125e3b9678830047738c868eeb77e56488 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Fri, 9 Jan 2015 08:19:32 +0100 Subject: [PATCH 21/61] Move FileInfoTruncated to files package This is where it's used, and it clarifies that it's never used over the wire. --- message.go | 61 ------------------------------ message_xdr.go | 100 ------------------------------------------------- 2 files changed, 161 deletions(-) diff --git a/message.go b/message.go index 3129f3de0..0d8de8070 100644 --- a/message.go +++ b/message.go @@ -71,67 +71,6 @@ func (f FileInfo) HasPermissionBits() bool { return f.Flags&FlagNoPermBits == 0 } -func (f FileInfo) ToTruncated() FileInfoTruncated { - return FileInfoTruncated{ - Name: f.Name, - Flags: f.Flags, - Modified: f.Modified, - Version: f.Version, - LocalVersion: f.LocalVersion, - NumBlocks: uint32(len(f.Blocks)), - } -} - -// Used for unmarshalling a FileInfo structure but skipping the actual block list -type FileInfoTruncated struct { - Name string // max:8192 - Flags uint32 - Modified int64 - Version uint64 - LocalVersion uint64 - NumBlocks uint32 -} - -func (f FileInfoTruncated) String() string { - return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%d, Size:%d, NumBlocks:%d}", - f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.NumBlocks) -} - -func BlocksToSize(num uint32) int64 { - if num < 2 { - return BlockSize / 2 - } - return int64(num-1)*BlockSize + BlockSize/2 -} - -// Returns a statistical guess on the size, not the exact figure -func (f FileInfoTruncated) Size() int64 { - if f.IsDeleted() || f.IsDirectory() { - return 128 - } - return BlocksToSize(f.NumBlocks) -} - -func (f FileInfoTruncated) IsDeleted() bool { - return f.Flags&FlagDeleted != 0 -} - -func (f FileInfoTruncated) IsInvalid() bool { - return f.Flags&FlagInvalid != 0 -} - -func (f FileInfoTruncated) IsDirectory() bool { - return f.Flags&FlagDirectory != 0 -} - -func (f FileInfoTruncated) IsSymlink() bool { - return f.Flags&FlagSymlink != 0 -} - -func (f FileInfoTruncated) HasPermissionBits() bool { - return f.Flags&FlagNoPermBits == 0 -} - type BlockInfo struct { Offset int64 // noencode (cache only) Size uint32 diff --git a/message_xdr.go b/message_xdr.go index da13111c2..fa7b5802b 100644 --- a/message_xdr.go +++ b/message_xdr.go @@ -245,106 +245,6 @@ func (o *FileInfo) decodeXDR(xr *xdr.Reader) error { /* -FileInfoTruncated Structure: - - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Length of Name | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Name (variable length) \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Flags | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | -+ Modified (64 bits) + -| | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | -+ Version (64 bits) + -| | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | -+ Local Version (64 bits) + -| | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Num Blocks | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - -struct FileInfoTruncated { - string Name<8192>; - unsigned int Flags; - hyper Modified; - unsigned hyper Version; - unsigned hyper LocalVersion; - unsigned int NumBlocks; -} - -*/ - -func (o FileInfoTruncated) EncodeXDR(w io.Writer) (int, error) { - var xw = xdr.NewWriter(w) - return o.encodeXDR(xw) -} - -func (o FileInfoTruncated) MarshalXDR() ([]byte, error) { - return o.AppendXDR(make([]byte, 0, 128)) -} - -func (o FileInfoTruncated) MustMarshalXDR() []byte { - bs, err := o.MarshalXDR() - if err != nil { - panic(err) - } - return bs -} - -func (o FileInfoTruncated) AppendXDR(bs []byte) ([]byte, error) { - var aw = xdr.AppendWriter(bs) - var xw = xdr.NewWriter(&aw) - _, err := o.encodeXDR(xw) - return []byte(aw), err -} - -func (o FileInfoTruncated) encodeXDR(xw *xdr.Writer) (int, error) { - if l := len(o.Name); l > 8192 { - return xw.Tot(), xdr.ElementSizeExceeded("Name", l, 8192) - } - xw.WriteString(o.Name) - xw.WriteUint32(o.Flags) - xw.WriteUint64(uint64(o.Modified)) - xw.WriteUint64(o.Version) - xw.WriteUint64(o.LocalVersion) - xw.WriteUint32(o.NumBlocks) - return xw.Tot(), xw.Error() -} - -func (o *FileInfoTruncated) DecodeXDR(r io.Reader) error { - xr := xdr.NewReader(r) - return o.decodeXDR(xr) -} - -func (o *FileInfoTruncated) UnmarshalXDR(bs []byte) error { - var br = bytes.NewReader(bs) - var xr = xdr.NewReader(br) - return o.decodeXDR(xr) -} - -func (o *FileInfoTruncated) decodeXDR(xr *xdr.Reader) error { - o.Name = xr.ReadStringMax(8192) - o.Flags = xr.ReadUint32() - o.Modified = int64(xr.ReadUint64()) - o.Version = xr.ReadUint64() - o.LocalVersion = xr.ReadUint64() - o.NumBlocks = xr.ReadUint32() - return xr.Error() -} - -/* - BlockInfo Structure: 0 1 2 3 From 7a0a702ec0bcd4adfd6798695df5e8913586edf5 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Sun, 11 Jan 2015 13:24:56 +0100 Subject: [PATCH 22/61] Refactor readerLoop to switch on message type directly --- protocol.go | 58 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/protocol.go b/protocol.go index a55256799..1febf8b71 100644 --- a/protocol.go +++ b/protocol.go @@ -71,6 +71,10 @@ var ( ErrClosed = errors.New("connection closed") ) +// Specific variants of empty messages... +type pingMessage struct{ EmptyMessage } +type pongMessage struct{ EmptyMessage } + type Model interface { // An index was received from the peer device Index(deviceID DeviceID, folder string, files []FileInfo) @@ -289,48 +293,51 @@ func (c *rawConnection) readerLoop() (err error) { return err } - switch hdr.msgType { - case messageTypeIndex: - if c.state < stateCCRcvd { - return fmt.Errorf("protocol error: index message in state %d", c.state) - } - c.handleIndex(msg.(IndexMessage)) - c.state = stateIdxRcvd + switch msg := msg.(type) { + case IndexMessage: + switch hdr.msgType { + case messageTypeIndex: + if c.state < stateCCRcvd { + return fmt.Errorf("protocol error: index message in state %d", c.state) + } + c.handleIndex(msg) + c.state = stateIdxRcvd - case messageTypeIndexUpdate: - if c.state < stateIdxRcvd { - return fmt.Errorf("protocol error: index update message in state %d", c.state) + case messageTypeIndexUpdate: + if c.state < stateIdxRcvd { + return fmt.Errorf("protocol error: index update message in state %d", c.state) + } + c.handleIndexUpdate(msg) } - c.handleIndexUpdate(msg.(IndexMessage)) - case messageTypeRequest: + case RequestMessage: if c.state < stateIdxRcvd { return fmt.Errorf("protocol error: request message in state %d", c.state) } // Requests are handled asynchronously - go c.handleRequest(hdr.msgID, msg.(RequestMessage)) + go c.handleRequest(hdr.msgID, msg) - case messageTypeResponse: + case ResponseMessage: if c.state < stateIdxRcvd { return fmt.Errorf("protocol error: response message in state %d", c.state) } - c.handleResponse(hdr.msgID, msg.(ResponseMessage)) + c.handleResponse(hdr.msgID, msg) - case messageTypePing: - c.send(hdr.msgID, messageTypePong, EmptyMessage{}) + case pingMessage: + c.send(hdr.msgID, messageTypePong, pongMessage{}) - case messageTypePong: + case pongMessage: c.handlePong(hdr.msgID) - case messageTypeClusterConfig: + case ClusterConfigMessage: if c.state != stateInitial { return fmt.Errorf("protocol error: cluster config message in state %d", c.state) } - go c.receiver.ClusterConfig(c.id, msg.(ClusterConfigMessage)) + go c.receiver.ClusterConfig(c.id, msg) c.state = stateCCRcvd - case messageTypeClose: - return errors.New(msg.(CloseMessage).Reason) + case CloseMessage: + return errors.New(msg.Reason) default: return fmt.Errorf("protocol error: %s: unknown message type %#x", c.id, hdr.msgType) @@ -428,8 +435,11 @@ func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) { } msg = resp - case messageTypePing, messageTypePong: - msg = EmptyMessage{} + case messageTypePing: + msg = pingMessage{} + + case messageTypePong: + msg = pongMessage{} case messageTypeClusterConfig: var cc ClusterConfigMessage From cd34eea017f9a955c415744db145be240f5774ae Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Sun, 11 Jan 2015 13:25:22 +0100 Subject: [PATCH 23/61] Reject Index and Request messages with unexpected flags --- protocol.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/protocol.go b/protocol.go index 1febf8b71..0c59ef56e 100644 --- a/protocol.go +++ b/protocol.go @@ -295,6 +295,11 @@ func (c *rawConnection) readerLoop() (err error) { switch msg := msg.(type) { case IndexMessage: + if msg.Flags != 0 { + // We don't currently support or expect any flags. + return fmt.Errorf("protocol error: unknown flags 0x%x in Index(Update) message", msg.Flags) + } + switch hdr.msgType { case messageTypeIndex: if c.state < stateCCRcvd { @@ -311,6 +316,10 @@ func (c *rawConnection) readerLoop() (err error) { } case RequestMessage: + if msg.Flags != 0 { + // We don't currently support or expect any flags. + return fmt.Errorf("protocol error: unknown flags 0x%x in Request message", msg.Flags) + } if c.state < stateIdxRcvd { return fmt.Errorf("protocol error: request message in state %d", c.state) } From 6213c4f2cdb09f12a51090ab1e389f9e2718902d Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Wed, 7 Jan 2015 15:44:36 +0100 Subject: [PATCH 24/61] Remove nil filenames from database and indexes (fixes #1243) --- protocol.go | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/protocol.go b/protocol.go index 0c59ef56e..5fe7cc344 100644 --- a/protocol.go +++ b/protocol.go @@ -477,14 +477,37 @@ func (c *rawConnection) handleIndex(im IndexMessage) { if debug { l.Debugf("Index(%v, %v, %d files)", c.id, im.Folder, len(im.Files)) } - c.receiver.Index(c.id, im.Folder, im.Files) + c.receiver.Index(c.id, im.Folder, filterIndexMessageFiles(im.Files)) } func (c *rawConnection) handleIndexUpdate(im IndexMessage) { if debug { l.Debugf("queueing IndexUpdate(%v, %v, %d files)", c.id, im.Folder, len(im.Files)) } - c.receiver.IndexUpdate(c.id, im.Folder, im.Files) + c.receiver.IndexUpdate(c.id, im.Folder, filterIndexMessageFiles(im.Files)) +} + +func filterIndexMessageFiles(fs []FileInfo) []FileInfo { + var out []FileInfo + for i, f := range fs { + if f.Name == "" { + l.Infoln("Dropping nil filename from incoming index") + if out == nil { + // Most incoming updates won't contain anything invalid, so we + // delay the allocation and copy to output slice until we + // really need to do it, then copy all the so var valid files + // to it. + out = make([]FileInfo, i, len(fs)-1) + copy(out, fs) + } + } else if out != nil { + out = append(out, f) + } + } + if out != nil { + return out + } + return fs } func (c *rawConnection) handleRequest(msgID int, req RequestMessage) { From d02158c0effa009fa4bb61421a868882aad0e470 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 13 Jan 2015 12:28:35 +0100 Subject: [PATCH 25/61] Also filter out some other obviously invalid filenames (ref #1243) --- protocol.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/protocol.go b/protocol.go index 5fe7cc344..414704286 100644 --- a/protocol.go +++ b/protocol.go @@ -490,8 +490,9 @@ func (c *rawConnection) handleIndexUpdate(im IndexMessage) { func filterIndexMessageFiles(fs []FileInfo) []FileInfo { var out []FileInfo for i, f := range fs { - if f.Name == "" { - l.Infoln("Dropping nil filename from incoming index") + switch f.Name { + case "", ".", "..", "/": // A few obviously invalid filenames + l.Infof("Dropping invalid filename %q from incoming index", f.Name) if out == nil { // Most incoming updates won't contain anything invalid, so we // delay the allocation and copy to output slice until we @@ -500,8 +501,10 @@ func filterIndexMessageFiles(fs []FileInfo) []FileInfo { out = make([]FileInfo, i, len(fs)-1) copy(out, fs) } - } else if out != nil { - out = append(out, f) + default: + if out != nil { + out = append(out, f) + } } } if out != nil { From 2ceaca88288923524ae402537bbdd9062c0aba39 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 13 Jan 2015 13:09:59 +0100 Subject: [PATCH 26/61] Add documentation copied from Syncthing --- AUTHORS | 4 + CONTRIBUTING.md | 76 ++++++ LICENSE | 674 ++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 13 + 4 files changed, 767 insertions(+) create mode 100644 AUTHORS create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 README.md diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 000000000..f10d40d01 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,4 @@ +# This is the official list of authors for copyright purposes. + +Audrius Butkevicius +Jakob Borg diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..856dde7d8 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,76 @@ +## Reporting Bugs + +Please file bugs in the [Github Issue +Tracker](https://github.com/syncthing/protocol/issues). + +## Contributing Code + +Every contribution is welcome. Following the points below will make this +a smoother process. + +Individuals making significant and valuable contributions are given +commit-access to the project. If you make a significant contribution and +are not considered for commit-access, please contact any of the +Syncthing core team members. + +All nontrivial contributions should go through the pull request +mechanism for internal review. Determining what is "nontrivial" is left +at the discretion of the contributor. + +### Authorship + +All code authors are listed in the AUTHORS file. Commits must be made +with the same name and email as listed in the AUTHORS file. To +accomplish this, ensure that your git configuration is set correctly +prior to making your first commit; + + $ git config --global user.name "Jane Doe" + $ git config --global user.email janedoe@example.com + +You must be reachable on the given email address. If you do not wish to +use your real name for whatever reason, using a nickname or pseudonym is +perfectly acceptable. + +## Coding Style + +- Follow the conventions laid out in [Effective Go](https://golang.org/doc/effective_go.html) + as much as makes sense. + +- All text files use Unix line endings. + +- Each commit should be `go fmt` clean. + +- The commit message subject should be a single short sentence + describing the change, starting with a capital letter. + +- Commits that resolve an existing issue must include the issue number + as `(fixes #123)` at the end of the commit message subject. + +- Imports are grouped per `goimports` standard; that is, standard + library first, then third party libraries after a blank line. + +- A contribution solving a single issue or introducing a single new + feature should probably be a single commit based on the current + `master` branch. You may be asked to "rebase" or "squash" your pull + request to make sure this is the case, especially if there have been + amendments during review. + +## Licensing + +All contributions are made under the same GPL license as the rest of the +project, except documentation, user interface text and translation +strings which are licensed under the Creative Commons Attribution 4.0 +International License. You retain the copyright to code you have +written. + +When accepting your first contribution, the maintainer of the project +will ensure that you are added to the AUTHORS file. You are welcome to +add yourself as a separate commit in your first pull request. + +## Tests + +Yes please! + +## License + +GPLv3 diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..94a9ed024 --- /dev/null +++ b/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/README.md b/README.md new file mode 100644 index 000000000..f7dcf8dfb --- /dev/null +++ b/README.md @@ -0,0 +1,13 @@ +The BEPv1 Protocol +================== + +[![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/protocol.svg?style=flat-square)](http://build.syncthing.net/job/protocol/lastBuild/) +[![API Documentation](http://img.shields.io/badge/api-Godoc-blue.svg?style=flat-square)](http://godoc.org/github.com/syncthing/protocol) +[![GPL License](http://img.shields.io/badge/license-GPL-blue.svg?style=flat-square)](http://opensource.org/licenses/GPL-3.0) + +This is the protocol implementation used by Syncthing. + +License +======= + +GPLv3 From 4833b6085c5b152914620d5fa70e548fdd019ca7 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 13 Jan 2015 13:20:29 +0100 Subject: [PATCH 27/61] The luhn package moved --- deviceid.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deviceid.go b/deviceid.go index 93753d7fa..21b323255 100644 --- a/deviceid.go +++ b/deviceid.go @@ -24,7 +24,7 @@ import ( "regexp" "strings" - "github.com/syncthing/syncthing/internal/luhn" + "github.com/calmh/luhn" ) type DeviceID [32]byte From d84a8e64043f8d6c41cc8d6b7d5ab31c0a25b4c2 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 13 Jan 2015 13:31:14 +0100 Subject: [PATCH 28/61] Relicense as MIT --- AUTHORS | 2 +- CONTRIBUTING.md | 4 +- LICENSE | 693 ++--------------------------------------- README.md | 4 +- common_test.go | 15 +- counting.go | 15 +- debug.go | 15 +- deviceid.go | 15 +- deviceid_test.go | 15 +- doc.go | 15 +- header.go | 15 +- message.go | 15 +- nativemodel_darwin.go | 15 +- nativemodel_unix.go | 15 +- nativemodel_windows.go | 15 +- protocol.go | 15 +- protocol_test.go | 15 +- wireformat.go | 15 +- 18 files changed, 38 insertions(+), 875 deletions(-) diff --git a/AUTHORS b/AUTHORS index f10d40d01..d84404ee2 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,4 +1,4 @@ -# This is the official list of authors for copyright purposes. +# This is the official list of Protocol Authors for copyright purposes. Audrius Butkevicius Jakob Borg diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 856dde7d8..67e6a9c70 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -57,7 +57,7 @@ perfectly acceptable. ## Licensing -All contributions are made under the same GPL license as the rest of the +All contributions are made under the same MIT license as the rest of the project, except documentation, user interface text and translation strings which are licensed under the Creative Commons Attribution 4.0 International License. You retain the copyright to code you have @@ -73,4 +73,4 @@ Yes please! ## License -GPLv3 +MIT diff --git a/LICENSE b/LICENSE index 94a9ed024..6f6960a75 100644 --- a/LICENSE +++ b/LICENSE @@ -1,674 +1,19 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - Copyright (C) - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. +Copyright (C) 2014-2015 The Protocol Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +- The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index f7dcf8dfb..bcba44b42 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,11 @@ The BEPv1 Protocol [![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/protocol.svg?style=flat-square)](http://build.syncthing.net/job/protocol/lastBuild/) [![API Documentation](http://img.shields.io/badge/api-Godoc-blue.svg?style=flat-square)](http://godoc.org/github.com/syncthing/protocol) -[![GPL License](http://img.shields.io/badge/license-GPL-blue.svg?style=flat-square)](http://opensource.org/licenses/GPL-3.0) +[![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat-square)](http://opensource.org/licenses/MIT) This is the protocol implementation used by Syncthing. License ======= -GPLv3 +MIT diff --git a/common_test.go b/common_test.go index e38da00a8..f67fb4812 100644 --- a/common_test.go +++ b/common_test.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. package protocol diff --git a/counting.go b/counting.go index ac4aeaa9a..490b77fd1 100644 --- a/counting.go +++ b/counting.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. package protocol diff --git a/debug.go b/debug.go index 4664b0e23..435d7f5d2 100644 --- a/debug.go +++ b/debug.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. package protocol diff --git a/deviceid.go b/deviceid.go index 21b323255..f3b3c5a31 100644 --- a/deviceid.go +++ b/deviceid.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. package protocol diff --git a/deviceid_test.go b/deviceid_test.go index da398c6fb..613557d32 100644 --- a/deviceid_test.go +++ b/deviceid_test.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. package protocol diff --git a/doc.go b/doc.go index 4f73e6c86..2c6ea8ef2 100644 --- a/doc.go +++ b/doc.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. // Package protocol implements the Block Exchange Protocol. package protocol diff --git a/header.go b/header.go index 8da092e5c..846ee48cd 100644 --- a/header.go +++ b/header.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. package protocol diff --git a/message.go b/message.go index 0d8de8070..d85f21cc6 100644 --- a/message.go +++ b/message.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. //go:generate -command genxdr go run ../../Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go //go:generate genxdr -o message_xdr.go message.go diff --git a/nativemodel_darwin.go b/nativemodel_darwin.go index ba30d1ae1..5b4b9be67 100644 --- a/nativemodel_darwin.go +++ b/nativemodel_darwin.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. // +build darwin diff --git a/nativemodel_unix.go b/nativemodel_unix.go index bf6499e63..2fb1654c7 100644 --- a/nativemodel_unix.go +++ b/nativemodel_unix.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. // +build !windows,!darwin diff --git a/nativemodel_windows.go b/nativemodel_windows.go index d2c079bdc..d4feea326 100644 --- a/nativemodel_windows.go +++ b/nativemodel_windows.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. // +build windows diff --git a/protocol.go b/protocol.go index 414704286..7cf59af1f 100644 --- a/protocol.go +++ b/protocol.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. package protocol diff --git a/protocol_test.go b/protocol_test.go index 75e0086ee..1ccb4525f 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. package protocol diff --git a/wireformat.go b/wireformat.go index 84da71423..4eab3d37e 100644 --- a/wireformat.go +++ b/wireformat.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. package protocol From f76b5d800208131015b14cfc1cfd3e6dd1d5e979 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 13 Jan 2015 13:47:16 +0100 Subject: [PATCH 29/61] rm '.gitignore' --- .gitignore | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .gitignore diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 2211df63d..000000000 --- a/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.txt From 3450b5f80cbf69777693cb4762685817ee2489ad Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Sun, 18 Jan 2015 01:26:52 +0100 Subject: [PATCH 30/61] Integer type policy Integers are for numbers, enabling arithmetic like subtractions and for loops without getting shot in the foot. Unsigneds are for bitfields. - "int" for numbers that will always be laughably smaller than four billion, and where we don't care about the serialization format. - "int32" for numbers that will always be laughably smaller than four billion, and will be serialized to four bytes. - "int64" for numbers that may approach four billion or will be serialized to eight bytes. - "uint32" and "uint64" for bitfields, depending on required number of bits and serialization format. Likewise "uint8" and "uint16", although rare in this project since they don't exist in XDR. - "int8", "int16" and plain "uint" are almost never useful. --- counting.go | 32 ++++++++++++++++---------------- message.go | 17 ++++++++--------- message_xdr.go | 48 ++++++++++++++++++++++++------------------------ protocol.go | 8 ++++---- 4 files changed, 52 insertions(+), 53 deletions(-) diff --git a/counting.go b/counting.go index 490b77fd1..d441ed311 100644 --- a/counting.go +++ b/counting.go @@ -10,25 +10,25 @@ import ( type countingReader struct { io.Reader - tot uint64 // bytes - last int64 // unix nanos + tot int64 // bytes + last int64 // unix nanos } var ( - totalIncoming uint64 - totalOutgoing uint64 + totalIncoming int64 + totalOutgoing int64 ) func (c *countingReader) Read(bs []byte) (int, error) { n, err := c.Reader.Read(bs) - atomic.AddUint64(&c.tot, uint64(n)) - atomic.AddUint64(&totalIncoming, uint64(n)) + atomic.AddInt64(&c.tot, int64(n)) + atomic.AddInt64(&totalIncoming, int64(n)) atomic.StoreInt64(&c.last, time.Now().UnixNano()) return n, err } -func (c *countingReader) Tot() uint64 { - return atomic.LoadUint64(&c.tot) +func (c *countingReader) Tot() int64 { + return atomic.LoadInt64(&c.tot) } func (c *countingReader) Last() time.Time { @@ -37,26 +37,26 @@ func (c *countingReader) Last() time.Time { type countingWriter struct { io.Writer - tot uint64 // bytes - last int64 // unix nanos + tot int64 // bytes + last int64 // unix nanos } func (c *countingWriter) Write(bs []byte) (int, error) { n, err := c.Writer.Write(bs) - atomic.AddUint64(&c.tot, uint64(n)) - atomic.AddUint64(&totalOutgoing, uint64(n)) + atomic.AddInt64(&c.tot, int64(n)) + atomic.AddInt64(&totalOutgoing, int64(n)) atomic.StoreInt64(&c.last, time.Now().UnixNano()) return n, err } -func (c *countingWriter) Tot() uint64 { - return atomic.LoadUint64(&c.tot) +func (c *countingWriter) Tot() int64 { + return atomic.LoadInt64(&c.tot) } func (c *countingWriter) Last() time.Time { return time.Unix(0, atomic.LoadInt64(&c.last)) } -func TotalInOut() (uint64, uint64) { - return atomic.LoadUint64(&totalIncoming), atomic.LoadUint64(&totalOutgoing) +func TotalInOut() (int64, int64) { + return atomic.LoadInt64(&totalIncoming), atomic.LoadInt64(&totalOutgoing) } diff --git a/message.go b/message.go index d85f21cc6..dbaf526ac 100644 --- a/message.go +++ b/message.go @@ -1,6 +1,5 @@ // Copyright (C) 2014 The Protocol Authors. -//go:generate -command genxdr go run ../../Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go //go:generate genxdr -o message_xdr.go message.go package protocol @@ -18,8 +17,8 @@ type FileInfo struct { Name string // max:8192 Flags uint32 Modified int64 - Version uint64 - LocalVersion uint64 + Version int64 + LocalVersion int64 Blocks []BlockInfo } @@ -60,7 +59,7 @@ func (f FileInfo) HasPermissionBits() bool { type BlockInfo struct { Offset int64 // noencode (cache only) - Size uint32 + Size int32 Hash []byte // max:64 } @@ -71,8 +70,8 @@ func (b BlockInfo) String() string { type RequestMessage struct { Folder string // max:64 Name string // max:8192 - Offset uint64 - Size uint32 + Offset int64 + Size int32 Hash []byte // max:64 Flags uint32 Options []Option // max:64 @@ -80,7 +79,7 @@ type RequestMessage struct { type ResponseMessage struct { Data []byte - Error uint32 + Error int32 } type ClusterConfigMessage struct { @@ -107,7 +106,7 @@ type Folder struct { type Device struct { ID []byte // max:32 Flags uint32 - MaxLocalVersion uint64 + MaxLocalVersion int64 } type Option struct { @@ -117,7 +116,7 @@ type Option struct { type CloseMessage struct { Reason string // max:1024 - Code uint32 + Code int32 } type EmptyMessage struct{} diff --git a/message_xdr.go b/message_xdr.go index fa7b5802b..54d030da1 100644 --- a/message_xdr.go +++ b/message_xdr.go @@ -168,8 +168,8 @@ struct FileInfo { string Name<8192>; unsigned int Flags; hyper Modified; - unsigned hyper Version; - unsigned hyper LocalVersion; + hyper Version; + hyper LocalVersion; BlockInfo Blocks<>; } @@ -206,8 +206,8 @@ func (o FileInfo) encodeXDR(xw *xdr.Writer) (int, error) { xw.WriteString(o.Name) xw.WriteUint32(o.Flags) xw.WriteUint64(uint64(o.Modified)) - xw.WriteUint64(o.Version) - xw.WriteUint64(o.LocalVersion) + xw.WriteUint64(uint64(o.Version)) + xw.WriteUint64(uint64(o.LocalVersion)) xw.WriteUint32(uint32(len(o.Blocks))) for i := range o.Blocks { _, err := o.Blocks[i].encodeXDR(xw) @@ -233,8 +233,8 @@ func (o *FileInfo) decodeXDR(xr *xdr.Reader) error { o.Name = xr.ReadStringMax(8192) o.Flags = xr.ReadUint32() o.Modified = int64(xr.ReadUint64()) - o.Version = xr.ReadUint64() - o.LocalVersion = xr.ReadUint64() + o.Version = int64(xr.ReadUint64()) + o.LocalVersion = int64(xr.ReadUint64()) _BlocksSize := int(xr.ReadUint32()) o.Blocks = make([]BlockInfo, _BlocksSize) for i := range o.Blocks { @@ -261,7 +261,7 @@ BlockInfo Structure: struct BlockInfo { - unsigned int Size; + int Size; opaque Hash<64>; } @@ -292,7 +292,7 @@ func (o BlockInfo) AppendXDR(bs []byte) ([]byte, error) { } func (o BlockInfo) encodeXDR(xw *xdr.Writer) (int, error) { - xw.WriteUint32(o.Size) + xw.WriteUint32(uint32(o.Size)) if l := len(o.Hash); l > 64 { return xw.Tot(), xdr.ElementSizeExceeded("Hash", l, 64) } @@ -312,7 +312,7 @@ func (o *BlockInfo) UnmarshalXDR(bs []byte) error { } func (o *BlockInfo) decodeXDR(xr *xdr.Reader) error { - o.Size = xr.ReadUint32() + o.Size = int32(xr.ReadUint32()) o.Hash = xr.ReadBytesMax(64) return xr.Error() } @@ -361,8 +361,8 @@ RequestMessage Structure: struct RequestMessage { string Folder<64>; string Name<8192>; - unsigned hyper Offset; - unsigned int Size; + hyper Offset; + int Size; opaque Hash<64>; unsigned int Flags; Option Options<64>; @@ -403,8 +403,8 @@ func (o RequestMessage) encodeXDR(xw *xdr.Writer) (int, error) { return xw.Tot(), xdr.ElementSizeExceeded("Name", l, 8192) } xw.WriteString(o.Name) - xw.WriteUint64(o.Offset) - xw.WriteUint32(o.Size) + xw.WriteUint64(uint64(o.Offset)) + xw.WriteUint32(uint32(o.Size)) if l := len(o.Hash); l > 64 { return xw.Tot(), xdr.ElementSizeExceeded("Hash", l, 64) } @@ -437,8 +437,8 @@ func (o *RequestMessage) UnmarshalXDR(bs []byte) error { func (o *RequestMessage) decodeXDR(xr *xdr.Reader) error { o.Folder = xr.ReadStringMax(64) o.Name = xr.ReadStringMax(8192) - o.Offset = xr.ReadUint64() - o.Size = xr.ReadUint32() + o.Offset = int64(xr.ReadUint64()) + o.Size = int32(xr.ReadUint32()) o.Hash = xr.ReadBytesMax(64) o.Flags = xr.ReadUint32() _OptionsSize := int(xr.ReadUint32()) @@ -471,7 +471,7 @@ ResponseMessage Structure: struct ResponseMessage { opaque Data<>; - unsigned int Error; + int Error; } */ @@ -502,7 +502,7 @@ func (o ResponseMessage) AppendXDR(bs []byte) ([]byte, error) { func (o ResponseMessage) encodeXDR(xw *xdr.Writer) (int, error) { xw.WriteBytes(o.Data) - xw.WriteUint32(o.Error) + xw.WriteUint32(uint32(o.Error)) return xw.Tot(), xw.Error() } @@ -519,7 +519,7 @@ func (o *ResponseMessage) UnmarshalXDR(bs []byte) error { func (o *ResponseMessage) decodeXDR(xr *xdr.Reader) error { o.Data = xr.ReadBytes() - o.Error = xr.ReadUint32() + o.Error = int32(xr.ReadUint32()) return xr.Error() } @@ -766,7 +766,7 @@ Device Structure: struct Device { opaque ID<32>; unsigned int Flags; - unsigned hyper MaxLocalVersion; + hyper MaxLocalVersion; } */ @@ -801,7 +801,7 @@ func (o Device) encodeXDR(xw *xdr.Writer) (int, error) { } xw.WriteBytes(o.ID) xw.WriteUint32(o.Flags) - xw.WriteUint64(o.MaxLocalVersion) + xw.WriteUint64(uint64(o.MaxLocalVersion)) return xw.Tot(), xw.Error() } @@ -819,7 +819,7 @@ func (o *Device) UnmarshalXDR(bs []byte) error { func (o *Device) decodeXDR(xr *xdr.Reader) error { o.ID = xr.ReadBytesMax(32) o.Flags = xr.ReadUint32() - o.MaxLocalVersion = xr.ReadUint64() + o.MaxLocalVersion = int64(xr.ReadUint64()) return xr.Error() } @@ -923,7 +923,7 @@ CloseMessage Structure: struct CloseMessage { string Reason<1024>; - unsigned int Code; + int Code; } */ @@ -957,7 +957,7 @@ func (o CloseMessage) encodeXDR(xw *xdr.Writer) (int, error) { return xw.Tot(), xdr.ElementSizeExceeded("Reason", l, 1024) } xw.WriteString(o.Reason) - xw.WriteUint32(o.Code) + xw.WriteUint32(uint32(o.Code)) return xw.Tot(), xw.Error() } @@ -974,7 +974,7 @@ func (o *CloseMessage) UnmarshalXDR(bs []byte) error { func (o *CloseMessage) decodeXDR(xr *xdr.Reader) error { o.Reason = xr.ReadStringMax(1024) - o.Code = xr.ReadUint32() + o.Code = int32(xr.ReadUint32()) return xr.Error() } diff --git a/protocol.go b/protocol.go index 7cf59af1f..eeaf0f46b 100644 --- a/protocol.go +++ b/protocol.go @@ -222,8 +222,8 @@ func (c *rawConnection) Request(folder string, name string, offset int64, size i ok := c.send(id, messageTypeRequest, RequestMessage{ Folder: folder, Name: name, - Offset: uint64(offset), - Size: uint32(size), + Offset: offset, + Size: int32(size), }) if !ok { return nil, ErrClosed @@ -706,8 +706,8 @@ func (c *rawConnection) pingerLoop() { type Statistics struct { At time.Time - InBytesTotal uint64 - OutBytesTotal uint64 + InBytesTotal int64 + OutBytesTotal int64 } func (c *rawConnection) Statistics() Statistics { From aba915037fd1b129c258a26e45fab5e8f2c4e7bf Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Fri, 23 Jan 2015 22:25:34 +0000 Subject: [PATCH 31/61] Add FlagsAll bit mask --- protocol.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/protocol.go b/protocol.go index eeaf0f46b..7f1dd4598 100644 --- a/protocol.go +++ b/protocol.go @@ -43,6 +43,8 @@ const ( FlagSymlink = 1 << 16 FlagSymlinkMissingTarget = 1 << 17 + FlagsAll = (1 << iota) - 1 + SymlinkTypeMask = FlagDirectory | FlagSymlinkMissingTarget ) From 35f0e355bf93b3d3dd5737f908a01c31465e18b1 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Thu, 12 Feb 2015 21:59:33 +0000 Subject: [PATCH 32/61] We are not using iota --- protocol.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protocol.go b/protocol.go index 7f1dd4598..f5c34dbff 100644 --- a/protocol.go +++ b/protocol.go @@ -43,7 +43,7 @@ const ( FlagSymlink = 1 << 16 FlagSymlinkMissingTarget = 1 << 17 - FlagsAll = (1 << iota) - 1 + FlagsAll = (1 << 18) - 1 SymlinkTypeMask = FlagDirectory | FlagSymlinkMissingTarget ) From a10c621e3340a9164221edef9ae27a8363ad28e3 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Sun, 8 Mar 2015 16:55:01 +0000 Subject: [PATCH 33/61] Remove 64 folder limit --- message.go | 4 ++-- message_xdr.go | 15 +++------------ 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/message.go b/message.go index dbaf526ac..9959fce30 100644 --- a/message.go +++ b/message.go @@ -7,7 +7,7 @@ package protocol import "fmt" type IndexMessage struct { - Folder string // max:64 + Folder string Files []FileInfo Flags uint32 Options []Option // max:64 @@ -85,7 +85,7 @@ type ResponseMessage struct { type ClusterConfigMessage struct { ClientName string // max:64 ClientVersion string // max:64 - Folders []Folder // max:64 + Folders []Folder Options []Option // max:64 } diff --git a/message_xdr.go b/message_xdr.go index 54d030da1..9e18f87b3 100644 --- a/message_xdr.go +++ b/message_xdr.go @@ -41,7 +41,7 @@ IndexMessage Structure: struct IndexMessage { - string Folder<64>; + string Folder<>; FileInfo Files<>; unsigned int Flags; Option Options<64>; @@ -74,9 +74,6 @@ func (o IndexMessage) AppendXDR(bs []byte) ([]byte, error) { } func (o IndexMessage) encodeXDR(xw *xdr.Writer) (int, error) { - if l := len(o.Folder); l > 64 { - return xw.Tot(), xdr.ElementSizeExceeded("Folder", l, 64) - } xw.WriteString(o.Folder) xw.WriteUint32(uint32(len(o.Files))) for i := range o.Files { @@ -111,7 +108,7 @@ func (o *IndexMessage) UnmarshalXDR(bs []byte) error { } func (o *IndexMessage) decodeXDR(xr *xdr.Reader) error { - o.Folder = xr.ReadStringMax(64) + o.Folder = xr.ReadString() _FilesSize := int(xr.ReadUint32()) o.Files = make([]FileInfo, _FilesSize) for i := range o.Files { @@ -559,7 +556,7 @@ ClusterConfigMessage Structure: struct ClusterConfigMessage { string ClientName<64>; string ClientVersion<64>; - Folder Folders<64>; + Folder Folders<>; Option Options<64>; } @@ -598,9 +595,6 @@ func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) { return xw.Tot(), xdr.ElementSizeExceeded("ClientVersion", l, 64) } xw.WriteString(o.ClientVersion) - if l := len(o.Folders); l > 64 { - return xw.Tot(), xdr.ElementSizeExceeded("Folders", l, 64) - } xw.WriteUint32(uint32(len(o.Folders))) for i := range o.Folders { _, err := o.Folders[i].encodeXDR(xw) @@ -636,9 +630,6 @@ func (o *ClusterConfigMessage) decodeXDR(xr *xdr.Reader) error { o.ClientName = xr.ReadStringMax(64) o.ClientVersion = xr.ReadStringMax(64) _FoldersSize := int(xr.ReadUint32()) - if _FoldersSize > 64 { - return xdr.ElementSizeExceeded("Folders", _FoldersSize, 64) - } o.Folders = make([]Folder, _FoldersSize) for i := range o.Folders { (&o.Folders[i]).decodeXDR(xr) From cd0cce4195105cefab80fce84664188b614032e8 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Mon, 9 Mar 2015 21:21:20 +0100 Subject: [PATCH 34/61] gofmt --- message.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/message.go b/message.go index 9959fce30..3b91161bc 100644 --- a/message.go +++ b/message.go @@ -83,8 +83,8 @@ type ResponseMessage struct { } type ClusterConfigMessage struct { - ClientName string // max:64 - ClientVersion string // max:64 + ClientName string // max:64 + ClientVersion string // max:64 Folders []Folder Options []Option // max:64 } From 108b4e2e104610bdf416f2f156f35ee769276caf Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Mon, 23 Feb 2015 09:30:47 +0100 Subject: [PATCH 35/61] Add more fine grained compression control --- compression.go | 53 +++++++++++++++++++++++++++++++++++++++++++++ compression_test.go | 51 +++++++++++++++++++++++++++++++++++++++++++ protocol.go | 38 +++++++++++++++++--------------- protocol_test.go | 20 ++++++++--------- 4 files changed, 135 insertions(+), 27 deletions(-) create mode 100644 compression.go create mode 100644 compression_test.go diff --git a/compression.go b/compression.go new file mode 100644 index 000000000..9e17213b6 --- /dev/null +++ b/compression.go @@ -0,0 +1,53 @@ +// Copyright (C) 2015 The Protocol Authors. + +package protocol + +import "fmt" + +type Compression int + +const ( + CompressMetadata Compression = iota // zero value is the default, default should be "metadata" + CompressNever + CompressAlways + + compressionThreshold = 128 // don't bother compressing messages smaller than this many bytes +) + +var compressionMarshal = map[Compression]string{ + CompressNever: "never", + CompressMetadata: "metadata", + CompressAlways: "always", +} + +var compressionUnmarshal = map[string]Compression{ + // Legacy + "false": CompressNever, + "true": CompressMetadata, + + // Current + "never": CompressNever, + "metadata": CompressMetadata, + "always": CompressAlways, +} + +func (c Compression) String() string { + s, ok := compressionMarshal[c] + if !ok { + return fmt.Sprintf("unknown:%d", c) + } + return s +} + +func (c Compression) GoString() string { + return fmt.Sprintf("%q", c.String()) +} + +func (c Compression) MarshalText() ([]byte, error) { + return []byte(compressionMarshal[c]), nil +} + +func (c *Compression) UnmarshalText(bs []byte) error { + *c = compressionUnmarshal[string(bs)] + return nil +} diff --git a/compression_test.go b/compression_test.go new file mode 100644 index 000000000..932297c32 --- /dev/null +++ b/compression_test.go @@ -0,0 +1,51 @@ +// Copyright (C) 2015 The Protocol Authors. + +package protocol + +import "testing" + +func TestCompressionMarshal(t *testing.T) { + uTestcases := []struct { + s string + c Compression + }{ + {"true", CompressMetadata}, + {"false", CompressNever}, + {"never", CompressNever}, + {"metadata", CompressMetadata}, + {"filedata", CompressFiledata}, + {"always", CompressAlways}, + {"whatever", CompressNever}, + } + + mTestcases := []struct { + s string + c Compression + }{ + {"never", CompressNever}, + {"metadata", CompressMetadata}, + {"filedata", CompressFiledata}, + {"always", CompressAlways}, + } + + var c Compression + for _, tc := range uTestcases { + err := c.UnmarshalText([]byte(tc.s)) + if err != nil { + t.Error(err) + } + if c != tc.c { + t.Errorf("%s unmarshalled to %d, not %d", tc.s, c, tc.c) + } + } + + for _, tc := range mTestcases { + bs, err := tc.c.MarshalText() + if err != nil { + t.Error(err) + } + if s := string(bs); s != tc.s { + t.Errorf("%d marshalled to %q, not %q", tc.c, s, tc.s) + } + } +} diff --git a/protocol.go b/protocol.go index f5c34dbff..e7b6fe275 100644 --- a/protocol.go +++ b/protocol.go @@ -106,7 +106,7 @@ type rawConnection struct { closed chan struct{} once sync.Once - compressionThreshold int // compress messages larger than this many bytes + compression Compression rdbuf0 []byte // used & reused by readMessage rdbuf1 []byte // used & reused by readMessage @@ -135,25 +135,21 @@ const ( pingIdleTime = 60 * time.Second ) -func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiver Model, name string, compress bool) Connection { +func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiver Model, name string, compress Compression) Connection { cr := &countingReader{Reader: reader} cw := &countingWriter{Writer: writer} - compThres := 1<<31 - 1 // compression disabled - if compress { - compThres = 128 // compress messages that are 128 bytes long or larger - } c := rawConnection{ - id: deviceID, - name: name, - receiver: nativeModel{receiver}, - state: stateInitial, - cr: cr, - cw: cw, - outbox: make(chan hdrMsg), - nextID: make(chan int), - closed: make(chan struct{}), - compressionThreshold: compThres, + id: deviceID, + name: name, + receiver: nativeModel{receiver}, + state: stateInitial, + cr: cr, + cw: cw, + outbox: make(chan hdrMsg), + nextID: make(chan int), + closed: make(chan struct{}), + compression: compress, } go c.readerLoop() @@ -571,7 +567,15 @@ func (c *rawConnection) writerLoop() { return } - if len(uncBuf) >= c.compressionThreshold { + compress := false + switch c.compression { + case CompressAlways: + compress = true + case CompressMetadata: + compress = hm.hdr.msgType != messageTypeResponse + } + + if compress && len(uncBuf) >= compressionThreshold { // Use compression for large messages hm.hdr.compression = true diff --git a/protocol_test.go b/protocol_test.go index 1ccb4525f..c1048cdcf 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -67,8 +67,8 @@ func TestPing(t *testing.T) { ar, aw := io.Pipe() br, bw := io.Pipe() - c0 := NewConnection(c0ID, ar, bw, nil, "name", true).(wireFormatConnection).next.(*rawConnection) - c1 := NewConnection(c1ID, br, aw, nil, "name", true).(wireFormatConnection).next.(*rawConnection) + c0 := NewConnection(c0ID, ar, bw, nil, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + c1 := NewConnection(c1ID, br, aw, nil, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) if ok := c0.ping(); !ok { t.Error("c0 ping failed") @@ -91,8 +91,8 @@ func TestPingErr(t *testing.T) { eaw := &ErrPipe{PipeWriter: *aw, max: i, err: e} ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e} - c0 := NewConnection(c0ID, ar, ebw, m0, "name", true).(wireFormatConnection).next.(*rawConnection) - NewConnection(c1ID, br, eaw, m1, "name", true) + c0 := NewConnection(c0ID, ar, ebw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + NewConnection(c1ID, br, eaw, m1, "name", CompressAlways) res := c0.ping() if (i < 8 || j < 8) && res { @@ -167,8 +167,8 @@ func TestVersionErr(t *testing.T) { ar, aw := io.Pipe() br, bw := io.Pipe() - c0 := NewConnection(c0ID, ar, bw, m0, "name", true).(wireFormatConnection).next.(*rawConnection) - NewConnection(c1ID, br, aw, m1, "name", true) + c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + NewConnection(c1ID, br, aw, m1, "name", CompressAlways) w := xdr.NewWriter(c0.cw) w.WriteUint32(encodeHeader(header{ @@ -190,8 +190,8 @@ func TestTypeErr(t *testing.T) { ar, aw := io.Pipe() br, bw := io.Pipe() - c0 := NewConnection(c0ID, ar, bw, m0, "name", true).(wireFormatConnection).next.(*rawConnection) - NewConnection(c1ID, br, aw, m1, "name", true) + c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + NewConnection(c1ID, br, aw, m1, "name", CompressAlways) w := xdr.NewWriter(c0.cw) w.WriteUint32(encodeHeader(header{ @@ -213,8 +213,8 @@ func TestClose(t *testing.T) { ar, aw := io.Pipe() br, bw := io.Pipe() - c0 := NewConnection(c0ID, ar, bw, m0, "name", true).(wireFormatConnection).next.(*rawConnection) - NewConnection(c1ID, br, aw, m1, "name", true) + c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + NewConnection(c1ID, br, aw, m1, "name", CompressAlways) c0.close(nil) From 1a4398cc55c8fe82a964097eaf59f2475b020a49 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Wed, 11 Mar 2015 21:10:44 +0100 Subject: [PATCH 36/61] Tests should actually pass --- compression_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/compression_test.go b/compression_test.go index 932297c32..90312344c 100644 --- a/compression_test.go +++ b/compression_test.go @@ -13,9 +13,8 @@ func TestCompressionMarshal(t *testing.T) { {"false", CompressNever}, {"never", CompressNever}, {"metadata", CompressMetadata}, - {"filedata", CompressFiledata}, {"always", CompressAlways}, - {"whatever", CompressNever}, + {"whatever", CompressMetadata}, } mTestcases := []struct { @@ -24,7 +23,6 @@ func TestCompressionMarshal(t *testing.T) { }{ {"never", CompressNever}, {"metadata", CompressMetadata}, - {"filedata", CompressFiledata}, {"always", CompressAlways}, } From d2ec40bb67846f34d3c1e59714351127a2e869e9 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Wed, 25 Mar 2015 21:20:04 +0100 Subject: [PATCH 37/61] Add flags and options for future extensibility --- message.go | 5 +++- message_xdr.go | 65 ++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/message.go b/message.go index 3b91161bc..f6ed9e1a8 100644 --- a/message.go +++ b/message.go @@ -101,12 +101,15 @@ func (o *ClusterConfigMessage) GetOption(key string) string { type Folder struct { ID string // max:64 Devices []Device + Flags uint32 + Options []Option // max:64 } type Device struct { ID []byte // max:32 - Flags uint32 MaxLocalVersion int64 + Flags uint32 + Options []Option // max:64 } type Option struct { diff --git a/message_xdr.go b/message_xdr.go index 9e18f87b3..243aea879 100644 --- a/message_xdr.go +++ b/message_xdr.go @@ -664,11 +664,21 @@ Folder Structure: \ Zero or more Device Structures \ / / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Flags | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Number of Options | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Zero or more Option Structures \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ struct Folder { string ID<64>; Device Devices<>; + unsigned int Flags; + Option Options<64>; } */ @@ -709,6 +719,17 @@ func (o Folder) encodeXDR(xw *xdr.Writer) (int, error) { return xw.Tot(), err } } + xw.WriteUint32(o.Flags) + if l := len(o.Options); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("Options", l, 64) + } + xw.WriteUint32(uint32(len(o.Options))) + for i := range o.Options { + _, err := o.Options[i].encodeXDR(xw) + if err != nil { + return xw.Tot(), err + } + } return xw.Tot(), xw.Error() } @@ -730,6 +751,15 @@ func (o *Folder) decodeXDR(xr *xdr.Reader) error { for i := range o.Devices { (&o.Devices[i]).decodeXDR(xr) } + o.Flags = xr.ReadUint32() + _OptionsSize := int(xr.ReadUint32()) + if _OptionsSize > 64 { + return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) + } + o.Options = make([]Option, _OptionsSize) + for i := range o.Options { + (&o.Options[i]).decodeXDR(xr) + } return xr.Error() } @@ -746,18 +776,25 @@ Device Structure: \ ID (variable length) \ / / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Flags | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | | + Max Local Version (64 bits) + | | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Flags | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Number of Options | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Zero or more Option Structures \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ struct Device { opaque ID<32>; - unsigned int Flags; hyper MaxLocalVersion; + unsigned int Flags; + Option Options<64>; } */ @@ -791,8 +828,18 @@ func (o Device) encodeXDR(xw *xdr.Writer) (int, error) { return xw.Tot(), xdr.ElementSizeExceeded("ID", l, 32) } xw.WriteBytes(o.ID) - xw.WriteUint32(o.Flags) xw.WriteUint64(uint64(o.MaxLocalVersion)) + xw.WriteUint32(o.Flags) + if l := len(o.Options); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("Options", l, 64) + } + xw.WriteUint32(uint32(len(o.Options))) + for i := range o.Options { + _, err := o.Options[i].encodeXDR(xw) + if err != nil { + return xw.Tot(), err + } + } return xw.Tot(), xw.Error() } @@ -809,8 +856,16 @@ func (o *Device) UnmarshalXDR(bs []byte) error { func (o *Device) decodeXDR(xr *xdr.Reader) error { o.ID = xr.ReadBytesMax(32) - o.Flags = xr.ReadUint32() o.MaxLocalVersion = int64(xr.ReadUint64()) + o.Flags = xr.ReadUint32() + _OptionsSize := int(xr.ReadUint32()) + if _OptionsSize > 64 { + return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) + } + o.Options = make([]Option, _OptionsSize) + for i := range o.Options { + (&o.Options[i]).decodeXDR(xr) + } return xr.Error() } From f9132cae85dcda1caba2f4ba78996d348b00ac6c Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Fri, 20 Mar 2015 09:58:32 +0100 Subject: [PATCH 38/61] Implement version vectors --- deviceid.go | 6 + message.go | 4 +- message_xdr.go | 183 +++++++++++++++--------------- vector.go | 105 +++++++++++++++++ vector_compare.go | 89 +++++++++++++++ vector_compare_test.go | 249 +++++++++++++++++++++++++++++++++++++++++ vector_test.go | 122 ++++++++++++++++++++ vector_xdr.go | 38 +++++++ 8 files changed, 704 insertions(+), 92 deletions(-) create mode 100644 vector.go create mode 100644 vector_compare.go create mode 100644 vector_compare_test.go create mode 100644 vector_test.go create mode 100644 vector_xdr.go diff --git a/deviceid.go b/deviceid.go index f3b3c5a31..2e0334a6a 100644 --- a/deviceid.go +++ b/deviceid.go @@ -6,6 +6,7 @@ import ( "bytes" "crypto/sha256" "encoding/base32" + "encoding/binary" "errors" "fmt" "regexp" @@ -67,6 +68,11 @@ func (n DeviceID) Equals(other DeviceID) bool { return bytes.Compare(n[:], other[:]) == 0 } +// Short returns an integer representing bits 0-63 of the device ID. +func (n DeviceID) Short() uint64 { + return binary.BigEndian.Uint64(n[:]) +} + func (n *DeviceID) MarshalText() ([]byte, error) { return []byte(n.String()), nil } diff --git a/message.go b/message.go index f6ed9e1a8..91c331903 100644 --- a/message.go +++ b/message.go @@ -17,13 +17,13 @@ type FileInfo struct { Name string // max:8192 Flags uint32 Modified int64 - Version int64 + Version Vector LocalVersion int64 Blocks []BlockInfo } func (f FileInfo) String() string { - return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%d, Size:%d, Blocks:%v}", + return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%v, Size:%d, Blocks:%v}", f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.Blocks) } diff --git a/message_xdr.go b/message_xdr.go index 243aea879..95d72eb1a 100644 --- a/message_xdr.go +++ b/message_xdr.go @@ -51,7 +51,7 @@ struct IndexMessage { func (o IndexMessage) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) - return o.encodeXDR(xw) + return o.EncodeXDRInto(xw) } func (o IndexMessage) MarshalXDR() ([]byte, error) { @@ -69,15 +69,15 @@ func (o IndexMessage) MustMarshalXDR() []byte { func (o IndexMessage) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - _, err := o.encodeXDR(xw) + _, err := o.EncodeXDRInto(xw) return []byte(aw), err } -func (o IndexMessage) encodeXDR(xw *xdr.Writer) (int, error) { +func (o IndexMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) { xw.WriteString(o.Folder) xw.WriteUint32(uint32(len(o.Files))) for i := range o.Files { - _, err := o.Files[i].encodeXDR(xw) + _, err := o.Files[i].EncodeXDRInto(xw) if err != nil { return xw.Tot(), err } @@ -88,7 +88,7 @@ func (o IndexMessage) encodeXDR(xw *xdr.Writer) (int, error) { } xw.WriteUint32(uint32(len(o.Options))) for i := range o.Options { - _, err := o.Options[i].encodeXDR(xw) + _, err := o.Options[i].EncodeXDRInto(xw) if err != nil { return xw.Tot(), err } @@ -98,21 +98,21 @@ func (o IndexMessage) encodeXDR(xw *xdr.Writer) (int, error) { func (o *IndexMessage) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } func (o *IndexMessage) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } -func (o *IndexMessage) decodeXDR(xr *xdr.Reader) error { +func (o *IndexMessage) DecodeXDRFrom(xr *xdr.Reader) error { o.Folder = xr.ReadString() _FilesSize := int(xr.ReadUint32()) o.Files = make([]FileInfo, _FilesSize) for i := range o.Files { - (&o.Files[i]).decodeXDR(xr) + (&o.Files[i]).DecodeXDRFrom(xr) } o.Flags = xr.ReadUint32() _OptionsSize := int(xr.ReadUint32()) @@ -121,7 +121,7 @@ func (o *IndexMessage) decodeXDR(xr *xdr.Reader) error { } o.Options = make([]Option, _OptionsSize) for i := range o.Options { - (&o.Options[i]).decodeXDR(xr) + (&o.Options[i]).DecodeXDRFrom(xr) } return xr.Error() } @@ -145,9 +145,9 @@ FileInfo Structure: + Modified (64 bits) + | | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | -+ Version (64 bits) + -| | +/ / +\ Vector Structure \ +/ / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | | + Local Version (64 bits) + @@ -165,7 +165,7 @@ struct FileInfo { string Name<8192>; unsigned int Flags; hyper Modified; - hyper Version; + Vector Version; hyper LocalVersion; BlockInfo Blocks<>; } @@ -174,7 +174,7 @@ struct FileInfo { func (o FileInfo) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) - return o.encodeXDR(xw) + return o.EncodeXDRInto(xw) } func (o FileInfo) MarshalXDR() ([]byte, error) { @@ -192,22 +192,25 @@ func (o FileInfo) MustMarshalXDR() []byte { func (o FileInfo) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - _, err := o.encodeXDR(xw) + _, err := o.EncodeXDRInto(xw) return []byte(aw), err } -func (o FileInfo) encodeXDR(xw *xdr.Writer) (int, error) { +func (o FileInfo) EncodeXDRInto(xw *xdr.Writer) (int, error) { if l := len(o.Name); l > 8192 { return xw.Tot(), xdr.ElementSizeExceeded("Name", l, 8192) } xw.WriteString(o.Name) xw.WriteUint32(o.Flags) xw.WriteUint64(uint64(o.Modified)) - xw.WriteUint64(uint64(o.Version)) + _, err := o.Version.EncodeXDRInto(xw) + if err != nil { + return xw.Tot(), err + } xw.WriteUint64(uint64(o.LocalVersion)) xw.WriteUint32(uint32(len(o.Blocks))) for i := range o.Blocks { - _, err := o.Blocks[i].encodeXDR(xw) + _, err := o.Blocks[i].EncodeXDRInto(xw) if err != nil { return xw.Tot(), err } @@ -217,25 +220,25 @@ func (o FileInfo) encodeXDR(xw *xdr.Writer) (int, error) { func (o *FileInfo) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } func (o *FileInfo) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } -func (o *FileInfo) decodeXDR(xr *xdr.Reader) error { +func (o *FileInfo) DecodeXDRFrom(xr *xdr.Reader) error { o.Name = xr.ReadStringMax(8192) o.Flags = xr.ReadUint32() o.Modified = int64(xr.ReadUint64()) - o.Version = int64(xr.ReadUint64()) + (&o.Version).DecodeXDRFrom(xr) o.LocalVersion = int64(xr.ReadUint64()) _BlocksSize := int(xr.ReadUint32()) o.Blocks = make([]BlockInfo, _BlocksSize) for i := range o.Blocks { - (&o.Blocks[i]).decodeXDR(xr) + (&o.Blocks[i]).DecodeXDRFrom(xr) } return xr.Error() } @@ -266,7 +269,7 @@ struct BlockInfo { func (o BlockInfo) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) - return o.encodeXDR(xw) + return o.EncodeXDRInto(xw) } func (o BlockInfo) MarshalXDR() ([]byte, error) { @@ -284,11 +287,11 @@ func (o BlockInfo) MustMarshalXDR() []byte { func (o BlockInfo) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - _, err := o.encodeXDR(xw) + _, err := o.EncodeXDRInto(xw) return []byte(aw), err } -func (o BlockInfo) encodeXDR(xw *xdr.Writer) (int, error) { +func (o BlockInfo) EncodeXDRInto(xw *xdr.Writer) (int, error) { xw.WriteUint32(uint32(o.Size)) if l := len(o.Hash); l > 64 { return xw.Tot(), xdr.ElementSizeExceeded("Hash", l, 64) @@ -299,16 +302,16 @@ func (o BlockInfo) encodeXDR(xw *xdr.Writer) (int, error) { func (o *BlockInfo) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } func (o *BlockInfo) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } -func (o *BlockInfo) decodeXDR(xr *xdr.Reader) error { +func (o *BlockInfo) DecodeXDRFrom(xr *xdr.Reader) error { o.Size = int32(xr.ReadUint32()) o.Hash = xr.ReadBytesMax(64) return xr.Error() @@ -369,7 +372,7 @@ struct RequestMessage { func (o RequestMessage) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) - return o.encodeXDR(xw) + return o.EncodeXDRInto(xw) } func (o RequestMessage) MarshalXDR() ([]byte, error) { @@ -387,11 +390,11 @@ func (o RequestMessage) MustMarshalXDR() []byte { func (o RequestMessage) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - _, err := o.encodeXDR(xw) + _, err := o.EncodeXDRInto(xw) return []byte(aw), err } -func (o RequestMessage) encodeXDR(xw *xdr.Writer) (int, error) { +func (o RequestMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) { if l := len(o.Folder); l > 64 { return xw.Tot(), xdr.ElementSizeExceeded("Folder", l, 64) } @@ -412,7 +415,7 @@ func (o RequestMessage) encodeXDR(xw *xdr.Writer) (int, error) { } xw.WriteUint32(uint32(len(o.Options))) for i := range o.Options { - _, err := o.Options[i].encodeXDR(xw) + _, err := o.Options[i].EncodeXDRInto(xw) if err != nil { return xw.Tot(), err } @@ -422,16 +425,16 @@ func (o RequestMessage) encodeXDR(xw *xdr.Writer) (int, error) { func (o *RequestMessage) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } func (o *RequestMessage) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } -func (o *RequestMessage) decodeXDR(xr *xdr.Reader) error { +func (o *RequestMessage) DecodeXDRFrom(xr *xdr.Reader) error { o.Folder = xr.ReadStringMax(64) o.Name = xr.ReadStringMax(8192) o.Offset = int64(xr.ReadUint64()) @@ -444,7 +447,7 @@ func (o *RequestMessage) decodeXDR(xr *xdr.Reader) error { } o.Options = make([]Option, _OptionsSize) for i := range o.Options { - (&o.Options[i]).decodeXDR(xr) + (&o.Options[i]).DecodeXDRFrom(xr) } return xr.Error() } @@ -475,7 +478,7 @@ struct ResponseMessage { func (o ResponseMessage) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) - return o.encodeXDR(xw) + return o.EncodeXDRInto(xw) } func (o ResponseMessage) MarshalXDR() ([]byte, error) { @@ -493,11 +496,11 @@ func (o ResponseMessage) MustMarshalXDR() []byte { func (o ResponseMessage) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - _, err := o.encodeXDR(xw) + _, err := o.EncodeXDRInto(xw) return []byte(aw), err } -func (o ResponseMessage) encodeXDR(xw *xdr.Writer) (int, error) { +func (o ResponseMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) { xw.WriteBytes(o.Data) xw.WriteUint32(uint32(o.Error)) return xw.Tot(), xw.Error() @@ -505,16 +508,16 @@ func (o ResponseMessage) encodeXDR(xw *xdr.Writer) (int, error) { func (o *ResponseMessage) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } func (o *ResponseMessage) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } -func (o *ResponseMessage) decodeXDR(xr *xdr.Reader) error { +func (o *ResponseMessage) DecodeXDRFrom(xr *xdr.Reader) error { o.Data = xr.ReadBytes() o.Error = int32(xr.ReadUint32()) return xr.Error() @@ -564,7 +567,7 @@ struct ClusterConfigMessage { func (o ClusterConfigMessage) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) - return o.encodeXDR(xw) + return o.EncodeXDRInto(xw) } func (o ClusterConfigMessage) MarshalXDR() ([]byte, error) { @@ -582,11 +585,11 @@ func (o ClusterConfigMessage) MustMarshalXDR() []byte { func (o ClusterConfigMessage) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - _, err := o.encodeXDR(xw) + _, err := o.EncodeXDRInto(xw) return []byte(aw), err } -func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) { +func (o ClusterConfigMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) { if l := len(o.ClientName); l > 64 { return xw.Tot(), xdr.ElementSizeExceeded("ClientName", l, 64) } @@ -597,7 +600,7 @@ func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) { xw.WriteString(o.ClientVersion) xw.WriteUint32(uint32(len(o.Folders))) for i := range o.Folders { - _, err := o.Folders[i].encodeXDR(xw) + _, err := o.Folders[i].EncodeXDRInto(xw) if err != nil { return xw.Tot(), err } @@ -607,7 +610,7 @@ func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) { } xw.WriteUint32(uint32(len(o.Options))) for i := range o.Options { - _, err := o.Options[i].encodeXDR(xw) + _, err := o.Options[i].EncodeXDRInto(xw) if err != nil { return xw.Tot(), err } @@ -617,22 +620,22 @@ func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) { func (o *ClusterConfigMessage) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } func (o *ClusterConfigMessage) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } -func (o *ClusterConfigMessage) decodeXDR(xr *xdr.Reader) error { +func (o *ClusterConfigMessage) DecodeXDRFrom(xr *xdr.Reader) error { o.ClientName = xr.ReadStringMax(64) o.ClientVersion = xr.ReadStringMax(64) _FoldersSize := int(xr.ReadUint32()) o.Folders = make([]Folder, _FoldersSize) for i := range o.Folders { - (&o.Folders[i]).decodeXDR(xr) + (&o.Folders[i]).DecodeXDRFrom(xr) } _OptionsSize := int(xr.ReadUint32()) if _OptionsSize > 64 { @@ -640,7 +643,7 @@ func (o *ClusterConfigMessage) decodeXDR(xr *xdr.Reader) error { } o.Options = make([]Option, _OptionsSize) for i := range o.Options { - (&o.Options[i]).decodeXDR(xr) + (&o.Options[i]).DecodeXDRFrom(xr) } return xr.Error() } @@ -685,7 +688,7 @@ struct Folder { func (o Folder) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) - return o.encodeXDR(xw) + return o.EncodeXDRInto(xw) } func (o Folder) MarshalXDR() ([]byte, error) { @@ -703,18 +706,18 @@ func (o Folder) MustMarshalXDR() []byte { func (o Folder) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - _, err := o.encodeXDR(xw) + _, err := o.EncodeXDRInto(xw) return []byte(aw), err } -func (o Folder) encodeXDR(xw *xdr.Writer) (int, error) { +func (o Folder) EncodeXDRInto(xw *xdr.Writer) (int, error) { if l := len(o.ID); l > 64 { return xw.Tot(), xdr.ElementSizeExceeded("ID", l, 64) } xw.WriteString(o.ID) xw.WriteUint32(uint32(len(o.Devices))) for i := range o.Devices { - _, err := o.Devices[i].encodeXDR(xw) + _, err := o.Devices[i].EncodeXDRInto(xw) if err != nil { return xw.Tot(), err } @@ -725,7 +728,7 @@ func (o Folder) encodeXDR(xw *xdr.Writer) (int, error) { } xw.WriteUint32(uint32(len(o.Options))) for i := range o.Options { - _, err := o.Options[i].encodeXDR(xw) + _, err := o.Options[i].EncodeXDRInto(xw) if err != nil { return xw.Tot(), err } @@ -735,21 +738,21 @@ func (o Folder) encodeXDR(xw *xdr.Writer) (int, error) { func (o *Folder) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } func (o *Folder) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } -func (o *Folder) decodeXDR(xr *xdr.Reader) error { +func (o *Folder) DecodeXDRFrom(xr *xdr.Reader) error { o.ID = xr.ReadStringMax(64) _DevicesSize := int(xr.ReadUint32()) o.Devices = make([]Device, _DevicesSize) for i := range o.Devices { - (&o.Devices[i]).decodeXDR(xr) + (&o.Devices[i]).DecodeXDRFrom(xr) } o.Flags = xr.ReadUint32() _OptionsSize := int(xr.ReadUint32()) @@ -758,7 +761,7 @@ func (o *Folder) decodeXDR(xr *xdr.Reader) error { } o.Options = make([]Option, _OptionsSize) for i := range o.Options { - (&o.Options[i]).decodeXDR(xr) + (&o.Options[i]).DecodeXDRFrom(xr) } return xr.Error() } @@ -801,7 +804,7 @@ struct Device { func (o Device) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) - return o.encodeXDR(xw) + return o.EncodeXDRInto(xw) } func (o Device) MarshalXDR() ([]byte, error) { @@ -819,11 +822,11 @@ func (o Device) MustMarshalXDR() []byte { func (o Device) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - _, err := o.encodeXDR(xw) + _, err := o.EncodeXDRInto(xw) return []byte(aw), err } -func (o Device) encodeXDR(xw *xdr.Writer) (int, error) { +func (o Device) EncodeXDRInto(xw *xdr.Writer) (int, error) { if l := len(o.ID); l > 32 { return xw.Tot(), xdr.ElementSizeExceeded("ID", l, 32) } @@ -835,7 +838,7 @@ func (o Device) encodeXDR(xw *xdr.Writer) (int, error) { } xw.WriteUint32(uint32(len(o.Options))) for i := range o.Options { - _, err := o.Options[i].encodeXDR(xw) + _, err := o.Options[i].EncodeXDRInto(xw) if err != nil { return xw.Tot(), err } @@ -845,16 +848,16 @@ func (o Device) encodeXDR(xw *xdr.Writer) (int, error) { func (o *Device) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } func (o *Device) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } -func (o *Device) decodeXDR(xr *xdr.Reader) error { +func (o *Device) DecodeXDRFrom(xr *xdr.Reader) error { o.ID = xr.ReadBytesMax(32) o.MaxLocalVersion = int64(xr.ReadUint64()) o.Flags = xr.ReadUint32() @@ -864,7 +867,7 @@ func (o *Device) decodeXDR(xr *xdr.Reader) error { } o.Options = make([]Option, _OptionsSize) for i := range o.Options { - (&o.Options[i]).decodeXDR(xr) + (&o.Options[i]).DecodeXDRFrom(xr) } return xr.Error() } @@ -899,7 +902,7 @@ struct Option { func (o Option) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) - return o.encodeXDR(xw) + return o.EncodeXDRInto(xw) } func (o Option) MarshalXDR() ([]byte, error) { @@ -917,11 +920,11 @@ func (o Option) MustMarshalXDR() []byte { func (o Option) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - _, err := o.encodeXDR(xw) + _, err := o.EncodeXDRInto(xw) return []byte(aw), err } -func (o Option) encodeXDR(xw *xdr.Writer) (int, error) { +func (o Option) EncodeXDRInto(xw *xdr.Writer) (int, error) { if l := len(o.Key); l > 64 { return xw.Tot(), xdr.ElementSizeExceeded("Key", l, 64) } @@ -935,16 +938,16 @@ func (o Option) encodeXDR(xw *xdr.Writer) (int, error) { func (o *Option) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } func (o *Option) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } -func (o *Option) decodeXDR(xr *xdr.Reader) error { +func (o *Option) DecodeXDRFrom(xr *xdr.Reader) error { o.Key = xr.ReadStringMax(64) o.Value = xr.ReadStringMax(1024) return xr.Error() @@ -976,7 +979,7 @@ struct CloseMessage { func (o CloseMessage) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) - return o.encodeXDR(xw) + return o.EncodeXDRInto(xw) } func (o CloseMessage) MarshalXDR() ([]byte, error) { @@ -994,11 +997,11 @@ func (o CloseMessage) MustMarshalXDR() []byte { func (o CloseMessage) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - _, err := o.encodeXDR(xw) + _, err := o.EncodeXDRInto(xw) return []byte(aw), err } -func (o CloseMessage) encodeXDR(xw *xdr.Writer) (int, error) { +func (o CloseMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) { if l := len(o.Reason); l > 1024 { return xw.Tot(), xdr.ElementSizeExceeded("Reason", l, 1024) } @@ -1009,16 +1012,16 @@ func (o CloseMessage) encodeXDR(xw *xdr.Writer) (int, error) { func (o *CloseMessage) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } func (o *CloseMessage) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } -func (o *CloseMessage) decodeXDR(xr *xdr.Reader) error { +func (o *CloseMessage) DecodeXDRFrom(xr *xdr.Reader) error { o.Reason = xr.ReadStringMax(1024) o.Code = int32(xr.ReadUint32()) return xr.Error() @@ -1040,7 +1043,7 @@ struct EmptyMessage { func (o EmptyMessage) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) - return o.encodeXDR(xw) + return o.EncodeXDRInto(xw) } func (o EmptyMessage) MarshalXDR() ([]byte, error) { @@ -1058,25 +1061,25 @@ func (o EmptyMessage) MustMarshalXDR() []byte { func (o EmptyMessage) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - _, err := o.encodeXDR(xw) + _, err := o.EncodeXDRInto(xw) return []byte(aw), err } -func (o EmptyMessage) encodeXDR(xw *xdr.Writer) (int, error) { +func (o EmptyMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) { return xw.Tot(), xw.Error() } func (o *EmptyMessage) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } func (o *EmptyMessage) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } -func (o *EmptyMessage) decodeXDR(xr *xdr.Reader) error { +func (o *EmptyMessage) DecodeXDRFrom(xr *xdr.Reader) error { return xr.Error() } diff --git a/vector.go b/vector.go new file mode 100644 index 000000000..048594522 --- /dev/null +++ b/vector.go @@ -0,0 +1,105 @@ +// Copyright (C) 2015 The Protocol Authors. + +package protocol + +// The Vector type represents a version vector. The zero value is a usable +// version vector. The vector has slice semantics and some operations on it +// are "append-like" in that they may return the same vector modified, or a +// new allocated Vector with the modified contents. +type Vector []Counter + +// Counter represents a single counter in the version vector. +type Counter struct { + ID uint64 + Value uint64 +} + +// Update returns a Vector with the index for the specific ID incremented by +// one. If it is possible, the vector v is updated and returned. If it is not, +// a copy will be created, updated and returned. +func (v Vector) Update(ID uint64) Vector { + for i := range v { + if v[i].ID == ID { + // Update an existing index + v[i].Value++ + return v + } else if v[i].ID > ID { + // Insert a new index + nv := make(Vector, len(v)+1) + copy(nv, v[:i]) + nv[i].ID = ID + nv[i].Value = 1 + copy(nv[i+1:], v[i:]) + return nv + } + } + // Append a new new index + return append(v, Counter{ID, 1}) +} + +// Merge returns the vector containing the maximum indexes from a and b. If it +// is possible, the vector a is updated and returned. If it is not, a copy +// will be created, updated and returned. +func (a Vector) Merge(b Vector) Vector { + var ai, bi int + for bi < len(b) { + if ai == len(a) { + // We've reach the end of a, all that remains are appends + return append(a, b[bi:]...) + } + + if a[ai].ID > b[bi].ID { + // The index from b should be inserted here + n := make(Vector, len(a)+1) + copy(n, a[:ai]) + n[ai] = b[bi] + copy(n[ai+1:], a[ai:]) + a = n + } + + if a[ai].ID == b[bi].ID { + if v := b[bi].Value; v > a[ai].Value { + a[ai].Value = v + } + } + + if bi < len(b) && a[ai].ID == b[bi].ID { + bi++ + } + ai++ + } + + return a +} + +// Copy returns an identical vector that is not shared with v. +func (v Vector) Copy() Vector { + nv := make(Vector, len(v)) + copy(nv, v) + return nv +} + +// Equal returns true when the two vectors are equivalent. +func (a Vector) Equal(b Vector) bool { + return a.Compare(b) == Equal +} + +// LesserEqual returns true when the two vectors are equivalent or a is Lesser +// than b. +func (a Vector) LesserEqual(b Vector) bool { + comp := a.Compare(b) + return comp == Lesser || comp == Equal +} + +// LesserEqual returns true when the two vectors are equivalent or a is Greater +// than b. +func (a Vector) GreaterEqual(b Vector) bool { + comp := a.Compare(b) + return comp == Greater || comp == Equal +} + +// Concurrent returns true when the two vectors are concrurrent. +func (a Vector) Concurrent(b Vector) bool { + comp := a.Compare(b) + return comp == ConcurrentGreater || comp == ConcurrentLesser +} diff --git a/vector_compare.go b/vector_compare.go new file mode 100644 index 000000000..9735ec9d1 --- /dev/null +++ b/vector_compare.go @@ -0,0 +1,89 @@ +// Copyright (C) 2015 The Protocol Authors. + +package protocol + +// Ordering represents the relationship between two Vectors. +type Ordering int + +const ( + Equal Ordering = iota + Greater + Lesser + ConcurrentLesser + ConcurrentGreater +) + +// There's really no such thing as "concurrent lesser" and "concurrent +// greater" in version vectors, just "concurrent". But it's useful to be able +// to get a strict ordering between versions for stable sorts and so on, so we +// return both variants. The convenience method Concurrent() can be used to +// check for either case. + +// Compare returns the Ordering that describes a's relation to b. +func (a Vector) Compare(b Vector) Ordering { + var ai, bi int // index into a and b + var av, bv Counter // value at current index + + result := Equal + + for ai < len(a) || bi < len(b) { + var aMissing, bMissing bool + + if ai < len(a) { + av = a[ai] + } else { + av = Counter{} + aMissing = true + } + + if bi < len(b) { + bv = b[bi] + } else { + bv = Counter{} + bMissing = true + } + + switch { + case av.ID == bv.ID: + // We have a counter value for each side + if av.Value > bv.Value { + if result == Lesser { + return ConcurrentLesser + } + result = Greater + } else if av.Value < bv.Value { + if result == Greater { + return ConcurrentGreater + } + result = Lesser + } + + case !aMissing && av.ID < bv.ID || bMissing: + // Value is missing on the b side + if av.Value > 0 { + if result == Lesser { + return ConcurrentLesser + } + result = Greater + } + + case !bMissing && bv.ID < av.ID || aMissing: + // Value is missing on the a side + if bv.Value > 0 { + if result == Greater { + return ConcurrentGreater + } + result = Lesser + } + } + + if ai < len(a) && (av.ID <= bv.ID || bMissing) { + ai++ + } + if bi < len(b) && (bv.ID <= av.ID || aMissing) { + bi++ + } + } + + return result +} diff --git a/vector_compare_test.go b/vector_compare_test.go new file mode 100644 index 000000000..78b6abe43 --- /dev/null +++ b/vector_compare_test.go @@ -0,0 +1,249 @@ +// Copyright (C) 2015 The Protocol Authors. + +package protocol + +import ( + "math" + "testing" +) + +func TestCompare(t *testing.T) { + testcases := []struct { + a, b Vector + r Ordering + }{ + // Empty vectors are identical + {Vector{}, Vector{}, Equal}, + {Vector{}, nil, Equal}, + {nil, Vector{}, Equal}, + {nil, Vector{Counter{42, 0}}, Equal}, + {Vector{}, Vector{Counter{42, 0}}, Equal}, + {Vector{Counter{42, 0}}, nil, Equal}, + {Vector{Counter{42, 0}}, Vector{}, Equal}, + + // Zero is the implied value for a missing Counter + { + Vector{Counter{42, 0}}, + Vector{Counter{77, 0}}, + Equal, + }, + + // Equal vectors are equal + { + Vector{Counter{42, 33}}, + Vector{Counter{42, 33}}, + Equal, + }, + { + Vector{Counter{42, 33}, Counter{77, 24}}, + Vector{Counter{42, 33}, Counter{77, 24}}, + Equal, + }, + + // These a-vectors are all greater than the b-vector + { + Vector{Counter{42, 1}}, + nil, + Greater, + }, + { + Vector{Counter{42, 1}}, + Vector{}, + Greater, + }, + { + Vector{Counter{0, 1}}, + Vector{Counter{0, 0}}, + Greater, + }, + { + Vector{Counter{42, 1}}, + Vector{Counter{42, 0}}, + Greater, + }, + { + Vector{Counter{math.MaxUint64, 1}}, + Vector{Counter{math.MaxUint64, 0}}, + Greater, + }, + { + Vector{Counter{0, math.MaxUint64}}, + Vector{Counter{0, 0}}, + Greater, + }, + { + Vector{Counter{42, math.MaxUint64}}, + Vector{Counter{42, 0}}, + Greater, + }, + { + Vector{Counter{math.MaxUint64, math.MaxUint64}}, + Vector{Counter{math.MaxUint64, 0}}, + Greater, + }, + { + Vector{Counter{0, math.MaxUint64}}, + Vector{Counter{0, math.MaxUint64 - 1}}, + Greater, + }, + { + Vector{Counter{42, math.MaxUint64}}, + Vector{Counter{42, math.MaxUint64 - 1}}, + Greater, + }, + { + Vector{Counter{math.MaxUint64, math.MaxUint64}}, + Vector{Counter{math.MaxUint64, math.MaxUint64 - 1}}, + Greater, + }, + { + Vector{Counter{42, 2}}, + Vector{Counter{42, 1}}, + Greater, + }, + { + Vector{Counter{22, 22}, Counter{42, 2}}, + Vector{Counter{22, 22}, Counter{42, 1}}, + Greater, + }, + { + Vector{Counter{42, 2}, Counter{77, 3}}, + Vector{Counter{42, 1}, Counter{77, 3}}, + Greater, + }, + { + Vector{Counter{22, 22}, Counter{42, 2}, Counter{77, 3}}, + Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}}, + Greater, + }, + { + Vector{Counter{22, 23}, Counter{42, 2}, Counter{77, 4}}, + Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}}, + Greater, + }, + + // These a-vectors are all lesser than the b-vector + {nil, Vector{Counter{42, 1}}, Lesser}, + {Vector{}, Vector{Counter{42, 1}}, Lesser}, + { + Vector{Counter{42, 0}}, + Vector{Counter{42, 1}}, + Lesser, + }, + { + Vector{Counter{42, 1}}, + Vector{Counter{42, 2}}, + Lesser, + }, + { + Vector{Counter{22, 22}, Counter{42, 1}}, + Vector{Counter{22, 22}, Counter{42, 2}}, + Lesser, + }, + { + Vector{Counter{42, 1}, Counter{77, 3}}, + Vector{Counter{42, 2}, Counter{77, 3}}, + Lesser, + }, + { + Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}}, + Vector{Counter{22, 22}, Counter{42, 2}, Counter{77, 3}}, + Lesser, + }, + { + Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}}, + Vector{Counter{22, 23}, Counter{42, 2}, Counter{77, 4}}, + Lesser, + }, + + // These are all in conflict + { + Vector{Counter{42, 2}}, + Vector{Counter{43, 1}}, + ConcurrentGreater, + }, + { + Vector{Counter{43, 1}}, + Vector{Counter{42, 2}}, + ConcurrentLesser, + }, + { + Vector{Counter{22, 23}, Counter{42, 1}}, + Vector{Counter{22, 22}, Counter{42, 2}}, + ConcurrentGreater, + }, + { + Vector{Counter{22, 21}, Counter{42, 2}}, + Vector{Counter{22, 22}, Counter{42, 1}}, + ConcurrentLesser, + }, + { + Vector{Counter{22, 21}, Counter{42, 2}, Counter{43, 1}}, + Vector{Counter{20, 1}, Counter{22, 22}, Counter{42, 1}}, + ConcurrentLesser, + }, + } + + for i, tc := range testcases { + // Test real Compare + if r := tc.a.Compare(tc.b); r != tc.r { + t.Errorf("%d: %+v.Compare(%+v) == %v (expected %v)", i, tc.a, tc.b, r, tc.r) + } + + // Test convenience functions + switch tc.r { + case Greater: + if tc.a.Equal(tc.b) { + t.Errorf("%+v == %+v", tc.a, tc.b) + } + if tc.a.Concurrent(tc.b) { + t.Errorf("%+v concurrent %+v", tc.a, tc.b) + } + if !tc.a.GreaterEqual(tc.b) { + t.Errorf("%+v not >= %+v", tc.a, tc.b) + } + if tc.a.LesserEqual(tc.b) { + t.Errorf("%+v <= %+v", tc.a, tc.b) + } + case Lesser: + if tc.a.Concurrent(tc.b) { + t.Errorf("%+v concurrent %+v", tc.a, tc.b) + } + if tc.a.Equal(tc.b) { + t.Errorf("%+v == %+v", tc.a, tc.b) + } + if tc.a.GreaterEqual(tc.b) { + t.Errorf("%+v >= %+v", tc.a, tc.b) + } + if !tc.a.LesserEqual(tc.b) { + t.Errorf("%+v not <= %+v", tc.a, tc.b) + } + case Equal: + if tc.a.Concurrent(tc.b) { + t.Errorf("%+v concurrent %+v", tc.a, tc.b) + } + if !tc.a.Equal(tc.b) { + t.Errorf("%+v not == %+v", tc.a, tc.b) + } + if !tc.a.GreaterEqual(tc.b) { + t.Errorf("%+v not <= %+v", tc.a, tc.b) + } + if !tc.a.LesserEqual(tc.b) { + t.Errorf("%+v not <= %+v", tc.a, tc.b) + } + case ConcurrentLesser, ConcurrentGreater: + if !tc.a.Concurrent(tc.b) { + t.Errorf("%+v not concurrent %+v", tc.a, tc.b) + } + if tc.a.Equal(tc.b) { + t.Errorf("%+v == %+v", tc.a, tc.b) + } + if tc.a.GreaterEqual(tc.b) { + t.Errorf("%+v >= %+v", tc.a, tc.b) + } + if tc.a.LesserEqual(tc.b) { + t.Errorf("%+v <= %+v", tc.a, tc.b) + } + } + } +} diff --git a/vector_test.go b/vector_test.go new file mode 100644 index 000000000..7815412c2 --- /dev/null +++ b/vector_test.go @@ -0,0 +1,122 @@ +// Copyright (C) 2015 The Protocol Authors. + +package protocol + +import "testing" + +func TestUpdate(t *testing.T) { + var v Vector + + // Append + + v = v.Update(42) + expected := Vector{Counter{42, 1}} + + if v.Compare(expected) != Equal { + t.Errorf("Update error, %+v != %+v", v, expected) + } + + // Insert at front + + v = v.Update(36) + expected = Vector{Counter{36, 1}, Counter{42, 1}} + + if v.Compare(expected) != Equal { + t.Errorf("Update error, %+v != %+v", v, expected) + } + + // Insert in moddle + + v = v.Update(37) + expected = Vector{Counter{36, 1}, Counter{37, 1}, Counter{42, 1}} + + if v.Compare(expected) != Equal { + t.Errorf("Update error, %+v != %+v", v, expected) + } + + // Update existing + + v = v.Update(37) + expected = Vector{Counter{36, 1}, Counter{37, 2}, Counter{42, 1}} + + if v.Compare(expected) != Equal { + t.Errorf("Update error, %+v != %+v", v, expected) + } +} + +func TestCopy(t *testing.T) { + v0 := Vector{Counter{42, 1}} + v1 := v0.Copy() + v1.Update(42) + if v0.Compare(v1) != Lesser { + t.Errorf("Copy error, %+v should be ancestor of %+v", v0, v1) + } +} + +func TestMerge(t *testing.T) { + testcases := []struct { + a, b, m Vector + }{ + // No-ops + { + Vector{}, + Vector{}, + Vector{}, + }, + { + Vector{Counter{22, 1}, Counter{42, 1}}, + Vector{Counter{22, 1}, Counter{42, 1}}, + Vector{Counter{22, 1}, Counter{42, 1}}, + }, + + // Appends + { + Vector{}, + Vector{Counter{22, 1}, Counter{42, 1}}, + Vector{Counter{22, 1}, Counter{42, 1}}, + }, + { + Vector{Counter{22, 1}}, + Vector{Counter{42, 1}}, + Vector{Counter{22, 1}, Counter{42, 1}}, + }, + { + Vector{Counter{22, 1}}, + Vector{Counter{22, 1}, Counter{42, 1}}, + Vector{Counter{22, 1}, Counter{42, 1}}, + }, + + // Insert + { + Vector{Counter{22, 1}, Counter{42, 1}}, + Vector{Counter{22, 1}, Counter{23, 2}, Counter{42, 1}}, + Vector{Counter{22, 1}, Counter{23, 2}, Counter{42, 1}}, + }, + { + Vector{Counter{42, 1}}, + Vector{Counter{22, 1}}, + Vector{Counter{22, 1}, Counter{42, 1}}, + }, + + // Update + { + Vector{Counter{22, 1}, Counter{42, 2}}, + Vector{Counter{22, 2}, Counter{42, 1}}, + Vector{Counter{22, 2}, Counter{42, 2}}, + }, + + // All of the above + { + Vector{Counter{10, 1}, Counter{20, 2}, Counter{30, 1}}, + Vector{Counter{5, 1}, Counter{10, 2}, Counter{15, 1}, Counter{20, 1}, Counter{25, 1}, Counter{35, 1}}, + Vector{Counter{5, 1}, Counter{10, 2}, Counter{15, 1}, Counter{20, 2}, Counter{25, 1}, Counter{30, 1}, Counter{35, 1}}, + }, + } + + for i, tc := range testcases { + if m := tc.a.Merge(tc.b); m.Compare(tc.m) != Equal { + t.Errorf("%d: %+v.Merge(%+v) == %+v (expected %+v)", i, tc.a, tc.b, m, tc.m) + } + } + +} diff --git a/vector_xdr.go b/vector_xdr.go new file mode 100644 index 000000000..a4b6b132b --- /dev/null +++ b/vector_xdr.go @@ -0,0 +1,38 @@ +// Copyright (C) 2015 The Protocol Authors. + +package protocol + +// This stuff is hacked up manually because genxdr doesn't support 'type +// Vector []Counter' declarations and it was tricky when I tried to add it... + +type xdrWriter interface { + WriteUint32(uint32) (int, error) + WriteUint64(uint64) (int, error) +} +type xdrReader interface { + ReadUint32() uint32 + ReadUint64() uint64 +} + +// EncodeXDRInto encodes the vector as an XDR object into the given XDR +// encoder. +func (v Vector) EncodeXDRInto(w xdrWriter) (int, error) { + w.WriteUint32(uint32(len(v))) + for i := range v { + w.WriteUint64(v[i].ID) + w.WriteUint64(v[i].Value) + } + return 4 + 16*len(v), nil +} + +// DecodeXDRFrom decodes the XDR objects from the given reader into itself. +func (v *Vector) DecodeXDRFrom(r xdrReader) error { + l := int(r.ReadUint32()) + n := make(Vector, l) + for i := range n { + n[i].ID = r.ReadUint64() + n[i].Value = r.ReadUint64() + } + *v = n + return nil +} From 1a59a5478f8c8bdb0c50043a2cfccaa597ee8374 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Sat, 24 Jan 2015 21:56:12 +0000 Subject: [PATCH 39/61] Expose hash, flags, options in Request --- common_test.go | 8 +++++++- nativemodel_darwin.go | 4 ++-- nativemodel_unix.go | 4 ++-- nativemodel_windows.go | 4 ++-- protocol.go | 19 +++++++++++-------- protocol_test.go | 2 +- wireformat.go | 4 ++-- 7 files changed, 27 insertions(+), 18 deletions(-) diff --git a/common_test.go b/common_test.go index f67fb4812..0f3795d5b 100644 --- a/common_test.go +++ b/common_test.go @@ -13,6 +13,9 @@ type TestModel struct { name string offset int64 size int + hash []byte + flags uint32 + options []Option closedCh chan bool } @@ -28,11 +31,14 @@ func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo) { func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) { } -func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, size int) ([]byte, error) { +func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { t.folder = folder t.name = name t.offset = offset t.size = size + t.hash = hash + t.flags = flags + t.options = options return t.data, nil } diff --git a/nativemodel_darwin.go b/nativemodel_darwin.go index 5b4b9be67..6001af694 100644 --- a/nativemodel_darwin.go +++ b/nativemodel_darwin.go @@ -26,9 +26,9 @@ func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileI m.next.IndexUpdate(deviceID, folder, files) } -func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error) { +func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { name = norm.NFD.String(name) - return m.next.Request(deviceID, folder, name, offset, size) + return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options) } func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { diff --git a/nativemodel_unix.go b/nativemodel_unix.go index 2fb1654c7..5f7b2e2c4 100644 --- a/nativemodel_unix.go +++ b/nativemodel_unix.go @@ -18,8 +18,8 @@ func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileI m.next.IndexUpdate(deviceID, folder, files) } -func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error) { - return m.next.Request(deviceID, folder, name, offset, size) +func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { + return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options) } func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { diff --git a/nativemodel_windows.go b/nativemodel_windows.go index d4feea326..4859cbaab 100644 --- a/nativemodel_windows.go +++ b/nativemodel_windows.go @@ -56,9 +56,9 @@ func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileI m.next.IndexUpdate(deviceID, folder, files) } -func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error) { +func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { name = filepath.FromSlash(name) - return m.next.Request(deviceID, folder, name, offset, size) + return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options) } func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { diff --git a/protocol.go b/protocol.go index e7b6fe275..7e2658775 100644 --- a/protocol.go +++ b/protocol.go @@ -70,7 +70,7 @@ type Model interface { // An index update was received from the peer device IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) // A request was made by the peer device - Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error) + Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) // A cluster configuration message was received ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) // The peer device closed the connection @@ -82,7 +82,7 @@ type Connection interface { Name() string Index(folder string, files []FileInfo) error IndexUpdate(folder string, files []FileInfo) error - Request(folder string, name string, offset int64, size int) ([]byte, error) + Request(folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) ClusterConfig(config ClusterConfigMessage) Statistics() Statistics } @@ -201,7 +201,7 @@ func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo) error { } // Request returns the bytes for the specified block after fetching them from the connected peer. -func (c *rawConnection) Request(folder string, name string, offset int64, size int) ([]byte, error) { +func (c *rawConnection) Request(folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { var id int select { case id = <-c.nextID: @@ -218,10 +218,13 @@ func (c *rawConnection) Request(folder string, name string, offset int64, size i c.awaitingMut.Unlock() ok := c.send(id, messageTypeRequest, RequestMessage{ - Folder: folder, - Name: name, - Offset: offset, - Size: int32(size), + Folder: folder, + Name: name, + Offset: offset, + Size: int32(size), + Hash: hash, + Flags: flags, + Options: options, }) if !ok { return nil, ErrClosed @@ -499,7 +502,7 @@ func filterIndexMessageFiles(fs []FileInfo) []FileInfo { } func (c *rawConnection) handleRequest(msgID int, req RequestMessage) { - data, _ := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), int(req.Size)) + data, _ := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), int(req.Size), req.Hash, req.Flags, req.Options) c.send(msgID, messageTypeResponse, ResponseMessage{ Data: data, diff --git a/protocol_test.go b/protocol_test.go index c1048cdcf..c660ba327 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -232,7 +232,7 @@ func TestClose(t *testing.T) { c0.Index("default", nil) c0.Index("default", nil) - if _, err := c0.Request("default", "foo", 0, 0); err == nil { + if _, err := c0.Request("default", "foo", 0, 0, nil, 0, nil); err == nil { t.Error("Request should return an error") } } diff --git a/wireformat.go b/wireformat.go index 4eab3d37e..23d347e1b 100644 --- a/wireformat.go +++ b/wireformat.go @@ -42,9 +42,9 @@ func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo) error { return c.next.IndexUpdate(folder, myFs) } -func (c wireFormatConnection) Request(folder, name string, offset int64, size int) ([]byte, error) { +func (c wireFormatConnection) Request(folder, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { name = norm.NFC.String(filepath.ToSlash(name)) - return c.next.Request(folder, name, offset, size) + return c.next.Request(folder, name, offset, size, hash, flags, options) } func (c wireFormatConnection) ClusterConfig(config ClusterConfigMessage) { From 1cb5875b200f07f0a768198c2425d59905957bae Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Wed, 4 Feb 2015 22:15:17 +0000 Subject: [PATCH 40/61] Expose flags, options in Index{,Update} --- common_test.go | 4 ++-- nativemodel_darwin.go | 8 ++++---- nativemodel_unix.go | 8 ++++---- nativemodel_windows.go | 8 ++++---- protocol.go | 32 ++++++++++++++++++-------------- protocol_test.go | 4 ++-- wireformat.go | 8 ++++---- 7 files changed, 38 insertions(+), 34 deletions(-) diff --git a/common_test.go b/common_test.go index 0f3795d5b..f46b6a8da 100644 --- a/common_test.go +++ b/common_test.go @@ -25,10 +25,10 @@ func newTestModel() *TestModel { } } -func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo) { +func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { } -func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) { +func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { } func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { diff --git a/nativemodel_darwin.go b/nativemodel_darwin.go index 6001af694..502a71f23 100644 --- a/nativemodel_darwin.go +++ b/nativemodel_darwin.go @@ -12,18 +12,18 @@ type nativeModel struct { next Model } -func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) { +func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { for i := range files { files[i].Name = norm.NFD.String(files[i].Name) } - m.next.Index(deviceID, folder, files) + m.next.Index(deviceID, folder, files, flags, options) } -func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) { +func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { for i := range files { files[i].Name = norm.NFD.String(files[i].Name) } - m.next.IndexUpdate(deviceID, folder, files) + m.next.IndexUpdate(deviceID, folder, files, flags, options) } func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { diff --git a/nativemodel_unix.go b/nativemodel_unix.go index 5f7b2e2c4..21585e308 100644 --- a/nativemodel_unix.go +++ b/nativemodel_unix.go @@ -10,12 +10,12 @@ type nativeModel struct { next Model } -func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) { - m.next.Index(deviceID, folder, files) +func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { + m.next.Index(deviceID, folder, files, flags, options) } -func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) { - m.next.IndexUpdate(deviceID, folder, files) +func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { + m.next.IndexUpdate(deviceID, folder, files, flags, options) } func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { diff --git a/nativemodel_windows.go b/nativemodel_windows.go index 4859cbaab..951f5b7e6 100644 --- a/nativemodel_windows.go +++ b/nativemodel_windows.go @@ -24,7 +24,7 @@ type nativeModel struct { next Model } -func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) { +func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { for i, f := range files { if strings.ContainsAny(f.Name, disallowedCharacters) { if f.IsDeleted() { @@ -37,10 +37,10 @@ func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) { } files[i].Name = filepath.FromSlash(f.Name) } - m.next.Index(deviceID, folder, files) + m.next.Index(deviceID, folder, files, flags, options) } -func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) { +func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { for i, f := range files { if strings.ContainsAny(f.Name, disallowedCharacters) { if f.IsDeleted() { @@ -53,7 +53,7 @@ func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileI } files[i].Name = filepath.FromSlash(files[i].Name) } - m.next.IndexUpdate(deviceID, folder, files) + m.next.IndexUpdate(deviceID, folder, files, flags, options) } func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { diff --git a/protocol.go b/protocol.go index 7e2658775..7d742a7fa 100644 --- a/protocol.go +++ b/protocol.go @@ -66,9 +66,9 @@ type pongMessage struct{ EmptyMessage } type Model interface { // An index was received from the peer device - Index(deviceID DeviceID, folder string, files []FileInfo) + Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) // An index update was received from the peer device - IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) + IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) // A request was made by the peer device Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) // A cluster configuration message was received @@ -80,8 +80,8 @@ type Model interface { type Connection interface { ID() DeviceID Name() string - Index(folder string, files []FileInfo) error - IndexUpdate(folder string, files []FileInfo) error + Index(folder string, files []FileInfo, flags uint32, options []Option) error + IndexUpdate(folder string, files []FileInfo, flags uint32, options []Option) error Request(folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) ClusterConfig(config ClusterConfigMessage) Statistics() Statistics @@ -169,7 +169,7 @@ func (c *rawConnection) Name() string { } // Index writes the list of file information to the connected peer device -func (c *rawConnection) Index(folder string, idx []FileInfo) error { +func (c *rawConnection) Index(folder string, idx []FileInfo, flags uint32, options []Option) error { select { case <-c.closed: return ErrClosed @@ -177,15 +177,17 @@ func (c *rawConnection) Index(folder string, idx []FileInfo) error { } c.idxMut.Lock() c.send(-1, messageTypeIndex, IndexMessage{ - Folder: folder, - Files: idx, + Folder: folder, + Files: idx, + Flags: flags, + Options: options, }) c.idxMut.Unlock() return nil } // IndexUpdate writes the list of file information to the connected peer device as an update -func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo) error { +func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo, flags uint32, options []Option) error { select { case <-c.closed: return ErrClosed @@ -193,8 +195,10 @@ func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo) error { } c.idxMut.Lock() c.send(-1, messageTypeIndexUpdate, IndexMessage{ - Folder: folder, - Files: idx, + Folder: folder, + Files: idx, + Flags: flags, + Options: options, }) c.idxMut.Unlock() return nil @@ -463,16 +467,16 @@ func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) { func (c *rawConnection) handleIndex(im IndexMessage) { if debug { - l.Debugf("Index(%v, %v, %d files)", c.id, im.Folder, len(im.Files)) + l.Debugf("Index(%v, %v, %d file, flags %x, opts: %s)", c.id, im.Folder, len(im.Files), im.Flags, im.Options) } - c.receiver.Index(c.id, im.Folder, filterIndexMessageFiles(im.Files)) + c.receiver.Index(c.id, im.Folder, filterIndexMessageFiles(im.Files), im.Flags, im.Options) } func (c *rawConnection) handleIndexUpdate(im IndexMessage) { if debug { - l.Debugf("queueing IndexUpdate(%v, %v, %d files)", c.id, im.Folder, len(im.Files)) + l.Debugf("queueing IndexUpdate(%v, %v, %d files, flags %x, opts: %s)", c.id, im.Folder, len(im.Files), im.Flags, im.Options) } - c.receiver.IndexUpdate(c.id, im.Folder, filterIndexMessageFiles(im.Files)) + c.receiver.IndexUpdate(c.id, im.Folder, filterIndexMessageFiles(im.Files), im.Flags, im.Options) } func filterIndexMessageFiles(fs []FileInfo) []FileInfo { diff --git a/protocol_test.go b/protocol_test.go index c660ba327..3ff1042cb 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -229,8 +229,8 @@ func TestClose(t *testing.T) { t.Error("Ping should not return true") } - c0.Index("default", nil) - c0.Index("default", nil) + c0.Index("default", nil, 0, nil) + c0.Index("default", nil, 0, nil) if _, err := c0.Request("default", "foo", 0, 0, nil, 0, nil); err == nil { t.Error("Request should return an error") diff --git a/wireformat.go b/wireformat.go index 23d347e1b..9411955ba 100644 --- a/wireformat.go +++ b/wireformat.go @@ -20,7 +20,7 @@ func (c wireFormatConnection) Name() string { return c.next.Name() } -func (c wireFormatConnection) Index(folder string, fs []FileInfo) error { +func (c wireFormatConnection) Index(folder string, fs []FileInfo, flags uint32, options []Option) error { var myFs = make([]FileInfo, len(fs)) copy(myFs, fs) @@ -28,10 +28,10 @@ func (c wireFormatConnection) Index(folder string, fs []FileInfo) error { myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name)) } - return c.next.Index(folder, myFs) + return c.next.Index(folder, myFs, flags, options) } -func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo) error { +func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo, flags uint32, options []Option) error { var myFs = make([]FileInfo, len(fs)) copy(myFs, fs) @@ -39,7 +39,7 @@ func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo) error { myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name)) } - return c.next.IndexUpdate(folder, myFs) + return c.next.IndexUpdate(folder, myFs, flags, options) } func (c wireFormatConnection) Request(folder, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { From fdf15f3ca323b647291e13b4970a0525926f9e4f Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Thu, 5 Feb 2015 23:01:17 +0000 Subject: [PATCH 41/61] Flag checking is now responsibility of the model --- protocol.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/protocol.go b/protocol.go index 7d742a7fa..48d43995a 100644 --- a/protocol.go +++ b/protocol.go @@ -287,11 +287,6 @@ func (c *rawConnection) readerLoop() (err error) { switch msg := msg.(type) { case IndexMessage: - if msg.Flags != 0 { - // We don't currently support or expect any flags. - return fmt.Errorf("protocol error: unknown flags 0x%x in Index(Update) message", msg.Flags) - } - switch hdr.msgType { case messageTypeIndex: if c.state < stateCCRcvd { @@ -308,10 +303,6 @@ func (c *rawConnection) readerLoop() (err error) { } case RequestMessage: - if msg.Flags != 0 { - // We don't currently support or expect any flags. - return fmt.Errorf("protocol error: unknown flags 0x%x in Request message", msg.Flags) - } if c.state < stateIdxRcvd { return fmt.Errorf("protocol error: request message in state %d", c.state) } From bf7fea9a0ac34db6c08db594e911b0dff2bd55e7 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Fri, 6 Feb 2015 21:34:51 +0000 Subject: [PATCH 42/61] Rename error to code, update xdr path --- message.go | 5 +++-- message_xdr.go | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/message.go b/message.go index 91c331903..c2d898944 100644 --- a/message.go +++ b/message.go @@ -1,5 +1,6 @@ // Copyright (C) 2014 The Protocol Authors. +//go:generate -command genxdr go run ../syncthing/Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go //go:generate genxdr -o message_xdr.go message.go package protocol @@ -78,8 +79,8 @@ type RequestMessage struct { } type ResponseMessage struct { - Data []byte - Error int32 + Data []byte + Code int32 } type ClusterConfigMessage struct { diff --git a/message_xdr.go b/message_xdr.go index 95d72eb1a..c179de769 100644 --- a/message_xdr.go +++ b/message_xdr.go @@ -465,13 +465,13 @@ ResponseMessage Structure: \ Data (variable length) \ / / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Error | +| Code | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ struct ResponseMessage { opaque Data<>; - int Error; + int Code; } */ @@ -502,7 +502,7 @@ func (o ResponseMessage) AppendXDR(bs []byte) ([]byte, error) { func (o ResponseMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) { xw.WriteBytes(o.Data) - xw.WriteUint32(uint32(o.Error)) + xw.WriteUint32(uint32(o.Code)) return xw.Tot(), xw.Error() } @@ -519,7 +519,7 @@ func (o *ResponseMessage) UnmarshalXDR(bs []byte) error { func (o *ResponseMessage) DecodeXDRFrom(xr *xdr.Reader) error { o.Data = xr.ReadBytes() - o.Error = int32(xr.ReadUint32()) + o.Code = int32(xr.ReadUint32()) return xr.Error() } From 34c2c1ec16729b4f1090db2a48893b129251e733 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Sun, 8 Feb 2015 11:04:01 +0000 Subject: [PATCH 43/61] Send and receive Request error codes --- errors.go | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ protocol.go | 5 +++-- 2 files changed, 54 insertions(+), 2 deletions(-) create mode 100755 errors.go diff --git a/errors.go b/errors.go new file mode 100755 index 000000000..31d27af0d --- /dev/null +++ b/errors.go @@ -0,0 +1,51 @@ +// Copyright (C) 2014 The Protocol Authors. + +package protocol + +import ( + "errors" +) + +const ( + ecNoError int32 = iota + ecGeneric + ecNoSuchFile + ecInvalid +) + +var ( + ErrNoError error = nil + ErrGeneric = errors.New("generic error") + ErrNoSuchFile = errors.New("no such file") + ErrInvalid = errors.New("file is invalid") +) + +var lookupError = map[int32]error{ + ecNoError: ErrNoError, + ecGeneric: ErrGeneric, + ecNoSuchFile: ErrNoSuchFile, + ecInvalid: ErrInvalid, +} + +var lookupCode = map[error]int32{ + ErrNoError: ecNoError, + ErrGeneric: ecGeneric, + ErrNoSuchFile: ecNoSuchFile, + ErrInvalid: ecInvalid, +} + +func codeToError(errcode int32) error { + err, ok := lookupError[errcode] + if !ok { + return ErrGeneric + } + return err +} + +func errorToCode(err error) int32 { + code, ok := lookupCode[err] + if !ok { + return ecGeneric + } + return code +} diff --git a/protocol.go b/protocol.go index 48d43995a..d46d70036 100644 --- a/protocol.go +++ b/protocol.go @@ -497,10 +497,11 @@ func filterIndexMessageFiles(fs []FileInfo) []FileInfo { } func (c *rawConnection) handleRequest(msgID int, req RequestMessage) { - data, _ := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), int(req.Size), req.Hash, req.Flags, req.Options) + data, err := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), int(req.Size), req.Hash, req.Flags, req.Options) c.send(msgID, messageTypeResponse, ResponseMessage{ Data: data, + Code: errorToCode(err), }) } @@ -508,7 +509,7 @@ func (c *rawConnection) handleResponse(msgID int, resp ResponseMessage) { c.awaitingMut.Lock() if rc := c.awaiting[msgID]; rc != nil { c.awaiting[msgID] = nil - rc <- asyncResult{resp.Data, nil} + rc <- asyncResult{resp.Data, codeToError(resp.Code)} close(rc) } c.awaitingMut.Unlock() From 1d76efcbcd24b408f4cdcea3f305aa9291985ba1 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Wed, 11 Feb 2015 22:11:44 +0000 Subject: [PATCH 44/61] Remove duplication --- nativemodel_windows.go | 41 +++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/nativemodel_windows.go b/nativemodel_windows.go index 951f5b7e6..072e1278b 100644 --- a/nativemodel_windows.go +++ b/nativemodel_windows.go @@ -25,34 +25,12 @@ type nativeModel struct { } func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { - for i, f := range files { - if strings.ContainsAny(f.Name, disallowedCharacters) { - if f.IsDeleted() { - // Don't complain if the file is marked as deleted, since it - // can't possibly exist here anyway. - continue - } - files[i].Flags |= FlagInvalid - l.Warnf("File name %q contains invalid characters; marked as invalid.", f.Name) - } - files[i].Name = filepath.FromSlash(f.Name) - } + fixupFiles(files) m.next.Index(deviceID, folder, files, flags, options) } func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { - for i, f := range files { - if strings.ContainsAny(f.Name, disallowedCharacters) { - if f.IsDeleted() { - // Don't complain if the file is marked as deleted, since it - // can't possibly exist here anyway. - continue - } - files[i].Flags |= FlagInvalid - l.Warnf("File name %q contains invalid characters; marked as invalid.", f.Name) - } - files[i].Name = filepath.FromSlash(files[i].Name) - } + fixupFiles(files) m.next.IndexUpdate(deviceID, folder, files, flags, options) } @@ -68,3 +46,18 @@ func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessag func (m nativeModel) Close(deviceID DeviceID, err error) { m.next.Close(deviceID, err) } + +func fixupFiles(files []FileInfo) { + for i, f := range files { + if strings.ContainsAny(f.Name, disallowedCharacters) { + if f.IsDeleted() { + // Don't complain if the file is marked as deleted, since it + // can't possibly exist here anyway. + continue + } + files[i].Flags |= FlagInvalid + l.Warnf("File name %q contains invalid characters; marked as invalid.", f.Name) + } + files[i].Name = filepath.FromSlash(files[i].Name) + } +} From aa9eda197930d3c6f64a680f3e92f95763c1ed63 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Fri, 13 Feb 2015 23:27:01 +0000 Subject: [PATCH 45/61] Add IndexTemporary and RequestTemporary flags --- protocol.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/protocol.go b/protocol.go index d46d70036..b9859203a 100644 --- a/protocol.go +++ b/protocol.go @@ -35,6 +35,7 @@ const ( stateIdxRcvd ) +// FileInfo flags const ( FlagDeleted uint32 = 1 << 12 FlagInvalid = 1 << 13 @@ -48,6 +49,17 @@ const ( SymlinkTypeMask = FlagDirectory | FlagSymlinkMissingTarget ) +// IndexMessage message flags (for IndexUpdate) +const ( + FlagIndexTemporary uint32 = 1 << iota +) + +// Request message flags +const ( + FlagRequestTemporary uint32 = 1 << iota +) + +// ClusterConfigMessage.Folders.Devices flags const ( FlagShareTrusted uint32 = 1 << 0 FlagShareReadOnly = 1 << 1 From 3d8a71fdb205fe2401a341a739208bc9d1e79a1b Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Wed, 8 Apr 2015 14:46:08 +0200 Subject: [PATCH 46/61] Generate with updated XDR package --- message_xdr.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/message_xdr.go b/message_xdr.go index c179de769..68d01b696 100644 --- a/message_xdr.go +++ b/message_xdr.go @@ -110,12 +110,18 @@ func (o *IndexMessage) UnmarshalXDR(bs []byte) error { func (o *IndexMessage) DecodeXDRFrom(xr *xdr.Reader) error { o.Folder = xr.ReadString() _FilesSize := int(xr.ReadUint32()) + if _FilesSize < 0 { + return xdr.ElementSizeExceeded("Files", _FilesSize, 0) + } o.Files = make([]FileInfo, _FilesSize) for i := range o.Files { (&o.Files[i]).DecodeXDRFrom(xr) } o.Flags = xr.ReadUint32() _OptionsSize := int(xr.ReadUint32()) + if _OptionsSize < 0 { + return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) + } if _OptionsSize > 64 { return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) } @@ -236,6 +242,9 @@ func (o *FileInfo) DecodeXDRFrom(xr *xdr.Reader) error { (&o.Version).DecodeXDRFrom(xr) o.LocalVersion = int64(xr.ReadUint64()) _BlocksSize := int(xr.ReadUint32()) + if _BlocksSize < 0 { + return xdr.ElementSizeExceeded("Blocks", _BlocksSize, 0) + } o.Blocks = make([]BlockInfo, _BlocksSize) for i := range o.Blocks { (&o.Blocks[i]).DecodeXDRFrom(xr) @@ -442,6 +451,9 @@ func (o *RequestMessage) DecodeXDRFrom(xr *xdr.Reader) error { o.Hash = xr.ReadBytesMax(64) o.Flags = xr.ReadUint32() _OptionsSize := int(xr.ReadUint32()) + if _OptionsSize < 0 { + return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) + } if _OptionsSize > 64 { return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) } @@ -633,11 +645,17 @@ func (o *ClusterConfigMessage) DecodeXDRFrom(xr *xdr.Reader) error { o.ClientName = xr.ReadStringMax(64) o.ClientVersion = xr.ReadStringMax(64) _FoldersSize := int(xr.ReadUint32()) + if _FoldersSize < 0 { + return xdr.ElementSizeExceeded("Folders", _FoldersSize, 0) + } o.Folders = make([]Folder, _FoldersSize) for i := range o.Folders { (&o.Folders[i]).DecodeXDRFrom(xr) } _OptionsSize := int(xr.ReadUint32()) + if _OptionsSize < 0 { + return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) + } if _OptionsSize > 64 { return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) } @@ -750,12 +768,18 @@ func (o *Folder) UnmarshalXDR(bs []byte) error { func (o *Folder) DecodeXDRFrom(xr *xdr.Reader) error { o.ID = xr.ReadStringMax(64) _DevicesSize := int(xr.ReadUint32()) + if _DevicesSize < 0 { + return xdr.ElementSizeExceeded("Devices", _DevicesSize, 0) + } o.Devices = make([]Device, _DevicesSize) for i := range o.Devices { (&o.Devices[i]).DecodeXDRFrom(xr) } o.Flags = xr.ReadUint32() _OptionsSize := int(xr.ReadUint32()) + if _OptionsSize < 0 { + return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) + } if _OptionsSize > 64 { return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) } @@ -862,6 +886,9 @@ func (o *Device) DecodeXDRFrom(xr *xdr.Reader) error { o.MaxLocalVersion = int64(xr.ReadUint64()) o.Flags = xr.ReadUint32() _OptionsSize := int(xr.ReadUint32()) + if _OptionsSize < 0 { + return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) + } if _OptionsSize > 64 { return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) } From e7db2648034fb71b051902a02bc25d4468ed492e Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Thu, 9 Apr 2015 12:51:21 +0200 Subject: [PATCH 47/61] Extract counter value from vector --- vector.go | 10 ++++++++++ vector_test.go | 14 +++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/vector.go b/vector.go index 048594522..edd156143 100644 --- a/vector.go +++ b/vector.go @@ -103,3 +103,13 @@ func (a Vector) Concurrent(b Vector) bool { comp := a.Compare(b) return comp == ConcurrentGreater || comp == ConcurrentLesser } + +// Counter returns the current value of the given counter ID. +func (v Vector) Counter(id uint64) uint64 { + for _, c := range v { + if c.ID == id { + return c.Value + } + } + return 0 +} diff --git a/vector_test.go b/vector_test.go index 7815412c2..c01255e7a 100644 --- a/vector_test.go +++ b/vector_test.go @@ -118,5 +118,17 @@ func TestMerge(t *testing.T) { t.Errorf("%d: %+v.Merge(%+v) == %+v (expected %+v)", i, tc.a, tc.b, m, tc.m) } } - +} + +func TestCounterValue(t *testing.T) { + v0 := Vector{Counter{42, 1}, Counter{64, 5}} + if v0.Counter(42) != 1 { + t.Error("Counter error, %d != %d", v0.Counter(42), 1) + } + if v0.Counter(64) != 5 { + t.Error("Counter error, %d != %d", v0.Counter(64), 5) + } + if v0.Counter(72) != 0 { + t.Error("Counter error, %d != %d", v0.Counter(72), 0) + } } From cbe44e1fff4bb67cf7761e1f5b2ec4501bb1aa26 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Fri, 26 Jun 2015 15:38:56 +0200 Subject: [PATCH 48/61] Enforce ClusterConfiguration at start, then no ordering --- protocol.go | 45 +++++++++++++++++++++++++-------------------- protocol_test.go | 30 +++++++++++++++++++++--------- 2 files changed, 46 insertions(+), 29 deletions(-) diff --git a/protocol.go b/protocol.go index b9859203a..605de4781 100644 --- a/protocol.go +++ b/protocol.go @@ -31,8 +31,7 @@ const ( const ( stateInitial = iota - stateCCRcvd - stateIdxRcvd + stateReady ) // FileInfo flags @@ -103,7 +102,6 @@ type rawConnection struct { id DeviceID name string receiver Model - state int cr *countingReader cw *countingWriter @@ -155,7 +153,6 @@ func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiv id: deviceID, name: name, receiver: nativeModel{receiver}, - state: stateInitial, cr: cr, cw: cw, outbox: make(chan hdrMsg), @@ -285,6 +282,7 @@ func (c *rawConnection) readerLoop() (err error) { c.close(err) }() + state := stateInitial for { select { case <-c.closed: @@ -298,47 +296,54 @@ func (c *rawConnection) readerLoop() (err error) { } switch msg := msg.(type) { + case ClusterConfigMessage: + if state != stateInitial { + return fmt.Errorf("protocol error: cluster config message in state %d", state) + } + go c.receiver.ClusterConfig(c.id, msg) + state = stateReady + case IndexMessage: switch hdr.msgType { case messageTypeIndex: - if c.state < stateCCRcvd { - return fmt.Errorf("protocol error: index message in state %d", c.state) + if state != stateReady { + return fmt.Errorf("protocol error: index message in state %d", state) } c.handleIndex(msg) - c.state = stateIdxRcvd + state = stateReady case messageTypeIndexUpdate: - if c.state < stateIdxRcvd { - return fmt.Errorf("protocol error: index update message in state %d", c.state) + if state != stateReady { + return fmt.Errorf("protocol error: index update message in state %d", state) } c.handleIndexUpdate(msg) + state = stateReady } case RequestMessage: - if c.state < stateIdxRcvd { - return fmt.Errorf("protocol error: request message in state %d", c.state) + if state != stateReady { + return fmt.Errorf("protocol error: request message in state %d", state) } // Requests are handled asynchronously go c.handleRequest(hdr.msgID, msg) case ResponseMessage: - if c.state < stateIdxRcvd { - return fmt.Errorf("protocol error: response message in state %d", c.state) + if state != stateReady { + return fmt.Errorf("protocol error: response message in state %d", state) } c.handleResponse(hdr.msgID, msg) case pingMessage: + if state != stateReady { + return fmt.Errorf("protocol error: ping message in state %d", state) + } c.send(hdr.msgID, messageTypePong, pongMessage{}) case pongMessage: - c.handlePong(hdr.msgID) - - case ClusterConfigMessage: - if c.state != stateInitial { - return fmt.Errorf("protocol error: cluster config message in state %d", c.state) + if state != stateReady { + return fmt.Errorf("protocol error: pong message in state %d", state) } - go c.receiver.ClusterConfig(c.id, msg) - c.state = stateCCRcvd + c.handlePong(hdr.msgID) case CloseMessage: return errors.New(msg.Reason) diff --git a/protocol_test.go b/protocol_test.go index 3ff1042cb..bb4fe7d95 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -67,8 +67,10 @@ func TestPing(t *testing.T) { ar, aw := io.Pipe() br, bw := io.Pipe() - c0 := NewConnection(c0ID, ar, bw, nil, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) - c1 := NewConnection(c1ID, br, aw, nil, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + c0 := NewConnection(c0ID, ar, bw, newTestModel(), "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + c1 := NewConnection(c1ID, br, aw, newTestModel(), "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + c0.ClusterConfig(ClusterConfigMessage{}) + c1.ClusterConfig(ClusterConfigMessage{}) if ok := c0.ping(); !ok { t.Error("c0 ping failed") @@ -81,8 +83,8 @@ func TestPing(t *testing.T) { func TestPingErr(t *testing.T) { e := errors.New("something broke") - for i := 0; i < 16; i++ { - for j := 0; j < 16; j++ { + for i := 0; i < 32; i++ { + for j := 0; j < 32; j++ { m0 := newTestModel() m1 := newTestModel() @@ -92,12 +94,16 @@ func TestPingErr(t *testing.T) { ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e} c0 := NewConnection(c0ID, ar, ebw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) - NewConnection(c1ID, br, eaw, m1, "name", CompressAlways) + c1 := NewConnection(c1ID, br, eaw, m1, "name", CompressAlways) + c0.ClusterConfig(ClusterConfigMessage{}) + c1.ClusterConfig(ClusterConfigMessage{}) res := c0.ping() if (i < 8 || j < 8) && res { + // This should have resulted in failure, as there is no way an empty ClusterConfig plus a Ping message fits in eight bytes. t.Errorf("Unexpected ping success; i=%d, j=%d", i, j) - } else if (i >= 12 && j >= 12) && !res { + } else if (i >= 28 && j >= 28) && !res { + // This should have worked though, as 28 bytes is plenty for both. t.Errorf("Unexpected ping fail; i=%d, j=%d", i, j) } } @@ -168,7 +174,9 @@ func TestVersionErr(t *testing.T) { br, bw := io.Pipe() c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) - NewConnection(c1ID, br, aw, m1, "name", CompressAlways) + c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways) + c0.ClusterConfig(ClusterConfigMessage{}) + c1.ClusterConfig(ClusterConfigMessage{}) w := xdr.NewWriter(c0.cw) w.WriteUint32(encodeHeader(header{ @@ -191,7 +199,9 @@ func TestTypeErr(t *testing.T) { br, bw := io.Pipe() c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) - NewConnection(c1ID, br, aw, m1, "name", CompressAlways) + c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways) + c0.ClusterConfig(ClusterConfigMessage{}) + c1.ClusterConfig(ClusterConfigMessage{}) w := xdr.NewWriter(c0.cw) w.WriteUint32(encodeHeader(header{ @@ -214,7 +224,9 @@ func TestClose(t *testing.T) { br, bw := io.Pipe() c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) - NewConnection(c1ID, br, aw, m1, "name", CompressAlways) + c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways) + c0.ClusterConfig(ClusterConfigMessage{}) + c1.ClusterConfig(ClusterConfigMessage{}) c0.close(nil) From 9f871a372629080e07662db3ccf4f075bdac7a6f Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Sat, 27 Jun 2015 11:07:44 +0100 Subject: [PATCH 49/61] Expose timeouts in protocol --- protocol.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/protocol.go b/protocol.go index 605de4781..50115c09c 100644 --- a/protocol.go +++ b/protocol.go @@ -140,9 +140,9 @@ type isEofer interface { IsEOF() bool } -const ( - pingTimeout = 30 * time.Second - pingIdleTime = 60 * time.Second +var ( + PingTimeout = 30 * time.Second + PingIdleTime = 60 * time.Second ) func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiver Model, name string, compress Compression) Connection { @@ -684,17 +684,17 @@ func (c *rawConnection) idGenerator() { func (c *rawConnection) pingerLoop() { var rc = make(chan bool, 1) - ticker := time.Tick(pingIdleTime / 2) + ticker := time.Tick(PingIdleTime / 2) for { select { case <-ticker: - if d := time.Since(c.cr.Last()); d < pingIdleTime { + if d := time.Since(c.cr.Last()); d < PingIdleTime { if debug { l.Debugln(c.id, "ping skipped after rd", d) } continue } - if d := time.Since(c.cw.Last()); d < pingIdleTime { + if d := time.Since(c.cw.Last()); d < PingIdleTime { if debug { l.Debugln(c.id, "ping skipped after wr", d) } @@ -714,7 +714,7 @@ func (c *rawConnection) pingerLoop() { if !ok { c.close(fmt.Errorf("ping failure")) } - case <-time.After(pingTimeout): + case <-time.After(PingTimeout): c.close(fmt.Errorf("ping timeout")) case <-c.closed: return From 9dd6f848bdcd3550606158a33d6aa98de6ea0cdc Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Thu, 9 Jul 2015 22:38:21 +0100 Subject: [PATCH 50/61] Name the folder in error messages --- nativemodel_windows.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nativemodel_windows.go b/nativemodel_windows.go index 072e1278b..f1a24898c 100644 --- a/nativemodel_windows.go +++ b/nativemodel_windows.go @@ -25,12 +25,12 @@ type nativeModel struct { } func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { - fixupFiles(files) + fixupFiles(folder, files) m.next.Index(deviceID, folder, files, flags, options) } func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { - fixupFiles(files) + fixupFiles(folder, files) m.next.IndexUpdate(deviceID, folder, files, flags, options) } @@ -47,7 +47,7 @@ func (m nativeModel) Close(deviceID DeviceID, err error) { m.next.Close(deviceID, err) } -func fixupFiles(files []FileInfo) { +func fixupFiles(folder string, files []FileInfo) { for i, f := range files { if strings.ContainsAny(f.Name, disallowedCharacters) { if f.IsDeleted() { @@ -56,7 +56,7 @@ func fixupFiles(files []FileInfo) { continue } files[i].Flags |= FlagInvalid - l.Warnf("File name %q contains invalid characters; marked as invalid.", f.Name) + l.Warnf("File name %q (folder %q) contains invalid characters; marked as invalid.", f.Name, folder) } files[i].Name = filepath.FromSlash(files[i].Name) } From b05c1a5bb9b30edb7691482befc2288f7fb7ce7b Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Fri, 10 Jul 2015 16:34:54 +1000 Subject: [PATCH 51/61] Connection now needs explicit Start() --- protocol.go | 9 +++++++-- protocol_test.go | 10 ++++++++++ wireformat.go | 4 ++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/protocol.go b/protocol.go index 50115c09c..d0e23055d 100644 --- a/protocol.go +++ b/protocol.go @@ -89,6 +89,7 @@ type Model interface { } type Connection interface { + Start() ID() DeviceID Name() string Index(folder string, files []FileInfo, flags uint32, options []Option) error @@ -161,12 +162,16 @@ func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiv compression: compress, } + return wireFormatConnection{&c} +} + +// Start creates the goroutines for sending and receiving of messages. It must +// be called exactly once after creating a connection. +func (c *rawConnection) Start() { go c.readerLoop() go c.writerLoop() go c.pingerLoop() go c.idGenerator() - - return wireFormatConnection{&c} } func (c *rawConnection) ID() DeviceID { diff --git a/protocol_test.go b/protocol_test.go index bb4fe7d95..051672411 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -68,7 +68,9 @@ func TestPing(t *testing.T) { br, bw := io.Pipe() c0 := NewConnection(c0ID, ar, bw, newTestModel(), "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + c0.Start() c1 := NewConnection(c1ID, br, aw, newTestModel(), "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + c1.Start() c0.ClusterConfig(ClusterConfigMessage{}) c1.ClusterConfig(ClusterConfigMessage{}) @@ -94,7 +96,9 @@ func TestPingErr(t *testing.T) { ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e} c0 := NewConnection(c0ID, ar, ebw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + c0.Start() c1 := NewConnection(c1ID, br, eaw, m1, "name", CompressAlways) + c1.Start() c0.ClusterConfig(ClusterConfigMessage{}) c1.ClusterConfig(ClusterConfigMessage{}) @@ -174,7 +178,9 @@ func TestVersionErr(t *testing.T) { br, bw := io.Pipe() c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + c0.Start() c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways) + c1.Start() c0.ClusterConfig(ClusterConfigMessage{}) c1.ClusterConfig(ClusterConfigMessage{}) @@ -199,7 +205,9 @@ func TestTypeErr(t *testing.T) { br, bw := io.Pipe() c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + c0.Start() c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways) + c1.Start() c0.ClusterConfig(ClusterConfigMessage{}) c1.ClusterConfig(ClusterConfigMessage{}) @@ -224,7 +232,9 @@ func TestClose(t *testing.T) { br, bw := io.Pipe() c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + c0.Start() c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways) + c1.Start() c0.ClusterConfig(ClusterConfigMessage{}) c1.ClusterConfig(ClusterConfigMessage{}) diff --git a/wireformat.go b/wireformat.go index 9411955ba..66b02ed6f 100644 --- a/wireformat.go +++ b/wireformat.go @@ -12,6 +12,10 @@ type wireFormatConnection struct { next Connection } +func (c wireFormatConnection) Start() { + c.next.Start() +} + func (c wireFormatConnection) ID() DeviceID { return c.next.ID() } From 516d88b0725588516d1ff75afe4dc5677b844342 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 21 Jul 2015 15:12:19 +0200 Subject: [PATCH 52/61] Base for better conflict resolution --- conflict_test.go | 23 +++++++++++++++++++++++ message.go | 25 +++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 conflict_test.go diff --git a/conflict_test.go b/conflict_test.go new file mode 100644 index 000000000..ef5c44d7e --- /dev/null +++ b/conflict_test.go @@ -0,0 +1,23 @@ +// Copyright (C) 2015 The Protocol Authors. + +package protocol + +import "testing" + +func TestWinsConflict(t *testing.T) { + testcases := [][2]FileInfo{ + // The first should always win over the second + {{Modified: 42}, {Modified: 41}}, + {{Modified: 41}, {Modified: 42, Flags: FlagDeleted}}, + {{Modified: 41, Version: Vector{{42, 2}, {43, 1}}}, {Modified: 41, Version: Vector{{42, 1}, {43, 2}}}}, + } + + for _, tc := range testcases { + if !tc[0].WinsConflict(tc[1]) { + t.Errorf("%v should win over %v", tc[0], tc[1]) + } + if tc[1].WinsConflict(tc[0]) { + t.Errorf("%v should not win over %v", tc[1], tc[0]) + } + } +} diff --git a/message.go b/message.go index c2d898944..49df7d4fa 100644 --- a/message.go +++ b/message.go @@ -58,6 +58,31 @@ func (f FileInfo) HasPermissionBits() bool { return f.Flags&FlagNoPermBits == 0 } +// WinsConflict returns true if "f" is the one to choose when it is in +// conflict with "other". +func (f FileInfo) WinsConflict(other FileInfo) bool { + // If a modification is in conflict with a delete, we pick the + // modification. + if !f.IsDeleted() && other.IsDeleted() { + return true + } + if f.IsDeleted() && !other.IsDeleted() { + return false + } + + // The one with the newer modification time wins. + if f.Modified > other.Modified { + return true + } + if f.Modified < other.Modified { + return false + } + + // The modification times were equal. Use the device ID in the version + // vector as tie breaker. + return f.Version.Compare(other.Version) == ConcurrentGreater +} + type BlockInfo struct { Offset int64 // noencode (cache only) Size int32 From ebcdea63c07327a342f65415bbadc497462b8f1f Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Wed, 29 Jul 2015 21:23:43 +0100 Subject: [PATCH 53/61] Use sync.Pool for response buffers --- common_test.go | 5 ++- nativemodel_darwin.go | 4 +- nativemodel_unix.go | 4 +- nativemodel_windows.go | 4 +- protocol.go | 91 ++++++++++++++++++++++++++++-------------- 5 files changed, 71 insertions(+), 37 deletions(-) diff --git a/common_test.go b/common_test.go index f46b6a8da..706a3b877 100644 --- a/common_test.go +++ b/common_test.go @@ -31,7 +31,7 @@ func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo, fl func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { } -func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { +func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, size int, hash []byte, flags uint32, options []Option, buf []byte) error { t.folder = folder t.name = name t.offset = offset @@ -39,7 +39,8 @@ func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64 t.hash = hash t.flags = flags t.options = options - return t.data, nil + copy(buf, t.data) + return nil } func (t *TestModel) Close(deviceID DeviceID, err error) { diff --git a/nativemodel_darwin.go b/nativemodel_darwin.go index 502a71f23..eb755a6e4 100644 --- a/nativemodel_darwin.go +++ b/nativemodel_darwin.go @@ -26,9 +26,9 @@ func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileI m.next.IndexUpdate(deviceID, folder, files, flags, options) } -func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { +func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error { name = norm.NFD.String(name) - return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options) + return m.next.Request(deviceID, folder, name, offset, hash, flags, options, buf) } func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { diff --git a/nativemodel_unix.go b/nativemodel_unix.go index 21585e308..0611865e1 100644 --- a/nativemodel_unix.go +++ b/nativemodel_unix.go @@ -18,8 +18,8 @@ func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileI m.next.IndexUpdate(deviceID, folder, files, flags, options) } -func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { - return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options) +func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error { + return m.next.Request(deviceID, folder, name, offset, hash, flags, options, buf) } func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { diff --git a/nativemodel_windows.go b/nativemodel_windows.go index f1a24898c..36a1d2749 100644 --- a/nativemodel_windows.go +++ b/nativemodel_windows.go @@ -34,9 +34,9 @@ func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileI m.next.IndexUpdate(deviceID, folder, files, flags, options) } -func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { +func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error { name = filepath.FromSlash(name) - return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options) + return m.next.Request(deviceID, folder, name, offset, hash, flags, options, buf) } func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { diff --git a/protocol.go b/protocol.go index d0e23055d..8b41c0138 100644 --- a/protocol.go +++ b/protocol.go @@ -81,7 +81,7 @@ type Model interface { // An index update was received from the peer device IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) // A request was made by the peer device - Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) + Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error // A cluster configuration message was received ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) // The peer device closed the connection @@ -112,11 +112,11 @@ type rawConnection struct { idxMut sync.Mutex // ensures serialization of Index calls - nextID chan int - outbox chan hdrMsg - closed chan struct{} - once sync.Once - + nextID chan int + outbox chan hdrMsg + closed chan struct{} + once sync.Once + pool sync.Pool compression Compression rdbuf0 []byte // used & reused by readMessage @@ -129,8 +129,9 @@ type asyncResult struct { } type hdrMsg struct { - hdr header - msg encodable + hdr header + msg encodable + done chan struct{} } type encodable interface { @@ -151,14 +152,19 @@ func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiv cw := &countingWriter{Writer: writer} c := rawConnection{ - id: deviceID, - name: name, - receiver: nativeModel{receiver}, - cr: cr, - cw: cw, - outbox: make(chan hdrMsg), - nextID: make(chan int), - closed: make(chan struct{}), + id: deviceID, + name: name, + receiver: nativeModel{receiver}, + cr: cr, + cw: cw, + outbox: make(chan hdrMsg), + nextID: make(chan int), + closed: make(chan struct{}), + pool: sync.Pool{ + New: func() interface{} { + return make([]byte, BlockSize) + }, + }, compression: compress, } @@ -195,7 +201,7 @@ func (c *rawConnection) Index(folder string, idx []FileInfo, flags uint32, optio Files: idx, Flags: flags, Options: options, - }) + }, nil) c.idxMut.Unlock() return nil } @@ -213,7 +219,7 @@ func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo, flags uint32, Files: idx, Flags: flags, Options: options, - }) + }, nil) c.idxMut.Unlock() return nil } @@ -243,7 +249,7 @@ func (c *rawConnection) Request(folder string, name string, offset int64, size i Hash: hash, Flags: flags, Options: options, - }) + }, nil) if !ok { return nil, ErrClosed } @@ -257,7 +263,7 @@ func (c *rawConnection) Request(folder string, name string, offset int64, size i // ClusterConfig send the cluster configuration message to the peer and returns any error func (c *rawConnection) ClusterConfig(config ClusterConfigMessage) { - c.send(-1, messageTypeClusterConfig, config) + c.send(-1, messageTypeClusterConfig, config, nil) } func (c *rawConnection) ping() bool { @@ -273,7 +279,7 @@ func (c *rawConnection) ping() bool { c.awaiting[id] = rc c.awaitingMut.Unlock() - ok := c.send(id, messageTypePing, nil) + ok := c.send(id, messageTypePing, nil, nil) if !ok { return false } @@ -342,7 +348,7 @@ func (c *rawConnection) readerLoop() (err error) { if state != stateReady { return fmt.Errorf("protocol error: ping message in state %d", state) } - c.send(hdr.msgID, messageTypePong, pongMessage{}) + c.send(hdr.msgID, messageTypePong, pongMessage{}, nil) case pongMessage: if state != stateReady { @@ -519,12 +525,36 @@ func filterIndexMessageFiles(fs []FileInfo) []FileInfo { } func (c *rawConnection) handleRequest(msgID int, req RequestMessage) { - data, err := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), int(req.Size), req.Hash, req.Flags, req.Options) + size := int(req.Size) + usePool := size <= BlockSize - c.send(msgID, messageTypeResponse, ResponseMessage{ - Data: data, - Code: errorToCode(err), - }) + var buf []byte + var done chan struct{} + + if usePool { + buf = c.pool.Get().([]byte)[:size] + done = make(chan struct{}) + } else { + buf = make([]byte, size) + } + + err := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), req.Hash, req.Flags, req.Options, buf) + if err != nil { + c.send(msgID, messageTypeResponse, ResponseMessage{ + Data: nil, + Code: errorToCode(err), + }, done) + } else { + c.send(msgID, messageTypeResponse, ResponseMessage{ + Data: buf, + Code: errorToCode(err), + }, done) + } + + if usePool { + <-done + c.pool.Put(buf) + } } func (c *rawConnection) handleResponse(msgID int, resp ResponseMessage) { @@ -547,7 +577,7 @@ func (c *rawConnection) handlePong(msgID int) { c.awaitingMut.Unlock() } -func (c *rawConnection) send(msgID int, msgType int, msg encodable) bool { +func (c *rawConnection) send(msgID int, msgType int, msg encodable, done chan struct{}) bool { if msgID < 0 { select { case id := <-c.nextID: @@ -564,7 +594,7 @@ func (c *rawConnection) send(msgID int, msgType int, msg encodable) bool { } select { - case c.outbox <- hdrMsg{hdr, msg}: + case c.outbox <- hdrMsg{hdr, msg, done}: return true case <-c.closed: return false @@ -583,6 +613,9 @@ func (c *rawConnection) writerLoop() { if hm.msg != nil { // Uncompressed message in uncBuf uncBuf, err = hm.msg.AppendXDR(uncBuf[:0]) + if hm.done != nil { + close(hm.done) + } if err != nil { c.close(err) return From a6936982794bf84da01bcea26d618a34708a7ea7 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Sun, 9 Aug 2015 10:00:28 +0200 Subject: [PATCH 54/61] Mend tests --- common_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common_test.go b/common_test.go index 706a3b877..8f1028078 100644 --- a/common_test.go +++ b/common_test.go @@ -31,11 +31,11 @@ func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo, fl func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { } -func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, size int, hash []byte, flags uint32, options []Option, buf []byte) error { +func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error { t.folder = folder t.name = name t.offset = offset - t.size = size + t.size = len(buf) t.hash = hash t.flags = flags t.options = options From c6f5075721db1a7fd1e770dac92a9ef9e6ef4959 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 18 Aug 2015 08:37:23 +0200 Subject: [PATCH 55/61] Enable testing with go-fuzz --- fuzz.go | 70 +++++++++++++++++++++++++++++++++++++++++ fuzz_test.go | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+) create mode 100644 fuzz.go create mode 100644 fuzz_test.go diff --git a/fuzz.go b/fuzz.go new file mode 100644 index 000000000..9b82abe7c --- /dev/null +++ b/fuzz.go @@ -0,0 +1,70 @@ +// Copyright (C) 2015 The Protocol Authors. + +// +build gofuzz + +package protocol + +import ( + "bytes" + "encoding/binary" + "encoding/hex" + "fmt" + "reflect" + "sync" +) + +func Fuzz(data []byte) int { + // Regenerate the length, or we'll most commonly exit quickly due to an + // unexpected eof which is unintestering. + if len(data) > 8 { + binary.BigEndian.PutUint32(data[4:], uint32(len(data))-8) + } + + // Setup a rawConnection we'll use to parse the message. + c := rawConnection{ + cr: &countingReader{Reader: bytes.NewReader(data)}, + closed: make(chan struct{}), + pool: sync.Pool{ + New: func() interface{} { + return make([]byte, BlockSize) + }, + }, + } + + // Attempt to parse the message. + hdr, msg, err := c.readMessage() + if err != nil { + return 0 + } + + // If parsing worked, attempt to encode it again. + newBs, err := msg.AppendXDR(nil) + if err != nil { + panic("not encodable") + } + + // Create an appriate header for the re-encoding. + newMsg := make([]byte, 8) + binary.BigEndian.PutUint32(newMsg, encodeHeader(hdr)) + binary.BigEndian.PutUint32(newMsg[4:], uint32(len(newBs))) + newMsg = append(newMsg, newBs...) + + // Use the rawConnection to parse the re-encoding. + c.cr = &countingReader{Reader: bytes.NewReader(newMsg)} + hdr2, msg2, err := c.readMessage() + if err != nil { + fmt.Println("Initial:\n" + hex.Dump(data)) + fmt.Println("New:\n" + hex.Dump(newMsg)) + panic("not parseable after re-encode: " + err.Error()) + } + + // Make sure the data is the same as it was before. + if hdr != hdr2 { + panic("headers differ") + } + if !reflect.DeepEqual(msg, msg2) { + panic("contents differ") + } + + return 1 +} diff --git a/fuzz_test.go b/fuzz_test.go new file mode 100644 index 000000000..65c2d9010 --- /dev/null +++ b/fuzz_test.go @@ -0,0 +1,89 @@ +// Copyright (C) 2015 The Protocol Authors. + +// +build gofuzz + +package protocol + +import ( + "encoding/binary" + "fmt" + "io/ioutil" + "os" + "strings" + "testing" + "testing/quick" +) + +// This can be used to generate a corpus of valid messages as a starting point +// for the fuzzer. +func TestGenerateCorpus(t *testing.T) { + t.Skip("Use to generate initial corpus only") + + n := 0 + check := func(idx IndexMessage) bool { + for i := range idx.Options { + if len(idx.Options[i].Key) > 64 { + idx.Options[i].Key = idx.Options[i].Key[:64] + } + } + hdr := header{ + version: 0, + msgID: 42, + msgType: messageTypeIndex, + compression: false, + } + + msgBs := idx.MustMarshalXDR() + + buf := make([]byte, 8) + binary.BigEndian.PutUint32(buf, encodeHeader(hdr)) + binary.BigEndian.PutUint32(buf[4:], uint32(len(msgBs))) + buf = append(buf, msgBs...) + + ioutil.WriteFile(fmt.Sprintf("testdata/corpus/test-%03d.xdr", n), buf, 0644) + n++ + return true + } + + if err := quick.Check(check, &quick.Config{MaxCount: 1000}); err != nil { + t.Fatal(err) + } +} + +// Tests any crashers found by the fuzzer, for closer investigation. +func TestCrashers(t *testing.T) { + testFiles(t, "testdata/crashers") +} + +// Tests the entire corpus, which should PASS before the fuzzer starts +// fuzzing. +func TestCorpus(t *testing.T) { + testFiles(t, "testdata/corpus") +} + +func testFiles(t *testing.T, dir string) { + fd, err := os.Open(dir) + if err != nil { + t.Fatal(err) + } + crashers, err := fd.Readdirnames(-1) + if err != nil { + t.Fatal(err) + } + for _, name := range crashers { + if strings.HasSuffix(name, ".output") { + continue + } + if strings.HasSuffix(name, ".quoted") { + continue + } + + t.Log(name) + crasher, err := ioutil.ReadFile(dir + "/" + name) + if err != nil { + t.Fatal(err) + } + + Fuzz(crasher) + } +} From f769df16e8b6ef2236183f8eab1b9ca22a94408c Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 18 Aug 2015 08:38:06 +0200 Subject: [PATCH 56/61] Reject unreasonably large messages We allocate a []byte to read the message into, so if the header says the messages is several gigabytes large we may run into trouble. In reality, a message should never be that large so we impose a limit. --- protocol.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/protocol.go b/protocol.go index 8b41c0138..420b20859 100644 --- a/protocol.go +++ b/protocol.go @@ -15,7 +15,11 @@ import ( ) const ( - BlockSize = 128 * 1024 + // Data block size (128 KiB) + BlockSize = 128 << 10 + + // We reject messages larger than this when encountered on the wire. (64 MiB) + MaxMessageLen = 64 << 20 ) const ( @@ -383,6 +387,11 @@ func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) { l.Debugf("read header %v (msglen=%d)", hdr, msglen) } + if msglen > MaxMessageLen { + err = fmt.Errorf("message length %d exceeds maximum %d", msglen, MaxMessageLen) + return + } + if hdr.version != 0 { err = fmt.Errorf("unknown protocol version 0x%x", hdr.version) return From 9c8b907ff195e46ac4ba0688104405f896104ccf Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 18 Aug 2015 08:41:34 +0200 Subject: [PATCH 57/61] All slice types must have limits The XDR unmarshaller allocates a []T when it sees a slice type and reads the expected length, so we must always limit the length in order to avoid allocating too much memory when encountering corruption. --- message.go | 14 +++++++------- message_xdr.go | 40 ++++++++++++++++++++++++++++++++-------- vector_xdr.go | 5 +++++ 3 files changed, 44 insertions(+), 15 deletions(-) diff --git a/message.go b/message.go index 49df7d4fa..0cfeaa381 100644 --- a/message.go +++ b/message.go @@ -9,7 +9,7 @@ import "fmt" type IndexMessage struct { Folder string - Files []FileInfo + Files []FileInfo // max:1000000 Flags uint32 Options []Option // max:64 } @@ -20,7 +20,7 @@ type FileInfo struct { Modified int64 Version Vector LocalVersion int64 - Blocks []BlockInfo + Blocks []BlockInfo // max:1000000 } func (f FileInfo) String() string { @@ -109,9 +109,9 @@ type ResponseMessage struct { } type ClusterConfigMessage struct { - ClientName string // max:64 - ClientVersion string // max:64 - Folders []Folder + ClientName string // max:64 + ClientVersion string // max:64 + Folders []Folder // max:1000000 Options []Option // max:64 } @@ -125,8 +125,8 @@ func (o *ClusterConfigMessage) GetOption(key string) string { } type Folder struct { - ID string // max:64 - Devices []Device + ID string // max:64 + Devices []Device // max:1000000 Flags uint32 Options []Option // max:64 } diff --git a/message_xdr.go b/message_xdr.go index 68d01b696..876fbb77c 100644 --- a/message_xdr.go +++ b/message_xdr.go @@ -42,7 +42,7 @@ IndexMessage Structure: struct IndexMessage { string Folder<>; - FileInfo Files<>; + FileInfo Files<1000000>; unsigned int Flags; Option Options<64>; } @@ -75,6 +75,9 @@ func (o IndexMessage) AppendXDR(bs []byte) ([]byte, error) { func (o IndexMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) { xw.WriteString(o.Folder) + if l := len(o.Files); l > 1000000 { + return xw.Tot(), xdr.ElementSizeExceeded("Files", l, 1000000) + } xw.WriteUint32(uint32(len(o.Files))) for i := range o.Files { _, err := o.Files[i].EncodeXDRInto(xw) @@ -111,7 +114,10 @@ func (o *IndexMessage) DecodeXDRFrom(xr *xdr.Reader) error { o.Folder = xr.ReadString() _FilesSize := int(xr.ReadUint32()) if _FilesSize < 0 { - return xdr.ElementSizeExceeded("Files", _FilesSize, 0) + return xdr.ElementSizeExceeded("Files", _FilesSize, 1000000) + } + if _FilesSize > 1000000 { + return xdr.ElementSizeExceeded("Files", _FilesSize, 1000000) } o.Files = make([]FileInfo, _FilesSize) for i := range o.Files { @@ -173,7 +179,7 @@ struct FileInfo { hyper Modified; Vector Version; hyper LocalVersion; - BlockInfo Blocks<>; + BlockInfo Blocks<1000000>; } */ @@ -214,6 +220,9 @@ func (o FileInfo) EncodeXDRInto(xw *xdr.Writer) (int, error) { return xw.Tot(), err } xw.WriteUint64(uint64(o.LocalVersion)) + if l := len(o.Blocks); l > 1000000 { + return xw.Tot(), xdr.ElementSizeExceeded("Blocks", l, 1000000) + } xw.WriteUint32(uint32(len(o.Blocks))) for i := range o.Blocks { _, err := o.Blocks[i].EncodeXDRInto(xw) @@ -243,7 +252,10 @@ func (o *FileInfo) DecodeXDRFrom(xr *xdr.Reader) error { o.LocalVersion = int64(xr.ReadUint64()) _BlocksSize := int(xr.ReadUint32()) if _BlocksSize < 0 { - return xdr.ElementSizeExceeded("Blocks", _BlocksSize, 0) + return xdr.ElementSizeExceeded("Blocks", _BlocksSize, 1000000) + } + if _BlocksSize > 1000000 { + return xdr.ElementSizeExceeded("Blocks", _BlocksSize, 1000000) } o.Blocks = make([]BlockInfo, _BlocksSize) for i := range o.Blocks { @@ -571,7 +583,7 @@ ClusterConfigMessage Structure: struct ClusterConfigMessage { string ClientName<64>; string ClientVersion<64>; - Folder Folders<>; + Folder Folders<1000000>; Option Options<64>; } @@ -610,6 +622,9 @@ func (o ClusterConfigMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) { return xw.Tot(), xdr.ElementSizeExceeded("ClientVersion", l, 64) } xw.WriteString(o.ClientVersion) + if l := len(o.Folders); l > 1000000 { + return xw.Tot(), xdr.ElementSizeExceeded("Folders", l, 1000000) + } xw.WriteUint32(uint32(len(o.Folders))) for i := range o.Folders { _, err := o.Folders[i].EncodeXDRInto(xw) @@ -646,7 +661,10 @@ func (o *ClusterConfigMessage) DecodeXDRFrom(xr *xdr.Reader) error { o.ClientVersion = xr.ReadStringMax(64) _FoldersSize := int(xr.ReadUint32()) if _FoldersSize < 0 { - return xdr.ElementSizeExceeded("Folders", _FoldersSize, 0) + return xdr.ElementSizeExceeded("Folders", _FoldersSize, 1000000) + } + if _FoldersSize > 1000000 { + return xdr.ElementSizeExceeded("Folders", _FoldersSize, 1000000) } o.Folders = make([]Folder, _FoldersSize) for i := range o.Folders { @@ -697,7 +715,7 @@ Folder Structure: struct Folder { string ID<64>; - Device Devices<>; + Device Devices<1000000>; unsigned int Flags; Option Options<64>; } @@ -733,6 +751,9 @@ func (o Folder) EncodeXDRInto(xw *xdr.Writer) (int, error) { return xw.Tot(), xdr.ElementSizeExceeded("ID", l, 64) } xw.WriteString(o.ID) + if l := len(o.Devices); l > 1000000 { + return xw.Tot(), xdr.ElementSizeExceeded("Devices", l, 1000000) + } xw.WriteUint32(uint32(len(o.Devices))) for i := range o.Devices { _, err := o.Devices[i].EncodeXDRInto(xw) @@ -769,7 +790,10 @@ func (o *Folder) DecodeXDRFrom(xr *xdr.Reader) error { o.ID = xr.ReadStringMax(64) _DevicesSize := int(xr.ReadUint32()) if _DevicesSize < 0 { - return xdr.ElementSizeExceeded("Devices", _DevicesSize, 0) + return xdr.ElementSizeExceeded("Devices", _DevicesSize, 1000000) + } + if _DevicesSize > 1000000 { + return xdr.ElementSizeExceeded("Devices", _DevicesSize, 1000000) } o.Devices = make([]Device, _DevicesSize) for i := range o.Devices { diff --git a/vector_xdr.go b/vector_xdr.go index a4b6b132b..01efa7e4e 100644 --- a/vector_xdr.go +++ b/vector_xdr.go @@ -2,6 +2,8 @@ package protocol +import "github.com/calmh/xdr" + // This stuff is hacked up manually because genxdr doesn't support 'type // Vector []Counter' declarations and it was tricky when I tried to add it... @@ -28,6 +30,9 @@ func (v Vector) EncodeXDRInto(w xdrWriter) (int, error) { // DecodeXDRFrom decodes the XDR objects from the given reader into itself. func (v *Vector) DecodeXDRFrom(r xdrReader) error { l := int(r.ReadUint32()) + if l > 1e6 { + return xdr.ElementSizeExceeded("number of counters", l, 1e6) + } n := make(Vector, l) for i := range n { n[i].ID = r.ReadUint64() From 6a36ec63d75f6ef354d8689027d23d3f16d08f8d Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 18 Aug 2015 08:42:39 +0200 Subject: [PATCH 58/61] Empty messages with the compression bit set should be accepted --- protocol.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protocol.go b/protocol.go index 420b20859..8e73afea5 100644 --- a/protocol.go +++ b/protocol.go @@ -412,7 +412,7 @@ func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) { } msgBuf := c.rdbuf0 - if hdr.compression { + if hdr.compression && msglen > 0 { c.rdbuf1 = c.rdbuf1[:cap(c.rdbuf1)] c.rdbuf1, err = lz4.Decode(c.rdbuf1, c.rdbuf0) if err != nil { From 68c5dcd83d9be8f28ae59e951a87cdcf01c6f5cb Mon Sep 17 00:00:00 2001 From: AudriusButkevicius Date: Wed, 26 Aug 2015 22:33:03 +0100 Subject: [PATCH 59/61] Add CachedSize field --- message.go | 1 + 1 file changed, 1 insertion(+) diff --git a/message.go b/message.go index 0cfeaa381..2a37136b5 100644 --- a/message.go +++ b/message.go @@ -20,6 +20,7 @@ type FileInfo struct { Modified int64 Version Vector LocalVersion int64 + CachedSize int64 // noencode (cache only) Blocks []BlockInfo // max:1000000 } From 84365882de255d2204d0eeda8dee288082a27f98 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Fri, 28 Aug 2015 09:01:21 +0200 Subject: [PATCH 60/61] Fix tests --- protocol_test.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/protocol_test.go b/protocol_test.go index 051672411..2c64a9b32 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -278,11 +278,12 @@ func TestMarshalIndexMessage(t *testing.T) { } f := func(m1 IndexMessage) bool { - for _, f := range m1.Files { - for i := range f.Blocks { - f.Blocks[i].Offset = 0 - if len(f.Blocks[i].Hash) == 0 { - f.Blocks[i].Hash = nil + for i, f := range m1.Files { + m1.Files[i].CachedSize = 0 + for j := range f.Blocks { + f.Blocks[j].Offset = 0 + if len(f.Blocks[j].Hash) == 0 { + f.Blocks[j].Hash = nil } } } From 05c79ac8c274ad3c207ce15f1faf25d6d0c3a3ec Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Mon, 21 Sep 2015 08:51:42 +0200 Subject: [PATCH 61/61] Simplify and improve the ping mechanism This should resolve the spurious ping timeouts we've had on low powered boxes. Those errors are the result of us requiring a timely Pong response to our Pings. However this is unnecessarily strict - as long as we've received *anything* recently, we know the other peer is alive. So the new mechanism removes the Pong message entirely and separates the ping check into two routines: - One that makes sure to send ping periodically, if nothing else has been sent. This guarantees a message sent every 45-90 seconds. - One that checks how long it was since we last received a message. If it's longer than 300 seconds, we trigger an ErrTimeout. So we're guaranteed to detect a connection failure in 300 + 300/2 seconds (due to how often the check runs) and we may detect it much sooner if we get an actual error on the ping write (a connection reset or so). This is more sluggish than before but I think that's an OK price to pay for making it actually work out of the box. This removes the configurability of it, as the timeout on one side is dependent on the send interval on the other side. Do we still need it configurable? --- protocol.go | 106 ++++++++++++++++++++++------------------------- protocol_test.go | 89 --------------------------------------- 2 files changed, 50 insertions(+), 145 deletions(-) diff --git a/protocol.go b/protocol.go index 8e73afea5..4c1364eaf 100644 --- a/protocol.go +++ b/protocol.go @@ -28,7 +28,6 @@ const ( messageTypeRequest = 2 messageTypeResponse = 3 messageTypePing = 4 - messageTypePong = 5 messageTypeIndexUpdate = 6 messageTypeClose = 7 ) @@ -71,13 +70,12 @@ const ( ) var ( - ErrClusterHash = fmt.Errorf("configuration error: mismatched cluster hash") - ErrClosed = errors.New("connection closed") + ErrClosed = errors.New("connection closed") + ErrTimeout = errors.New("read timeout") ) // Specific variants of empty messages... type pingMessage struct{ EmptyMessage } -type pongMessage struct{ EmptyMessage } type Model interface { // An index was received from the peer device @@ -146,9 +144,11 @@ type isEofer interface { IsEOF() bool } -var ( - PingTimeout = 30 * time.Second - PingIdleTime = 60 * time.Second +const ( + // We make sure to send a message at least this often, by triggering pings. + PingSendInterval = 90 * time.Second + // If we haven't received a message from the other side for this long, close the connection. + ReceiveTimeout = 300 * time.Second ) func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiver Model, name string, compress Compression) Connection { @@ -180,7 +180,8 @@ func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiv func (c *rawConnection) Start() { go c.readerLoop() go c.writerLoop() - go c.pingerLoop() + go c.pingSender() + go c.pingReceiver() go c.idGenerator() } @@ -278,18 +279,7 @@ func (c *rawConnection) ping() bool { return false } - rc := make(chan asyncResult, 1) - c.awaitingMut.Lock() - c.awaiting[id] = rc - c.awaitingMut.Unlock() - - ok := c.send(id, messageTypePing, nil, nil) - if !ok { - return false - } - - res, ok := <-rc - return ok && res.err == nil + return c.send(id, messageTypePing, nil, nil) } func (c *rawConnection) readerLoop() (err error) { @@ -352,13 +342,7 @@ func (c *rawConnection) readerLoop() (err error) { if state != stateReady { return fmt.Errorf("protocol error: ping message in state %d", state) } - c.send(hdr.msgID, messageTypePong, pongMessage{}, nil) - - case pongMessage: - if state != stateReady { - return fmt.Errorf("protocol error: pong message in state %d", state) - } - c.handlePong(hdr.msgID) + // Nothing case CloseMessage: return errors.New(msg.Reason) @@ -467,9 +451,6 @@ func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) { case messageTypePing: msg = pingMessage{} - case messageTypePong: - msg = pongMessage{} - case messageTypeClusterConfig: var cc ClusterConfigMessage err = cc.UnmarshalXDR(msgBuf) @@ -729,42 +710,55 @@ func (c *rawConnection) idGenerator() { } } -func (c *rawConnection) pingerLoop() { - var rc = make(chan bool, 1) - ticker := time.Tick(PingIdleTime / 2) +// The pingSender makes sure that we've sent a message within the last +// PingSendInterval. If we already have something sent in the last +// PingSendInterval/2, we do nothing. Otherwise we send a ping message. This +// results in an effecting ping interval of somewhere between +// PingSendInterval/2 and PingSendInterval. +func (c *rawConnection) pingSender() { + ticker := time.Tick(PingSendInterval / 2) + for { select { case <-ticker: - if d := time.Since(c.cr.Last()); d < PingIdleTime { - if debug { - l.Debugln(c.id, "ping skipped after rd", d) - } - continue - } - if d := time.Since(c.cw.Last()); d < PingIdleTime { + d := time.Since(c.cw.Last()) + if d < PingSendInterval/2 { if debug { l.Debugln(c.id, "ping skipped after wr", d) } continue } - go func() { + + if debug { + l.Debugln(c.id, "ping -> after", d) + } + c.ping() + + case <-c.closed: + return + } + } +} + +// The pingReciever checks that we've received a message (any message will do, +// but we expect pings in the absence of other messages) within the last +// ReceiveTimeout. If not, we close the connection with an ErrTimeout. +func (c *rawConnection) pingReceiver() { + ticker := time.Tick(ReceiveTimeout / 2) + + for { + select { + case <-ticker: + d := time.Since(c.cr.Last()) + if d > ReceiveTimeout { if debug { - l.Debugln(c.id, "ping ->") + l.Debugln(c.id, "ping timeout", d) } - rc <- c.ping() - }() - select { - case ok := <-rc: - if debug { - l.Debugln(c.id, "<- pong") - } - if !ok { - c.close(fmt.Errorf("ping failure")) - } - case <-time.After(PingTimeout): - c.close(fmt.Errorf("ping timeout")) - case <-c.closed: - return + c.close(ErrTimeout) + } + + if debug { + l.Debugln(c.id, "last read within", d) } case <-c.closed: diff --git a/protocol_test.go b/protocol_test.go index 2c64a9b32..8a4708843 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -6,7 +6,6 @@ import ( "bytes" "encoding/hex" "encoding/json" - "errors" "fmt" "io" "io/ioutil" @@ -82,94 +81,6 @@ func TestPing(t *testing.T) { } } -func TestPingErr(t *testing.T) { - e := errors.New("something broke") - - for i := 0; i < 32; i++ { - for j := 0; j < 32; j++ { - m0 := newTestModel() - m1 := newTestModel() - - ar, aw := io.Pipe() - br, bw := io.Pipe() - eaw := &ErrPipe{PipeWriter: *aw, max: i, err: e} - ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e} - - c0 := NewConnection(c0ID, ar, ebw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) - c0.Start() - c1 := NewConnection(c1ID, br, eaw, m1, "name", CompressAlways) - c1.Start() - c0.ClusterConfig(ClusterConfigMessage{}) - c1.ClusterConfig(ClusterConfigMessage{}) - - res := c0.ping() - if (i < 8 || j < 8) && res { - // This should have resulted in failure, as there is no way an empty ClusterConfig plus a Ping message fits in eight bytes. - t.Errorf("Unexpected ping success; i=%d, j=%d", i, j) - } else if (i >= 28 && j >= 28) && !res { - // This should have worked though, as 28 bytes is plenty for both. - t.Errorf("Unexpected ping fail; i=%d, j=%d", i, j) - } - } - } -} - -// func TestRequestResponseErr(t *testing.T) { -// e := errors.New("something broke") - -// var pass bool -// for i := 0; i < 48; i++ { -// for j := 0; j < 38; j++ { -// m0 := newTestModel() -// m0.data = []byte("response data") -// m1 := newTestModel() - -// ar, aw := io.Pipe() -// br, bw := io.Pipe() -// eaw := &ErrPipe{PipeWriter: *aw, max: i, err: e} -// ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e} - -// NewConnection(c0ID, ar, ebw, m0, nil) -// c1 := NewConnection(c1ID, br, eaw, m1, nil).(wireFormatConnection).next.(*rawConnection) - -// d, err := c1.Request("default", "tn", 1234, 5678) -// if err == e || err == ErrClosed { -// t.Logf("Error at %d+%d bytes", i, j) -// if !m1.isClosed() { -// t.Fatal("c1 not closed") -// } -// if !m0.isClosed() { -// t.Fatal("c0 not closed") -// } -// continue -// } -// if err != nil { -// t.Fatal(err) -// } -// if string(d) != "response data" { -// t.Fatalf("Incorrect response data %q", string(d)) -// } -// if m0.folder != "default" { -// t.Fatalf("Incorrect folder %q", m0.folder) -// } -// if m0.name != "tn" { -// t.Fatalf("Incorrect name %q", m0.name) -// } -// if m0.offset != 1234 { -// t.Fatalf("Incorrect offset %d", m0.offset) -// } -// if m0.size != 5678 { -// t.Fatalf("Incorrect size %d", m0.size) -// } -// t.Logf("Pass at %d+%d bytes", i, j) -// pass = true -// } -// } -// if !pass { -// t.Fatal("Never passed") -// } -// } - func TestVersionErr(t *testing.T) { m0 := newTestModel() m1 := newTestModel()