Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add version and invalid bit to protocol

This commit is contained in:
Jakob Borg 2014-01-07 22:44:21 +01:00
parent d4fe1400d2
commit 2cfb24892f
8 changed files with 100 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
model/model.go
View File

@ -56,8 +56,6 @@ type Model struct {
}
const (
FlagDeleted = 1 << 12
idxBcastHoldtime = 15 * time.Second // Wait at least this long after the last index modification
idxBcastMaxDelay = 120 * time.Second // Unless we've already waited this long
@ -65,7 +63,10 @@ const (
maxFileHoldTimeS = 600 // Always allow file changes at least this often
)
var ErrNoSuchFile = errors.New("no such file")
var (
ErrNoSuchFile = errors.New("no such file")
ErrInvalid = errors.New("file is invalid")
)
// NewModel creates and starts a new model. The model starts in read-only mode,
// where it sends index information to connected peers and responds to requests
@ -159,7 +160,7 @@ func (m *Model) GlobalSize() (files, deleted, bytes int) {
defer m.RUnlock()
for _, f := range m.global {
if f.Flags&FlagDeleted == 0 {
if f.Flags&protocol.FlagDeleted == 0 {
files++
bytes += f.Size()
} else {
@ -176,7 +177,7 @@ func (m *Model) LocalSize() (files, deleted, bytes int) {
defer m.RUnlock()
for _, f := range m.local {
if f.Flags&FlagDeleted == 0 {
if f.Flags&protocol.FlagDeleted == 0 {
files++
bytes += f.Size()
} else {
@ -193,7 +194,7 @@ func (m *Model) InSyncSize() (files, bytes int) {
defer m.RUnlock()
for n, f := range m.local {
if gf, ok := m.global[n]; ok && f.Modified == gf.Modified {
if gf, ok := m.global[n]; ok && f.Equals(gf) {
files++
bytes += f.Size()
}
@ -249,7 +250,7 @@ func (m *Model) IndexUpdate(nodeID string, fs []protocol.FileInfo) {
}
for _, f := range fs {
if f.Flags&FlagDeleted != 0 && !m.delete {
if f.Flags&protocol.FlagDeleted != 0 && !m.delete {
// Files marked as deleted do not even enter the model
continue
}
@ -284,13 +285,16 @@ func (m *Model) Close(node string, err error) {
func (m *Model) Request(nodeID, name string, offset uint64, size uint32, hash []byte) ([]byte, error) {
// Verify that the requested file exists in the local and global model.
m.RLock()
_, localOk := m.local[name]
lf, localOk := m.local[name]
_, globalOk := m.global[name]
m.RUnlock()
if !localOk || !globalOk {
log.Printf("SECURITY (nonexistent file) REQ(in): %s: %q o=%d s=%d h=%x", nodeID, name, offset, size, hash)
return nil, ErrNoSuchFile
}
if lf.Flags&protocol.FlagInvalid != 0 {
return nil, ErrInvalid
}
if m.trace["net"] && nodeID != "<local>" {
log.Printf("NET REQ(in): %s: %q o=%d s=%d h=%x", nodeID, name, offset, size, hash)
@ -322,7 +326,7 @@ func (m *Model) ReplaceLocal(fs []File) {
for _, f := range fs {
newLocal[f.Name] = f
if ef := m.local[f.Name]; ef.Modified != f.Modified {
if ef := m.local[f.Name]; !ef.Equals(f) {
updated = true
}
}
@ -430,7 +434,7 @@ func (m *Model) protocolIndex() []protocol.FileInfo {
mf := fileInfoFromFile(f)
if m.trace["idx"] {
var flagComment string
if mf.Flags&FlagDeleted != 0 {
if mf.Flags&protocol.FlagDeleted != 0 {
flagComment = " (deleted)"
}
log.Printf("IDX: %q m=%d f=%o%s (%d blocks)", mf.Name, mf.Modified, mf.Flags, flagComment, len(mf.Blocks))
@ -496,10 +500,10 @@ func (m *Model) markDeletedLocals(newLocal map[string]File) bool {
var updated bool
for n, f := range m.local {
if _, ok := newLocal[n]; !ok {
if gf := m.global[n]; gf.Modified <= f.Modified {
if f.Flags&FlagDeleted == 0 {
f.Flags = FlagDeleted
f.Modified = f.Modified + 1
if gf := m.global[n]; !gf.NewerThan(f) {
if f.Flags&protocol.FlagDeleted == 0 {
f.Flags = protocol.FlagDeleted
f.Version++
f.Blocks = nil
updated = true
}
@ -511,7 +515,7 @@ func (m *Model) markDeletedLocals(newLocal map[string]File) bool {
}
func (m *Model) updateLocal(f File) {
if ef, ok := m.local[f.Name]; !ok || ef.Modified != f.Modified {
if ef, ok := m.local[f.Name]; !ok || !ef.Equals(f) {
m.local[f.Name] = f
m.recomputeGlobal()
m.recomputeNeed()
@ -530,7 +534,7 @@ func (m *Model) recomputeGlobal() {
for _, fs := range m.remote {
for n, f := range fs {
if cf, ok := newGlobal[n]; !ok || cf.Modified < f.Modified {
if cf, ok := newGlobal[n]; !ok || f.NewerThan(cf) {
newGlobal[n] = f
}
}
@ -543,7 +547,7 @@ func (m *Model) recomputeGlobal() {
updated = true
} else {
for n, f0 := range newGlobal {
if f1, ok := m.global[n]; !ok || f0.Modified != f1.Modified {
if f1, ok := m.global[n]; !ok || !f0.Equals(f1) {
updated = true
break
}
@ -561,12 +565,16 @@ func (m *Model) recomputeNeed() {
m.need = make(map[string]bool)
for n, f := range m.global {
hf, ok := m.local[n]
if !ok || f.Modified > hf.Modified {
if f.Flags&FlagDeleted != 0 && !m.delete {
if !ok || f.NewerThan(hf) {
if f.Flags&protocol.FlagInvalid != 0 {
// Never attempt to sync invalid files
continue
}
if f.Flags&protocol.FlagDeleted != 0 && !m.delete {
// Don't want to delete files, so forget this need
continue
}
if f.Flags&FlagDeleted != 0 && !ok {
if f.Flags&protocol.FlagDeleted != 0 && !ok {
// Don't have the file, so don't need to delete it
continue
}
@ -584,7 +592,7 @@ func (m *Model) whoHas(name string) []string {
gf := m.global[name]
for node, files := range m.remote {
if file, ok := files[name]; ok && file.Modified == gf.Modified {
if file, ok := files[name]; ok && file.Equals(gf) {
remote = append(remote, node)
}
}
@ -607,6 +615,7 @@ func fileFromFileInfo(f protocol.FileInfo) File {
Name: f.Name,
Flags: f.Flags,
Modified: int64(f.Modified),
Version: f.Version,
Blocks: blocks,
}
}
@ -623,6 +632,7 @@ func fileInfoFromFile(f File) protocol.FileInfo {
Name: f.Name,
Flags: f.Flags,
Modified: int64(f.Modified),
Version: f.Version,
Blocks: blocks,
}
}

3
model/model_puller.go
View File

@ -25,6 +25,7 @@ import (
"time"
"github.com/calmh/syncthing/buffers"
"github.com/calmh/syncthing/protocol"
)
func (m *Model) pullFile(name string) error {
@ -171,7 +172,7 @@ func (m *Model) puller() {
}
var err error
if f.Flags&FlagDeleted == 0 {
if f.Flags&protocol.FlagDeleted == 0 {
if m.trace["file"] {
log.Printf("FILE: Pull %q", n)
}

26
model/model_test.go
View File

@ -228,9 +228,12 @@ func TestDelete(t *testing.T) {
if len(m.local["a new file"].Blocks) != 0 {
t.Error("Unexpected non-zero blocks for deleted file in local")
}
if ft := m.local["a new file"].Modified; ft != ot+1 {
if ft := m.local["a new file"].Modified; ft != ot {
t.Errorf("Unexpected time %d != %d for deleted file in local", ft, ot+1)
}
if fv := m.local["a new file"].Version; fv != 1 {
t.Errorf("Unexpected version %d != 1 for deleted file in local", fv)
}
if m.global["a new file"].Flags&(1<<12) == 0 {
t.Error("Unexpected deleted flag = 0 in global table")
@ -238,8 +241,11 @@ func TestDelete(t *testing.T) {
if len(m.global["a new file"].Blocks) != 0 {
t.Error("Unexpected non-zero blocks for deleted file in global")
}
if ft := m.local["a new file"].Modified; ft != ot+1 {
t.Errorf("Unexpected time %d != %d for deleted file in local", ft, ot+1)
if ft := m.global["a new file"].Modified; ft != ot {
t.Errorf("Unexpected time %d != %d for deleted file in global", ft, ot+1)
}
if fv := m.local["a new file"].Version; fv != 1 {
t.Errorf("Unexpected version %d != 1 for deleted file in global", fv)
}
// Another update should change nothing
@ -259,8 +265,11 @@ func TestDelete(t *testing.T) {
if len(m.local["a new file"].Blocks) != 0 {
t.Error("Unexpected non-zero blocks for deleted file in local")
}
if ft := m.local["a new file"].Modified; ft != ot+1 {
t.Errorf("Unexpected time %d != %d for deleted file in local", ft, ot+1)
if ft := m.local["a new file"].Modified; ft != ot {
t.Errorf("Unexpected time %d != %d for deleted file in local", ft, ot)
}
if fv := m.local["a new file"].Version; fv != 1 {
t.Errorf("Unexpected version %d != 1 for deleted file in local", fv)
}
if m.global["a new file"].Flags&(1<<12) == 0 {
@ -269,8 +278,11 @@ func TestDelete(t *testing.T) {
if len(m.global["a new file"].Blocks) != 0 {
t.Error("Unexpected non-zero blocks for deleted file in global")
}
if ft := m.local["a new file"].Modified; ft != ot+1 {
t.Errorf("Unexpected time %d != %d for deleted file in local", ft, ot+1)
if ft := m.global["a new file"].Modified; ft != ot {
t.Errorf("Unexpected time %d != %d for deleted file in global", ft, ot)
}
if fv := m.local["a new file"].Version; fv != 1 {
t.Errorf("Unexpected version %d != 1 for deleted file in global", fv)
}
}

19
model/walk.go
View File

@ -10,6 +10,8 @@ import (
"path/filepath"
"strings"
"time"
"github.com/calmh/syncthing/protocol"
)
const BlockSize = 128 * 1024
@ -18,6 +20,7 @@ type File struct {
Name string
Flags uint32
Modified int64
Version uint32
Blocks []Block
}
@ -28,6 +31,14 @@ func (f File) Size() (bytes int) {
return
}
func (f File) Equals(o File) bool {
return f.Modified == o.Modified && f.Version == o.Version
}
func (f File) NewerThan(o File) bool {
return f.Modified > o.Modified || (f.Modified == o.Modified && f.Version > o.Version)
}
func isTempName(name string) bool {
return strings.HasPrefix(path.Base(name), ".syncthing.")
}
@ -79,7 +90,10 @@ func (m *Model) genWalker(res *[]File, ign map[string][]string) filepath.WalkFun
m.RUnlock()
if ok && hf.Modified == modified {
// No change
if nf := uint32(info.Mode()); nf != hf.Flags {
hf.Flags = nf
hf.Version++
}
*res = append(*res, hf)
} else {
m.Lock()
@ -89,7 +103,8 @@ func (m *Model) genWalker(res *[]File, ign map[string][]string) filepath.WalkFun
}
if ok {
// Files that are ignored will be suppressed but don't actually exist in the local model
hf.Flags = protocol.FlagInvalid
hf.Version++
*res = append(*res, hf)
}
m.Unlock()

28
protocol/PROTOCOL.md
View File

@ -62,11 +62,10 @@ reserved bits must be set to zero.
All data following the message header is in XDR (RFC 1014) encoding.
The actual data types in use by BEP, in XDR naming convention, are:
- unsigned int -- unsigned 32 bit integer
- hyper -- signed 64 bit integer
- unsigned hyper -- signed 64 bit integer
- opaque<> -- variable length opaque data
- string<> -- variable length string
- (unsigned) int -- (unsigned) 32 bit integer
- (unsigned) hyper -- (unsigned) 64 bit integer
- opaque<> -- variable length opaque data
- string<> -- variable length string
The encoding of opaque<> and string<> are identical, the distinction is
solely in interpretation. Opaque data should not be interpreted as such,
@ -92,6 +91,7 @@ message.
string Name<>;
unsigned int Flags;
hyper Modified;
unsigned int Version;
BlockInfo Blocks<>;
}
@ -102,15 +102,19 @@ message.
The file name is the part relative to the repository root. The
modification time is expressed as the number of seconds since the Unix
Epoch. The hash algorithm is implied by the hash length. Currently, the
hash must be 32 bytes long and computed by SHA256.
Epoch. The version field is a counter that increments each time the file
changes but resets to zero each time the modification is updated. This
is used to signal changes to the file (or file metadata) while the
modification time remains unchanged. The hash algorithm is implied by
the hash length. Currently, the hash must be 32 bytes long and computed
by SHA256.
The flags field is made up of the following single bit flags:
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Reserved |D| Unix Perm. & Mode |
| Reserved |I|D| Unix Perm. & Mode |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- The lower 12 bits hold the common Unix permission and mode bits.
@ -118,9 +122,13 @@ The flags field is made up of the following single bit flags:
- Bit 19 ("D") is set when the file has been deleted. The block list
shall contain zero blocks and the modification time indicates the
time of deletion or, if deletion time is not reliably determinable,
one second past the last know modification time.
the last known modification time and a higher version number.
- Bit 0 through 18 are reserved for future use and shall be set to
- Bit 18 ("I") is set when the file is invalid and unavailable for
synchronization. A peer may set this bit to indicate that it can
temporarily not serve data for the file.
- Bit 0 through 17 are reserved for future use and shall be set to
zero.
### Request (Type = 2)

2
protocol/messages.go
View File

@ -39,6 +39,7 @@ func (w *marshalWriter) writeIndex(idx []FileInfo) {
w.writeString(f.Name)
w.writeUint32(f.Flags)
w.writeUint64(uint64(f.Modified))
w.writeUint32(f.Version)
w.writeUint32(uint32(len(f.Blocks)))
for _, b := range f.Blocks {
w.writeUint32(b.Length)
@ -77,6 +78,7 @@ func (r *marshalReader) readIndex() []FileInfo {
files[i].Name = r.readString()
files[i].Flags = r.readUint32()
files[i].Modified = int64(r.readUint64())
files[i].Version = r.readUint32()
nblocks := r.readUint32()
blocks := make([]BlockInfo, nblocks)
for j := range blocks {

6
protocol/messages_test.go
View File

@ -12,8 +12,9 @@ func TestIndex(t *testing.T) {
idx := []FileInfo{
{
"Foo",
0755,
FlagInvalid & FlagDeleted & 0755,
1234567890,
142,
[]BlockInfo{
{12345678, []byte("hash hash hash")},
{23456781, []byte("ash hash hashh")},
@ -23,6 +24,7 @@ func TestIndex(t *testing.T) {
"Quux/Quux",
0644,
2345678901,
232323232,
[]BlockInfo{
{45678123, []byte("4321 hash hash hash")},
{56781234, []byte("3214 ash hash hashh")},
@ -81,6 +83,7 @@ func BenchmarkWriteIndex(b *testing.B) {
"Foo",
0777,
1234567890,
424242,
[]BlockInfo{
{12345678, []byte("hash hash hash")},
{23456781, []byte("ash hash hashh")},
@ -90,6 +93,7 @@ func BenchmarkWriteIndex(b *testing.B) {
"Quux/Quux",
0644,
2345678901,
323232,
[]BlockInfo{
{45678123, []byte("4321 hash hash hash")},
{56781234, []byte("3214 ash hash hashh")},

6
protocol/protocol.go
View File

@ -20,10 +20,16 @@ const (
messageTypeIndexUpdate = 6
)
const (
FlagDeleted = 1 << 12
FlagInvalid = 1 << 13
)
type FileInfo struct {
Name string
Flags uint32
Modified int64
Version uint32
Blocks []BlockInfo
}