From 2c8b6270082d4beeaa61398f2db367230d0bfb27 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Sun, 18 Jan 2015 02:12:06 +0100 Subject: [PATCH] Integer type policy Integers are for numbers, enabling arithmetic like subtractions and for loops without getting shot in the foot. Unsigneds are for bitfields. - "int" for numbers that will always be laughably smaller than four billion, and where we don't care about the serialization format. - "int32" for numbers that will always be laughably smaller than four billion, and will be serialized to four bytes. - "int64" for numbers that may approach four billion or will be serialized to eight bytes. - "uint32" and "uint64" for bitfields, depending on required number of bits and serialization format. Likewise "uint8" and "uint16", although rare in this project since they don't exist in XDR. - "int8", "int16" and plain "uint" are almost never useful. --- Godeps/Godeps.json | 12 ++--- .../github.com/syncthing/protocol/counting.go | 32 ++++++------- .../github.com/syncthing/protocol/message.go | 17 ++++--- .../syncthing/protocol/message_xdr.go | 48 +++++++++---------- .../github.com/syncthing/protocol/protocol.go | 8 ++-- build.go | 2 +- cmd/syncthing/gui.go | 2 +- cmd/syncthing/memsize_darwin.go | 4 +- cmd/syncthing/memsize_linux.go | 4 +- cmd/syncthing/memsize_solaris.go | 4 +- cmd/syncthing/memsize_unimpl.go | 2 +- cmd/syncthing/memsize_windows.go | 4 +- cmd/syncthing/perfstats_unix.go | 2 +- internal/db/blockmap.go | 6 +-- internal/db/blockmap_test.go | 20 ++++---- internal/db/leveldb.go | 30 ++++++------ internal/db/leveldb_xdr.go | 6 +-- internal/db/set.go | 6 +-- internal/db/set_test.go | 2 +- internal/db/truncated.go | 10 ++-- internal/db/truncated_xdr.go | 18 +++---- internal/events/events.go | 2 +- internal/lamport/clock.go | 4 +- internal/lamport/clock_test.go | 4 +- internal/model/model.go | 14 ++++-- internal/model/model_test.go | 27 +++++++++++ internal/model/puller.go | 10 ++-- internal/model/puller_test.go | 6 +-- internal/model/sharedpullerstate.go | 24 +++++----- internal/scanner/blocks.go | 2 +- 30 files changed, 181 insertions(+), 151 deletions(-) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 577776f4a..25dd8eeb5 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -27,7 +27,7 @@ }, { "ImportPath": "github.com/calmh/xdr", - "Rev": "214788d8fedfc310c18eca9ed12be408a5054cd5" + "Rev": "ff948d7666c5e0fd18d398f6278881724d36a90b" }, { "ImportPath": "github.com/juju/ratelimit", @@ -35,7 +35,7 @@ }, { "ImportPath": "github.com/syncthing/protocol", - "Rev": "f76b5d800208131015b14cfc1cfd3e6dd1d5e979" + "Rev": "15bf5f583a88b7aaf0a5b810fcf5fb21da0a3b3f" }, { "ImportPath": "github.com/syndtr/goleveldb/leveldb", @@ -59,19 +59,19 @@ }, { "ImportPath": "golang.org/x/crypto/bcrypt", - "Rev": "731db29863ea7213d9556d0170afb38987f401d4" + "Rev": "4ed45ec682102c643324fae5dff8dab085b6c300" }, { "ImportPath": "golang.org/x/crypto/blowfish", - "Rev": "731db29863ea7213d9556d0170afb38987f401d4" + "Rev": "4ed45ec682102c643324fae5dff8dab085b6c300" }, { "ImportPath": "golang.org/x/text/transform", - "Rev": "985ee5acfaf1ff6712c7c99438752f8e09416ccb" + "Rev": "c980adc4a823548817b9c47d38c6ca6b7d7d8b6a" }, { "ImportPath": "golang.org/x/text/unicode/norm", - "Rev": "985ee5acfaf1ff6712c7c99438752f8e09416ccb" + "Rev": "c980adc4a823548817b9c47d38c6ca6b7d7d8b6a" } ] } diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/counting.go b/Godeps/_workspace/src/github.com/syncthing/protocol/counting.go index 490b77fd1..d441ed311 100644 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/counting.go +++ b/Godeps/_workspace/src/github.com/syncthing/protocol/counting.go @@ -10,25 +10,25 @@ import ( type countingReader struct { io.Reader - tot uint64 // bytes - last int64 // unix nanos + tot int64 // bytes + last int64 // unix nanos } var ( - totalIncoming uint64 - totalOutgoing uint64 + totalIncoming int64 + totalOutgoing int64 ) func (c *countingReader) Read(bs []byte) (int, error) { n, err := c.Reader.Read(bs) - atomic.AddUint64(&c.tot, uint64(n)) - atomic.AddUint64(&totalIncoming, uint64(n)) + atomic.AddInt64(&c.tot, int64(n)) + atomic.AddInt64(&totalIncoming, int64(n)) atomic.StoreInt64(&c.last, time.Now().UnixNano()) return n, err } -func (c *countingReader) Tot() uint64 { - return atomic.LoadUint64(&c.tot) +func (c *countingReader) Tot() int64 { + return atomic.LoadInt64(&c.tot) } func (c *countingReader) Last() time.Time { @@ -37,26 +37,26 @@ func (c *countingReader) Last() time.Time { type countingWriter struct { io.Writer - tot uint64 // bytes - last int64 // unix nanos + tot int64 // bytes + last int64 // unix nanos } func (c *countingWriter) Write(bs []byte) (int, error) { n, err := c.Writer.Write(bs) - atomic.AddUint64(&c.tot, uint64(n)) - atomic.AddUint64(&totalOutgoing, uint64(n)) + atomic.AddInt64(&c.tot, int64(n)) + atomic.AddInt64(&totalOutgoing, int64(n)) atomic.StoreInt64(&c.last, time.Now().UnixNano()) return n, err } -func (c *countingWriter) Tot() uint64 { - return atomic.LoadUint64(&c.tot) +func (c *countingWriter) Tot() int64 { + return atomic.LoadInt64(&c.tot) } func (c *countingWriter) Last() time.Time { return time.Unix(0, atomic.LoadInt64(&c.last)) } -func TotalInOut() (uint64, uint64) { - return atomic.LoadUint64(&totalIncoming), atomic.LoadUint64(&totalOutgoing) +func TotalInOut() (int64, int64) { + return atomic.LoadInt64(&totalIncoming), atomic.LoadInt64(&totalOutgoing) } diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/message.go b/Godeps/_workspace/src/github.com/syncthing/protocol/message.go index d85f21cc6..dbaf526ac 100644 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/message.go +++ b/Godeps/_workspace/src/github.com/syncthing/protocol/message.go @@ -1,6 +1,5 @@ // Copyright (C) 2014 The Protocol Authors. -//go:generate -command genxdr go run ../../Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go //go:generate genxdr -o message_xdr.go message.go package protocol @@ -18,8 +17,8 @@ type FileInfo struct { Name string // max:8192 Flags uint32 Modified int64 - Version uint64 - LocalVersion uint64 + Version int64 + LocalVersion int64 Blocks []BlockInfo } @@ -60,7 +59,7 @@ func (f FileInfo) HasPermissionBits() bool { type BlockInfo struct { Offset int64 // noencode (cache only) - Size uint32 + Size int32 Hash []byte // max:64 } @@ -71,8 +70,8 @@ func (b BlockInfo) String() string { type RequestMessage struct { Folder string // max:64 Name string // max:8192 - Offset uint64 - Size uint32 + Offset int64 + Size int32 Hash []byte // max:64 Flags uint32 Options []Option // max:64 @@ -80,7 +79,7 @@ type RequestMessage struct { type ResponseMessage struct { Data []byte - Error uint32 + Error int32 } type ClusterConfigMessage struct { @@ -107,7 +106,7 @@ type Folder struct { type Device struct { ID []byte // max:32 Flags uint32 - MaxLocalVersion uint64 + MaxLocalVersion int64 } type Option struct { @@ -117,7 +116,7 @@ type Option struct { type CloseMessage struct { Reason string // max:1024 - Code uint32 + Code int32 } type EmptyMessage struct{} diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/message_xdr.go b/Godeps/_workspace/src/github.com/syncthing/protocol/message_xdr.go index fa7b5802b..54d030da1 100644 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/message_xdr.go +++ b/Godeps/_workspace/src/github.com/syncthing/protocol/message_xdr.go @@ -168,8 +168,8 @@ struct FileInfo { string Name<8192>; unsigned int Flags; hyper Modified; - unsigned hyper Version; - unsigned hyper LocalVersion; + hyper Version; + hyper LocalVersion; BlockInfo Blocks<>; } @@ -206,8 +206,8 @@ func (o FileInfo) encodeXDR(xw *xdr.Writer) (int, error) { xw.WriteString(o.Name) xw.WriteUint32(o.Flags) xw.WriteUint64(uint64(o.Modified)) - xw.WriteUint64(o.Version) - xw.WriteUint64(o.LocalVersion) + xw.WriteUint64(uint64(o.Version)) + xw.WriteUint64(uint64(o.LocalVersion)) xw.WriteUint32(uint32(len(o.Blocks))) for i := range o.Blocks { _, err := o.Blocks[i].encodeXDR(xw) @@ -233,8 +233,8 @@ func (o *FileInfo) decodeXDR(xr *xdr.Reader) error { o.Name = xr.ReadStringMax(8192) o.Flags = xr.ReadUint32() o.Modified = int64(xr.ReadUint64()) - o.Version = xr.ReadUint64() - o.LocalVersion = xr.ReadUint64() + o.Version = int64(xr.ReadUint64()) + o.LocalVersion = int64(xr.ReadUint64()) _BlocksSize := int(xr.ReadUint32()) o.Blocks = make([]BlockInfo, _BlocksSize) for i := range o.Blocks { @@ -261,7 +261,7 @@ BlockInfo Structure: struct BlockInfo { - unsigned int Size; + int Size; opaque Hash<64>; } @@ -292,7 +292,7 @@ func (o BlockInfo) AppendXDR(bs []byte) ([]byte, error) { } func (o BlockInfo) encodeXDR(xw *xdr.Writer) (int, error) { - xw.WriteUint32(o.Size) + xw.WriteUint32(uint32(o.Size)) if l := len(o.Hash); l > 64 { return xw.Tot(), xdr.ElementSizeExceeded("Hash", l, 64) } @@ -312,7 +312,7 @@ func (o *BlockInfo) UnmarshalXDR(bs []byte) error { } func (o *BlockInfo) decodeXDR(xr *xdr.Reader) error { - o.Size = xr.ReadUint32() + o.Size = int32(xr.ReadUint32()) o.Hash = xr.ReadBytesMax(64) return xr.Error() } @@ -361,8 +361,8 @@ RequestMessage Structure: struct RequestMessage { string Folder<64>; string Name<8192>; - unsigned hyper Offset; - unsigned int Size; + hyper Offset; + int Size; opaque Hash<64>; unsigned int Flags; Option Options<64>; @@ -403,8 +403,8 @@ func (o RequestMessage) encodeXDR(xw *xdr.Writer) (int, error) { return xw.Tot(), xdr.ElementSizeExceeded("Name", l, 8192) } xw.WriteString(o.Name) - xw.WriteUint64(o.Offset) - xw.WriteUint32(o.Size) + xw.WriteUint64(uint64(o.Offset)) + xw.WriteUint32(uint32(o.Size)) if l := len(o.Hash); l > 64 { return xw.Tot(), xdr.ElementSizeExceeded("Hash", l, 64) } @@ -437,8 +437,8 @@ func (o *RequestMessage) UnmarshalXDR(bs []byte) error { func (o *RequestMessage) decodeXDR(xr *xdr.Reader) error { o.Folder = xr.ReadStringMax(64) o.Name = xr.ReadStringMax(8192) - o.Offset = xr.ReadUint64() - o.Size = xr.ReadUint32() + o.Offset = int64(xr.ReadUint64()) + o.Size = int32(xr.ReadUint32()) o.Hash = xr.ReadBytesMax(64) o.Flags = xr.ReadUint32() _OptionsSize := int(xr.ReadUint32()) @@ -471,7 +471,7 @@ ResponseMessage Structure: struct ResponseMessage { opaque Data<>; - unsigned int Error; + int Error; } */ @@ -502,7 +502,7 @@ func (o ResponseMessage) AppendXDR(bs []byte) ([]byte, error) { func (o ResponseMessage) encodeXDR(xw *xdr.Writer) (int, error) { xw.WriteBytes(o.Data) - xw.WriteUint32(o.Error) + xw.WriteUint32(uint32(o.Error)) return xw.Tot(), xw.Error() } @@ -519,7 +519,7 @@ func (o *ResponseMessage) UnmarshalXDR(bs []byte) error { func (o *ResponseMessage) decodeXDR(xr *xdr.Reader) error { o.Data = xr.ReadBytes() - o.Error = xr.ReadUint32() + o.Error = int32(xr.ReadUint32()) return xr.Error() } @@ -766,7 +766,7 @@ Device Structure: struct Device { opaque ID<32>; unsigned int Flags; - unsigned hyper MaxLocalVersion; + hyper MaxLocalVersion; } */ @@ -801,7 +801,7 @@ func (o Device) encodeXDR(xw *xdr.Writer) (int, error) { } xw.WriteBytes(o.ID) xw.WriteUint32(o.Flags) - xw.WriteUint64(o.MaxLocalVersion) + xw.WriteUint64(uint64(o.MaxLocalVersion)) return xw.Tot(), xw.Error() } @@ -819,7 +819,7 @@ func (o *Device) UnmarshalXDR(bs []byte) error { func (o *Device) decodeXDR(xr *xdr.Reader) error { o.ID = xr.ReadBytesMax(32) o.Flags = xr.ReadUint32() - o.MaxLocalVersion = xr.ReadUint64() + o.MaxLocalVersion = int64(xr.ReadUint64()) return xr.Error() } @@ -923,7 +923,7 @@ CloseMessage Structure: struct CloseMessage { string Reason<1024>; - unsigned int Code; + int Code; } */ @@ -957,7 +957,7 @@ func (o CloseMessage) encodeXDR(xw *xdr.Writer) (int, error) { return xw.Tot(), xdr.ElementSizeExceeded("Reason", l, 1024) } xw.WriteString(o.Reason) - xw.WriteUint32(o.Code) + xw.WriteUint32(uint32(o.Code)) return xw.Tot(), xw.Error() } @@ -974,7 +974,7 @@ func (o *CloseMessage) UnmarshalXDR(bs []byte) error { func (o *CloseMessage) decodeXDR(xr *xdr.Reader) error { o.Reason = xr.ReadStringMax(1024) - o.Code = xr.ReadUint32() + o.Code = int32(xr.ReadUint32()) return xr.Error() } diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/protocol.go b/Godeps/_workspace/src/github.com/syncthing/protocol/protocol.go index 7cf59af1f..eeaf0f46b 100644 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/protocol.go +++ b/Godeps/_workspace/src/github.com/syncthing/protocol/protocol.go @@ -222,8 +222,8 @@ func (c *rawConnection) Request(folder string, name string, offset int64, size i ok := c.send(id, messageTypeRequest, RequestMessage{ Folder: folder, Name: name, - Offset: uint64(offset), - Size: uint32(size), + Offset: offset, + Size: int32(size), }) if !ok { return nil, ErrClosed @@ -706,8 +706,8 @@ func (c *rawConnection) pingerLoop() { type Statistics struct { At time.Time - InBytesTotal uint64 - OutBytesTotal uint64 + InBytesTotal int64 + OutBytesTotal int64 } func (c *rawConnection) Statistics() Statistics { diff --git a/build.go b/build.go index ff8b8809b..f727e3113 100644 --- a/build.go +++ b/build.go @@ -301,7 +301,7 @@ func assets() { } func xdr() { - runPrint("go", "generate", "./internal/discover", "./internal/db", "./internal/protocol") + runPrint("go", "generate", "./internal/discover", "./internal/db") } func translate() { diff --git a/cmd/syncthing/gui.go b/cmd/syncthing/gui.go index fa2bb4b25..a8a8f4ec2 100644 --- a/cmd/syncthing/gui.go +++ b/cmd/syncthing/gui.go @@ -794,7 +794,7 @@ func toNeedSlice(fs []db.FileInfoTruncated) []map[string]interface{} { "Version": file.Version, "LocalVersion": file.LocalVersion, "NumBlocks": file.NumBlocks, - "Size": db.BlocksToSize(file.NumBlocks), + "Size": db.BlocksToSize(int(file.NumBlocks)), } } return output diff --git a/cmd/syncthing/memsize_darwin.go b/cmd/syncthing/memsize_darwin.go index f9978df9d..1ca19a247 100644 --- a/cmd/syncthing/memsize_darwin.go +++ b/cmd/syncthing/memsize_darwin.go @@ -22,7 +22,7 @@ import ( "strings" ) -func memorySize() (uint64, error) { +func memorySize() (int64, error) { cmd := exec.Command("sysctl", "hw.memsize") out, err := cmd.Output() if err != nil { @@ -32,7 +32,7 @@ func memorySize() (uint64, error) { if len(fs) != 2 { return 0, errors.New("sysctl parse error") } - bytes, err := strconv.ParseUint(fs[1], 10, 64) + bytes, err := strconv.ParseInt(fs[1], 10, 64) if err != nil { return 0, err } diff --git a/cmd/syncthing/memsize_linux.go b/cmd/syncthing/memsize_linux.go index 57023d11e..40a0cbcd7 100644 --- a/cmd/syncthing/memsize_linux.go +++ b/cmd/syncthing/memsize_linux.go @@ -23,7 +23,7 @@ import ( "strings" ) -func memorySize() (uint64, error) { +func memorySize() (int64, error) { f, err := os.Open("/proc/meminfo") if err != nil { return 0, err @@ -40,7 +40,7 @@ func memorySize() (uint64, error) { return 0, errors.New("/proc/meminfo parse error 2") } - kb, err := strconv.ParseUint(fs[1], 10, 64) + kb, err := strconv.ParseInt(fs[1], 10, 64) if err != nil { return 0, err } diff --git a/cmd/syncthing/memsize_solaris.go b/cmd/syncthing/memsize_solaris.go index 19a24b1e6..6c0b6a54e 100644 --- a/cmd/syncthing/memsize_solaris.go +++ b/cmd/syncthing/memsize_solaris.go @@ -22,14 +22,14 @@ import ( "strconv" ) -func memorySize() (uint64, error) { +func memorySize() (int64, error) { cmd := exec.Command("prtconf", "-m") out, err := cmd.CombinedOutput() if err != nil { return 0, err } - mb, err := strconv.ParseUint(string(out), 10, 64) + mb, err := strconv.ParseInt(string(out), 10, 64) if err != nil { return 0, err } diff --git a/cmd/syncthing/memsize_unimpl.go b/cmd/syncthing/memsize_unimpl.go index 556c3c077..704472379 100644 --- a/cmd/syncthing/memsize_unimpl.go +++ b/cmd/syncthing/memsize_unimpl.go @@ -19,6 +19,6 @@ package main import "errors" -func memorySize() (uint64, error) { +func memorySize() (int64, error) { return 0, errors.New("not implemented") } diff --git a/cmd/syncthing/memsize_windows.go b/cmd/syncthing/memsize_windows.go index 4058e17a0..23ad46b78 100644 --- a/cmd/syncthing/memsize_windows.go +++ b/cmd/syncthing/memsize_windows.go @@ -26,7 +26,7 @@ var ( globalMemoryStatusEx, _ = syscall.GetProcAddress(kernel32, "GlobalMemoryStatusEx") ) -func memorySize() (uint64, error) { +func memorySize() (int64, error) { var memoryStatusEx [64]byte binary.LittleEndian.PutUint32(memoryStatusEx[:], 64) p := uintptr(unsafe.Pointer(&memoryStatusEx[0])) @@ -36,5 +36,5 @@ func memorySize() (uint64, error) { return 0, callErr } - return binary.LittleEndian.Uint64(memoryStatusEx[8:]), nil + return int64(binary.LittleEndian.Uint64(memoryStatusEx[8:])), nil } diff --git a/cmd/syncthing/perfstats_unix.go b/cmd/syncthing/perfstats_unix.go index 1aa271f22..7a3603388 100644 --- a/cmd/syncthing/perfstats_unix.go +++ b/cmd/syncthing/perfstats_unix.go @@ -43,7 +43,7 @@ func savePerfStats(file string) { var prevTime int64 var rusage syscall.Rusage var memstats runtime.MemStats - var prevIn, prevOut uint64 + var prevIn, prevOut int64 t0 := time.Now() for t := range time.NewTicker(250 * time.Millisecond).C { diff --git a/internal/db/blockmap.go b/internal/db/blockmap.go index 5f2217399..cfc4a135f 100644 --- a/internal/db/blockmap.go +++ b/internal/db/blockmap.go @@ -160,7 +160,7 @@ func (f *BlockFinder) Changed(cfg config.Configuration) error { // the block) or false to continue iterating for whatever reason. // The iterator finally returns the result, whether or not a satisfying block // was eventually found. -func (f *BlockFinder) Iterate(hash []byte, iterFn func(string, string, uint32) bool) bool { +func (f *BlockFinder) Iterate(hash []byte, iterFn func(string, string, int32) bool) bool { f.mut.RLock() folders := f.folders f.mut.RUnlock() @@ -171,7 +171,7 @@ func (f *BlockFinder) Iterate(hash []byte, iterFn func(string, string, uint32) b for iter.Next() && iter.Error() == nil { folder, file := fromBlockKey(iter.Key()) - index := binary.BigEndian.Uint32(iter.Value()) + index := int32(binary.BigEndian.Uint32(iter.Value())) if iterFn(folder, osutil.NativeFilename(file), index) { return true } @@ -182,7 +182,7 @@ func (f *BlockFinder) Iterate(hash []byte, iterFn func(string, string, uint32) b // A method for repairing incorrect blockmap entries, removes the old entry // and replaces it with a new entry for the given block -func (f *BlockFinder) Fix(folder, file string, index uint32, oldHash, newHash []byte) error { +func (f *BlockFinder) Fix(folder, file string, index int32, oldHash, newHash []byte) error { buf := make([]byte, 4) binary.BigEndian.PutUint32(buf, uint32(index)) diff --git a/internal/db/blockmap_test.go b/internal/db/blockmap_test.go index b8c4bc173..0da76194e 100644 --- a/internal/db/blockmap_test.go +++ b/internal/db/blockmap_test.go @@ -32,7 +32,7 @@ func genBlocks(n int) []protocol.BlockInfo { for j := range h { h[j] = byte(i + j) } - b[i].Size = uint32(i) + b[i].Size = int32(i) b[i].Hash = h } return b @@ -103,21 +103,21 @@ func TestBlockMapAddUpdateWipe(t *testing.T) { t.Fatal(err) } - f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index uint32) bool { + f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index int32) bool { if folder != "folder1" || file != "f1" || index != 0 { t.Fatal("Mismatch") } return true }) - f.Iterate(f2.Blocks[0].Hash, func(folder, file string, index uint32) bool { + f.Iterate(f2.Blocks[0].Hash, func(folder, file string, index int32) bool { if folder != "folder1" || file != "f2" || index != 0 { t.Fatal("Mismatch") } return true }) - f.Iterate(f3.Blocks[0].Hash, func(folder, file string, index uint32) bool { + f.Iterate(f3.Blocks[0].Hash, func(folder, file string, index int32) bool { t.Fatal("Unexpected block") return true }) @@ -132,17 +132,17 @@ func TestBlockMapAddUpdateWipe(t *testing.T) { t.Fatal(err) } - f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index uint32) bool { + f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index int32) bool { t.Fatal("Unexpected block") return false }) - f.Iterate(f2.Blocks[0].Hash, func(folder, file string, index uint32) bool { + f.Iterate(f2.Blocks[0].Hash, func(folder, file string, index int32) bool { t.Fatal("Unexpected block") return false }) - f.Iterate(f3.Blocks[0].Hash, func(folder, file string, index uint32) bool { + f.Iterate(f3.Blocks[0].Hash, func(folder, file string, index int32) bool { if folder != "folder1" || file != "f3" || index != 0 { t.Fatal("Mismatch") } @@ -189,7 +189,7 @@ func TestBlockFinderLookup(t *testing.T) { } counter := 0 - f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index uint32) bool { + f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index int32) bool { counter++ switch counter { case 1: @@ -217,7 +217,7 @@ func TestBlockFinderLookup(t *testing.T) { } counter = 0 - f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index uint32) bool { + f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index int32) bool { counter++ switch counter { case 1: @@ -239,7 +239,7 @@ func TestBlockFinderLookup(t *testing.T) { func TestBlockFinderFix(t *testing.T) { db, f := setup() - iterFn := func(folder, file string, index uint32) bool { + iterFn := func(folder, file string, index int32) bool { return true } diff --git a/internal/db/leveldb.go b/internal/db/leveldb.go index ff81f594c..99b83ba0b 100644 --- a/internal/db/leveldb.go +++ b/internal/db/leveldb.go @@ -34,11 +34,11 @@ import ( ) var ( - clockTick uint64 + clockTick int64 clockMut sync.Mutex ) -func clock(v uint64) uint64 { +func clock(v int64) int64 { clockMut.Lock() defer clockMut.Unlock() if v > clockTick { @@ -56,7 +56,7 @@ const ( ) type fileVersion struct { - version uint64 + version int64 device []byte } @@ -164,9 +164,9 @@ func globalKeyFolder(key []byte) []byte { return folder[:izero] } -type deletionHandler func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) uint64 +type deletionHandler func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) int64 -func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, deleteFn deletionHandler) uint64 { +func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, deleteFn deletionHandler) int64 { runtime.GC() sort.Sort(fileList(fs)) // sort list on name, same as in the database @@ -197,7 +197,7 @@ func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.File moreDb := dbi.Next() fsi := 0 - var maxLocalVer uint64 + var maxLocalVer int64 for { var newName, oldName []byte @@ -288,9 +288,9 @@ func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.File return maxLocalVer } -func ldbReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) uint64 { +func ldbReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) int64 { // TODO: Return the remaining maxLocalVer? - return ldbGenericReplace(db, folder, device, fs, func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) uint64 { + return ldbGenericReplace(db, folder, device, fs, func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) int64 { // Database has a file that we are missing. Remove it. if debugDB { l.Debugf("delete; folder=%q device=%v name=%q", folder, protocol.DeviceIDFromBytes(device), name) @@ -304,8 +304,8 @@ func ldbReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) u }) } -func ldbReplaceWithDelete(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) uint64 { - return ldbGenericReplace(db, folder, device, fs, func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) uint64 { +func ldbReplaceWithDelete(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) int64 { + return ldbGenericReplace(db, folder, device, fs, func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) int64 { var tf FileInfoTruncated err := tf.UnmarshalXDR(dbi.Value()) if err != nil { @@ -335,7 +335,7 @@ func ldbReplaceWithDelete(db *leveldb.DB, folder, device []byte, fs []protocol.F }) } -func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) uint64 { +func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) int64 { runtime.GC() batch := new(leveldb.Batch) @@ -356,7 +356,7 @@ func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) ui snap.Release() }() - var maxLocalVer uint64 + var maxLocalVer int64 for _, f := range fs { name := []byte(f.Name) fk := deviceKey(folder, device, name) @@ -406,7 +406,7 @@ func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) ui return maxLocalVer } -func ldbInsert(batch dbWriter, folder, device []byte, file protocol.FileInfo) uint64 { +func ldbInsert(batch dbWriter, folder, device []byte, file protocol.FileInfo) int64 { if debugDB { l.Debugf("insert; folder=%q device=%v %v", folder, protocol.DeviceIDFromBytes(device), file) } @@ -428,7 +428,7 @@ func ldbInsert(batch dbWriter, folder, device []byte, file protocol.FileInfo) ui // ldbUpdateGlobal adds this device+version to the version list for the given // file. If the device is already present in the list, the version is updated. // If the file does not have an entry in the global list, it is created. -func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device, file []byte, version uint64) bool { +func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device, file []byte, version int64) bool { if debugDB { l.Debugf("update global; folder=%q device=%v file=%q version=%d", folder, protocol.DeviceIDFromBytes(device), file, version) } @@ -798,7 +798,7 @@ outer: have := false // If we have the file, any version need := false // If we have a lower version of the file - var haveVersion uint64 + var haveVersion int64 for _, v := range vl.versions { if bytes.Compare(v.device, device) == 0 { have = true diff --git a/internal/db/leveldb_xdr.go b/internal/db/leveldb_xdr.go index f98139269..23339a72a 100644 --- a/internal/db/leveldb_xdr.go +++ b/internal/db/leveldb_xdr.go @@ -31,7 +31,7 @@ fileVersion Structure: struct fileVersion { - unsigned hyper version; + hyper version; opaque device<>; } @@ -62,7 +62,7 @@ func (o fileVersion) AppendXDR(bs []byte) ([]byte, error) { } func (o fileVersion) encodeXDR(xw *xdr.Writer) (int, error) { - xw.WriteUint64(o.version) + xw.WriteUint64(uint64(o.version)) xw.WriteBytes(o.device) return xw.Tot(), xw.Error() } @@ -79,7 +79,7 @@ func (o *fileVersion) UnmarshalXDR(bs []byte) error { } func (o *fileVersion) decodeXDR(xr *xdr.Reader) error { - o.version = xr.ReadUint64() + o.version = int64(xr.ReadUint64()) o.device = xr.ReadBytes() return xr.Error() } diff --git a/internal/db/set.go b/internal/db/set.go index 0ddc38554..794df4bd0 100644 --- a/internal/db/set.go +++ b/internal/db/set.go @@ -31,7 +31,7 @@ import ( ) type FileSet struct { - localVersion map[protocol.DeviceID]uint64 + localVersion map[protocol.DeviceID]int64 mutex sync.Mutex folder string db *leveldb.DB @@ -56,7 +56,7 @@ type Iterator func(f FileIntf) bool func NewFileSet(folder string, db *leveldb.DB) *FileSet { var s = FileSet{ - localVersion: make(map[protocol.DeviceID]uint64), + localVersion: make(map[protocol.DeviceID]int64), folder: folder, db: db, blockmap: NewBlockMap(db, folder), @@ -212,7 +212,7 @@ func (s *FileSet) Availability(file string) []protocol.DeviceID { return ldbAvailability(s.db, []byte(s.folder), []byte(osutil.NormalizedFilename(file))) } -func (s *FileSet) LocalVersion(device protocol.DeviceID) uint64 { +func (s *FileSet) LocalVersion(device protocol.DeviceID) int64 { s.mutex.Lock() defer s.mutex.Unlock() return s.localVersion[device] diff --git a/internal/db/set_test.go b/internal/db/set_test.go index 8c1b5bc72..69808747a 100644 --- a/internal/db/set_test.go +++ b/internal/db/set_test.go @@ -43,7 +43,7 @@ func genBlocks(n int) []protocol.BlockInfo { for j := range h { h[j] = byte(i + j) } - b[i].Size = uint32(i) + b[i].Size = int32(i) b[i].Hash = h } return b diff --git a/internal/db/truncated.go b/internal/db/truncated.go index ab8668394..446af66b9 100644 --- a/internal/db/truncated.go +++ b/internal/db/truncated.go @@ -29,9 +29,9 @@ type FileInfoTruncated struct { Name string // max:8192 Flags uint32 Modified int64 - Version uint64 - LocalVersion uint64 - NumBlocks uint32 + Version int64 + LocalVersion int64 + NumBlocks int32 } func (f FileInfoTruncated) String() string { @@ -44,7 +44,7 @@ func (f FileInfoTruncated) Size() int64 { if f.IsDeleted() || f.IsDirectory() { return 128 } - return BlocksToSize(f.NumBlocks) + return BlocksToSize(int(f.NumBlocks)) } func (f FileInfoTruncated) IsDeleted() bool { @@ -67,7 +67,7 @@ func (f FileInfoTruncated) HasPermissionBits() bool { return f.Flags&protocol.FlagNoPermBits == 0 } -func BlocksToSize(num uint32) int64 { +func BlocksToSize(num int) int64 { if num < 2 { return protocol.BlockSize / 2 } diff --git a/internal/db/truncated_xdr.go b/internal/db/truncated_xdr.go index ac949abda..b3f301646 100644 --- a/internal/db/truncated_xdr.go +++ b/internal/db/truncated_xdr.go @@ -46,9 +46,9 @@ struct FileInfoTruncated { string Name<8192>; unsigned int Flags; hyper Modified; - unsigned hyper Version; - unsigned hyper LocalVersion; - unsigned int NumBlocks; + hyper Version; + hyper LocalVersion; + int NumBlocks; } */ @@ -84,9 +84,9 @@ func (o FileInfoTruncated) encodeXDR(xw *xdr.Writer) (int, error) { xw.WriteString(o.Name) xw.WriteUint32(o.Flags) xw.WriteUint64(uint64(o.Modified)) - xw.WriteUint64(o.Version) - xw.WriteUint64(o.LocalVersion) - xw.WriteUint32(o.NumBlocks) + xw.WriteUint64(uint64(o.Version)) + xw.WriteUint64(uint64(o.LocalVersion)) + xw.WriteUint32(uint32(o.NumBlocks)) return xw.Tot(), xw.Error() } @@ -105,8 +105,8 @@ func (o *FileInfoTruncated) decodeXDR(xr *xdr.Reader) error { o.Name = xr.ReadStringMax(8192) o.Flags = xr.ReadUint32() o.Modified = int64(xr.ReadUint64()) - o.Version = xr.ReadUint64() - o.LocalVersion = xr.ReadUint64() - o.NumBlocks = xr.ReadUint32() + o.Version = int64(xr.ReadUint64()) + o.LocalVersion = int64(xr.ReadUint64()) + o.NumBlocks = int32(xr.ReadUint32()) return xr.Error() } diff --git a/internal/events/events.go b/internal/events/events.go index 304957e2d..d911cd67c 100644 --- a/internal/events/events.go +++ b/internal/events/events.go @@ -22,7 +22,7 @@ import ( "time" ) -type EventType uint64 +type EventType int const ( Ping EventType = 1 << iota diff --git a/internal/lamport/clock.go b/internal/lamport/clock.go index 6a3e7c51a..865f1625e 100644 --- a/internal/lamport/clock.go +++ b/internal/lamport/clock.go @@ -21,11 +21,11 @@ import "sync" var Default = Clock{} type Clock struct { - val uint64 + val int64 mut sync.Mutex } -func (c *Clock) Tick(v uint64) uint64 { +func (c *Clock) Tick(v int64) int64 { c.mut.Lock() if v > c.val { c.val = v + 1 diff --git a/internal/lamport/clock_test.go b/internal/lamport/clock_test.go index ded50742a..850b4e3b7 100644 --- a/internal/lamport/clock_test.go +++ b/internal/lamport/clock_test.go @@ -17,12 +17,12 @@ package lamport import "testing" -var inputs = []uint64{0, 42, 2, 3, 4, 8, 9, 33, 44, 112, 100} +var inputs = []int64{0, 42, 2, 3, 4, 8, 9, 33, 44, 112, 100} func TestClock(t *testing.T) { c := Clock{} - var prev uint64 + var prev int64 for _, input := range inputs { cur := c.Tick(input) if cur <= prev || cur <= input { diff --git a/internal/model/model.go b/internal/model/model.go index c3cc89a8e..a505e3a92 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -701,6 +701,10 @@ func (m *Model) Close(device protocol.DeviceID, err error) { // Request returns the specified data segment by reading it from local disk. // Implements the protocol.Model interface. func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset int64, size int) ([]byte, error) { + if offset < 0 || size < 0 { + return nil, ErrNoSuchFile + } + if !m.folderSharedWith(folder, deviceID) { l.Warnf("Request from %s for file %s in unshared folder %q", deviceID, name, folder) return nil, ErrNoSuchFile @@ -970,12 +974,12 @@ func sendIndexes(conn protocol.Connection, folder string, fs *db.FileSet, ignore } } -func sendIndexTo(initial bool, minLocalVer uint64, conn protocol.Connection, folder string, fs *db.FileSet, ignores *ignore.Matcher) (uint64, error) { +func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, folder string, fs *db.FileSet, ignores *ignore.Matcher) (int64, error) { deviceID := conn.ID() name := conn.Name() batch := make([]protocol.FileInfo, 0, indexBatchSize) currentBatchSize := 0 - maxLocalVer := uint64(0) + maxLocalVer := int64(0) var err error fs.WithHave(protocol.LocalDeviceID, func(fi db.FileIntf) bool { @@ -1349,7 +1353,7 @@ func (m *Model) Override(folder string) { // CurrentLocalVersion returns the change version for the given folder. // This is guaranteed to increment if the contents of the local folder has // changed. -func (m *Model) CurrentLocalVersion(folder string) uint64 { +func (m *Model) CurrentLocalVersion(folder string) int64 { m.fmut.RLock() fs, ok := m.folderFiles[folder] m.fmut.RUnlock() @@ -1365,7 +1369,7 @@ func (m *Model) CurrentLocalVersion(folder string) uint64 { // RemoteLocalVersion returns the change version for the given folder, as // sent by remote peers. This is guaranteed to increment if the contents of // the remote or global folder has changed. -func (m *Model) RemoteLocalVersion(folder string) uint64 { +func (m *Model) RemoteLocalVersion(folder string) int64 { m.fmut.RLock() defer m.fmut.RUnlock() @@ -1376,7 +1380,7 @@ func (m *Model) RemoteLocalVersion(folder string) uint64 { return 0 } - var ver uint64 + var ver int64 for _, n := range m.folderDevices[folder] { ver += fs.LocalVersion(n) } diff --git a/internal/model/model_test.go b/internal/model/model_test.go index 03361943f..2e8285832 100644 --- a/internal/model/model_test.go +++ b/internal/model/model_test.go @@ -110,6 +110,33 @@ func TestRequest(t *testing.T) { if bs != nil { t.Errorf("Unexpected non nil data on insecure file read: %q", string(bs)) } + + // Larger block than available + bs, err = m.Request(device1, "default", "foo", 0, 42) + if err == nil { + t.Error("Unexpected nil error on insecure file read") + } + if bs != nil { + t.Errorf("Unexpected non nil data on insecure file read: %q", string(bs)) + } + + // Negative offset + bs, err = m.Request(device1, "default", "foo", -4, 6) + if err == nil { + t.Error("Unexpected nil error on insecure file read") + } + if bs != nil { + t.Errorf("Unexpected non nil data on insecure file read: %q", string(bs)) + } + + // Negative size + bs, err = m.Request(device1, "default", "foo", 4, -4) + if err == nil { + t.Error("Unexpected nil error on insecure file read") + } + if bs != nil { + t.Errorf("Unexpected non nil data on insecure file read: %q", string(bs)) + } } func genFiles(n int) []protocol.FileInfo { diff --git a/internal/model/puller.go b/internal/model/puller.go index 679f8e7fe..c0fbccbb7 100644 --- a/internal/model/puller.go +++ b/internal/model/puller.go @@ -100,7 +100,7 @@ func (p *Puller) Serve() { p.model.setState(p.folder, FolderIdle) }() - var prevVer uint64 + var prevVer int64 var prevIgnoreHash string // We don't start pulling files until a scan has been completed. @@ -623,9 +623,9 @@ func (p *Puller) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocksSt folder: p.folder, tempName: tempName, realName: realName, - copyTotal: uint32(len(blocks)), - copyNeeded: uint32(len(blocks)), - reused: uint32(reused), + copyTotal: len(blocks), + copyNeeded: len(blocks), + reused: reused, } if debug { @@ -720,7 +720,7 @@ func (p *Puller) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pullBl for _, block := range state.blocks { buf = buf[:int(block.Size)] - found := p.model.finder.Iterate(block.Hash, func(folder, file string, index uint32) bool { + found := p.model.finder.Iterate(block.Hash, func(folder, file string, index int32) bool { path := filepath.Join(folderRoots[folder], file) var fd *os.File diff --git a/internal/model/puller_test.go b/internal/model/puller_test.go index 8673b49dd..d20283ee1 100644 --- a/internal/model/puller_test.go +++ b/internal/model/puller_test.go @@ -199,7 +199,7 @@ func TestCopierFinder(t *testing.T) { // Update index m.updateLocal("default", existingFile) - iterFn := func(folder, file string, index uint32) bool { + iterFn := func(folder, file string, index int32) bool { return true } @@ -264,7 +264,7 @@ func TestCopierFinder(t *testing.T) { // Test that updating a file removes it's old blocks from the blockmap func TestCopierCleanup(t *testing.T) { - iterFn := func(folder, file string, index uint32) bool { + iterFn := func(folder, file string, index int32) bool { return true } @@ -341,7 +341,7 @@ func TestLastResortPulling(t *testing.T) { // with a different name (causing to copy that particular block) file.Name = "newfile" - iterFn := func(folder, file string, index uint32) bool { + iterFn := func(folder, file string, index int32) bool { return true } diff --git a/internal/model/sharedpullerstate.go b/internal/model/sharedpullerstate.go index 9dbd801fd..db12870fb 100644 --- a/internal/model/sharedpullerstate.go +++ b/internal/model/sharedpullerstate.go @@ -33,27 +33,27 @@ type sharedPullerState struct { folder string tempName string realName string - reused uint32 // Number of blocks reused from temporary file + reused int // Number of blocks reused from temporary file // Mutable, must be locked for access err error // The first error we hit fd *os.File // The fd of the temp file - copyTotal uint32 // Total number of copy actions for the whole job - pullTotal uint32 // Total number of pull actions for the whole job - copyOrigin uint32 // Number of blocks copied from the original file - copyNeeded uint32 // Number of copy actions still pending - pullNeeded uint32 // Number of block pulls still pending + copyTotal int // Total number of copy actions for the whole job + pullTotal int // Total number of pull actions for the whole job + copyOrigin int // Number of blocks copied from the original file + copyNeeded int // Number of copy actions still pending + pullNeeded int // Number of block pulls still pending mut sync.Mutex // Protects the above } // A momentary state representing the progress of the puller type pullerProgress struct { - Total uint32 - Reused uint32 - CopiedFromOrigin uint32 - CopiedFromElsewhere uint32 - Pulled uint32 - Pulling uint32 + Total int + Reused int + CopiedFromOrigin int + CopiedFromElsewhere int + Pulled int + Pulling int BytesDone int64 BytesTotal int64 } diff --git a/internal/scanner/blocks.go b/internal/scanner/blocks.go index 96994a43e..291cbf920 100644 --- a/internal/scanner/blocks.go +++ b/internal/scanner/blocks.go @@ -46,7 +46,7 @@ func Blocks(r io.Reader, blocksize int, sizehint int64) ([]protocol.BlockInfo, e } b := protocol.BlockInfo{ - Size: uint32(n), + Size: int32(n), Offset: offset, Hash: hf.Sum(nil), }