Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Refactor: add and use db.NamespacedKV

This commit is contained in:
Jakob Borg 2015-01-17 20:53:33 +01:00
parent 6f2b4b96cf
commit 6f8ac2b61c
7 changed files with 267 additions and 185 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internal/db/blockmap.go
View File

@ -199,7 +199,7 @@ func (f *BlockFinder) Fix(folder, file string, index int32, oldHash, newHash []b
// file name (variable size)
func toBlockKey(hash []byte, folder, file string) []byte {
o := make([]byte, 1+64+32+len(file))
o[0] = keyTypeBlock
o[0] = KeyTypeBlock
copy(o[1:], []byte(folder))
copy(o[1+64:], []byte(hash))
copy(o[1+64+32:], []byte(file))
@ -210,7 +210,7 @@ func fromBlockKey(data []byte) (string, string) {
if len(data) < 1+64+32+1 {
panic("Incorrect key length")
}
if data[0] != keyTypeBlock {
if data[0] != KeyTypeBlock {
panic("Incorrect key type")
}

18
internal/db/leveldb.go
View File

@ -50,9 +50,11 @@ func clock(v int64) int64 {
}
const (
keyTypeDevice = iota
keyTypeGlobal
keyTypeBlock
KeyTypeDevice = iota
KeyTypeGlobal
KeyTypeBlock
KeyTypeDeviceStatistic
KeyTypeFolderStatistic
)
type fileVersion struct {
@ -112,7 +114,7 @@ const batchFlushSize = 64
// name (variable size)
func deviceKey(folder, device, file []byte) []byte {
k := make([]byte, 1+64+32+len(file))
k[0] = keyTypeDevice
k[0] = KeyTypeDevice
if len(folder) > 64 {
panic("folder name too long")
}
@ -145,7 +147,7 @@ func deviceKeyDevice(key []byte) []byte {
// name (variable size)
func globalKey(folder, file []byte) []byte {
k := make([]byte, 1+64+len(file))
k[0] = keyTypeGlobal
k[0] = KeyTypeGlobal
if len(folder) > 64 {
panic("folder name too long")
}
@ -915,7 +917,7 @@ func ldbListFolders(db *leveldb.DB) []string {
snap.Release()
}()
dbi := snap.NewIterator(util.BytesPrefix([]byte{keyTypeGlobal}), nil)
dbi := snap.NewIterator(util.BytesPrefix([]byte{KeyTypeGlobal}), nil)
defer dbi.Release()
folderExists := make(map[string]bool)
@ -953,7 +955,7 @@ func ldbDropFolder(db *leveldb.DB, folder []byte) {
}()
// Remove all items related to the given folder from the device->file bucket
dbi := snap.NewIterator(util.BytesPrefix([]byte{keyTypeDevice}), nil)
dbi := snap.NewIterator(util.BytesPrefix([]byte{KeyTypeDevice}), nil)
for dbi.Next() {
itemFolder := deviceKeyFolder(dbi.Key())
if bytes.Compare(folder, itemFolder) == 0 {
@ -963,7 +965,7 @@ func ldbDropFolder(db *leveldb.DB, folder []byte) {
dbi.Release()
// Remove all items related to the given folder from the global bucket
dbi = snap.NewIterator(util.BytesPrefix([]byte{keyTypeGlobal}), nil)
dbi = snap.NewIterator(util.BytesPrefix([]byte{KeyTypeGlobal}), nil)
for dbi.Next() {
itemFolder := globalKeyFolder(dbi.Key())
if bytes.Compare(folder, itemFolder) == 0 {

106
internal/db/namespaced.go Normal file
View File

@ -0,0 +1,106 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This program is free software: you can redistribute it and/or modify it
// under the terms of the GNU General Public License as published by the Free
// Software Foundation, either version 3 of the License, or (at your option)
// any later version.
//
// This program is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
// more details.
//
// You should have received a copy of the GNU General Public License along
// with this program. If not, see <http://www.gnu.org/licenses/>.
package db
import (
"encoding/binary"
"time"
"github.com/syndtr/goleveldb/leveldb"
)
// NamespacedKV is a simple key-value store using a specific namespace within
// a leveldb.
type NamespacedKV struct {
db *leveldb.DB
prefix []byte
}
// NewNamespacedKV returns a new NamespacedKV that lives in the namespace
// specified by the prefix.
func NewNamespacedKV(db *leveldb.DB, prefix string) *NamespacedKV {
return &NamespacedKV{
db: db,
prefix: []byte(prefix),
}
}
// PutInt64 stores a new int64. Any existing value (even if of another type)
// is overwritten.
func (n *NamespacedKV) PutInt64(key string, val int64) {
keyBs := append(n.prefix, []byte(key)...)
var valBs [8]byte
binary.BigEndian.PutUint64(valBs[:], uint64(val))
n.db.Put(keyBs, valBs[:], nil)
}
// Int64 returns the stored value interpreted as an int64 and a boolean that
// is false if no value was stored at the key.
func (n *NamespacedKV) Int64(key string) (int64, bool) {
keyBs := append(n.prefix, []byte(key)...)
valBs, err := n.db.Get(keyBs, nil)
if err != nil {
return 0, false
}
val := binary.BigEndian.Uint64(valBs)
return int64(val), true
}
// PutTime stores a new time.Time. Any existing value (even if of another
// type) is overwritten.
func (n *NamespacedKV) PutTime(key string, val time.Time) {
keyBs := append(n.prefix, []byte(key)...)
valBs, _ := val.MarshalBinary() // never returns an error
n.db.Put(keyBs, valBs, nil)
}
// Time returns the stored value interpreted as a time.Time and a boolean
// that is false if no value was stored at the key.
func (n NamespacedKV) Time(key string) (time.Time, bool) {
var t time.Time
keyBs := append(n.prefix, []byte(key)...)
valBs, err := n.db.Get(keyBs, nil)
if err != nil {
return t, false
}
err = t.UnmarshalBinary(valBs)
return t, err == nil
}
// PutString stores a new string. Any existing value (even if of another type)
// is overwritten.
func (n *NamespacedKV) PutString(key, val string) {
keyBs := append(n.prefix, []byte(key)...)
n.db.Put(keyBs, []byte(val), nil)
}
// String returns the stored value interpreted as a string and a boolean that
// is false if no value was stored at the key.
func (n NamespacedKV) String(key string) (string, bool) {
keyBs := append(n.prefix, []byte(key)...)
valBs, err := n.db.Get(keyBs, nil)
if err != nil {
return "", false
}
return string(valBs), true
}
// Delete deletes the specified key. It is allowed to delete a nonexistent
// key.
func (n NamespacedKV) Delete(key string) {
keyBs := append(n.prefix, []byte(key)...)
n.db.Delete(keyBs, nil)
}

101
internal/db/namespaced_test.go Normal file
View File

@ -0,0 +1,101 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This program is free software: you can redistribute it and/or modify it
// under the terms of the GNU General Public License as published by the Free
// Software Foundation, either version 3 of the License, or (at your option)
// any later version.
//
// This program is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
// more details.
//
// You should have received a copy of the GNU General Public License along
// with this program. If not, see <http://www.gnu.org/licenses/>.
package db
import (
"testing"
"time"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/storage"
)
func TestNamespacedInt(t *testing.T) {
ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
if err != nil {
t.Fatal(err)
}
n1 := NewNamespacedKV(ldb, "foo")
n2 := NewNamespacedKV(ldb, "bar")
// Key is missing to start with
if v, ok := n1.Int64("test"); v != 0 || ok {
t.Errorf("Incorrect return v %v != 0 || ok %v != false", v, ok)
}
n1.PutInt64("test", 42)
// It should now exist in n1
if v, ok := n1.Int64("test"); v != 42 || !ok {
t.Errorf("Incorrect return v %v != 42 || ok %v != true", v, ok)
}
// ... but not in n2, which is in a different namespace
if v, ok := n2.Int64("test"); v != 0 || ok {
t.Errorf("Incorrect return v %v != 0 || ok %v != false", v, ok)
}
n1.Delete("test")
// It should no longer exist
if v, ok := n1.Int64("test"); v != 0 || ok {
t.Errorf("Incorrect return v %v != 0 || ok %v != false", v, ok)
}
}
func TestNamespacedTime(t *testing.T) {
ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
if err != nil {
t.Fatal(err)
}
n1 := NewNamespacedKV(ldb, "foo")
if v, ok := n1.Time("test"); v != (time.Time{}) || ok {
t.Errorf("Incorrect return v %v != %v || ok %v != false", v, time.Time{}, ok)
}
now := time.Now()
n1.PutTime("test", now)
if v, ok := n1.Time("test"); v != now || !ok {
t.Errorf("Incorrect return v %v != %v || ok %v != true", v, now, ok)
}
}
func TestNamespacedString(t *testing.T) {
ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
if err != nil {
t.Fatal(err)
}
n1 := NewNamespacedKV(ldb, "foo")
if v, ok := n1.String("test"); v != "" || ok {
t.Errorf("Incorrect return v %q != \"\" || ok %v != false", v, ok)
}
n1.PutString("test", "yo")
if v, ok := n1.String("test"); v != "yo" || !ok {
t.Errorf("Incorrect return v %q != \"yo\" || ok %v != true", v, ok)
}
}

70
internal/stats/device.go
View File

@ -19,91 +19,45 @@ import (
"time"
"github.com/syncthing/protocol"
"github.com/syncthing/syncthing/internal/db"
"github.com/syndtr/goleveldb/leveldb"
)
const (
deviceStatisticTypeLastSeen = iota
)
var deviceStatisticsTypes = []byte{
deviceStatisticTypeLastSeen,
}
type DeviceStatistics struct {
LastSeen time.Time
}
type DeviceStatisticsReference struct {
db *leveldb.DB
ns *db.NamespacedKV
device protocol.DeviceID
}
func NewDeviceStatisticsReference(db *leveldb.DB, device protocol.DeviceID) *DeviceStatisticsReference {
func NewDeviceStatisticsReference(ldb *leveldb.DB, device protocol.DeviceID) *DeviceStatisticsReference {
prefix := string(db.KeyTypeDeviceStatistic) + device.String()
return &DeviceStatisticsReference{
db: db,
ns: db.NewNamespacedKV(ldb, prefix),
device: device,
}
}
func (s *DeviceStatisticsReference) key(stat byte) []byte {
k := make([]byte, 1+1+32)
k[0] = keyTypeDeviceStatistic
k[1] = stat
copy(k[1+1:], s.device[:])
return k
}
func (s *DeviceStatisticsReference) GetLastSeen() time.Time {
value, err := s.db.Get(s.key(deviceStatisticTypeLastSeen), nil)
if err != nil {
if err != leveldb.ErrNotFound {
l.Warnln("DeviceStatisticsReference: Failed loading last seen value for", s.device, ":", err)
}
return time.Unix(0, 0)
}
rtime := time.Time{}
err = rtime.UnmarshalBinary(value)
if err != nil {
l.Warnln("DeviceStatisticsReference: Failed parsing last seen value for", s.device, ":", err)
t, ok := s.ns.Time("lastSeen")
if !ok {
// The default here is 1970-01-01 as opposed to the default
// time.Time{} from s.ns
return time.Unix(0, 0)
}
if debug {
l.Debugln("stats.DeviceStatisticsReference.GetLastSeen:", s.device, rtime)
l.Debugln("stats.DeviceStatisticsReference.GetLastSeen:", s.device, t)
}
return rtime
return t
}
func (s *DeviceStatisticsReference) WasSeen() {
if debug {
l.Debugln("stats.DeviceStatisticsReference.WasSeen:", s.device)
}
value, err := time.Now().MarshalBinary()
if err != nil {
l.Warnln("DeviceStatisticsReference: Failed serializing last seen value for", s.device, ":", err)
return
}
err = s.db.Put(s.key(deviceStatisticTypeLastSeen), value, nil)
if err != nil {
l.Warnln("Failed serializing last seen value for", s.device, ":", err)
}
}
// Never called, maybe because it's worth while to keep the data
// or maybe because we have no easy way of knowing that a device has been removed.
func (s *DeviceStatisticsReference) Delete() error {
for _, stype := range deviceStatisticsTypes {
err := s.db.Delete(s.key(stype), nil)
if debug && err == nil {
l.Debugln("stats.DeviceStatisticsReference.Delete:", s.device, stype)
}
if err != nil && err != leveldb.ErrNotFound {
return err
}
}
return nil
s.ns.PutTime("lastSeen", time.Now())
}
func (s *DeviceStatisticsReference) GetStatistics() DeviceStatistics {

131
internal/stats/folder.go
View File

@ -16,118 +16,59 @@
package stats
import (
"encoding/binary"
"time"
"github.com/syncthing/syncthing/internal/db"
"github.com/syndtr/goleveldb/leveldb"
)
const (
folderStatisticTypeLastFile = iota
)
var folderStatisticsTypes = []byte{
folderStatisticTypeLastFile,
}
type FolderStatistics struct {
LastFile *LastFile
LastFile LastFile
}
type FolderStatisticsReference struct {
db *leveldb.DB
ns *db.NamespacedKV
folder string
}
func NewFolderStatisticsReference(db *leveldb.DB, folder string) *FolderStatisticsReference {
return &FolderStatisticsReference{
db: db,
folder: folder,
}
}
func (s *FolderStatisticsReference) key(stat byte) []byte {
k := make([]byte, 1+1+64)
k[0] = keyTypeFolderStatistic
k[1] = stat
copy(k[1+1:], s.folder[:])
return k
}
func (s *FolderStatisticsReference) GetLastFile() *LastFile {
value, err := s.db.Get(s.key(folderStatisticTypeLastFile), nil)
if err != nil {
if err != leveldb.ErrNotFound {
l.Warnln("FolderStatisticsReference: Failed loading last file filename value for", s.folder, ":", err)
}
return nil
}
file := LastFile{}
err = file.UnmarshalBinary(value)
if err != nil {
l.Warnln("FolderStatisticsReference: Failed loading last file value for", s.folder, ":", err)
return nil
}
return &file
}
func (s *FolderStatisticsReference) ReceivedFile(filename string) {
f := LastFile{
Filename: filename,
At: time.Now(),
}
if debug {
l.Debugln("stats.FolderStatisticsReference.ReceivedFile:", s.folder)
}
value, err := f.MarshalBinary()
if err != nil {
l.Warnln("FolderStatisticsReference: Failed serializing last file value for", s.folder, ":", err)
return
}
err = s.db.Put(s.key(folderStatisticTypeLastFile), value, nil)
if err != nil {
l.Warnln("Failed update last file value for", s.folder, ":", err)
}
}
// Never called, maybe because it's worth while to keep the data
// or maybe because we have no easy way of knowing that a folder has been removed.
func (s *FolderStatisticsReference) Delete() error {
for _, stype := range folderStatisticsTypes {
err := s.db.Delete(s.key(stype), nil)
if debug && err == nil {
l.Debugln("stats.FolderStatisticsReference.Delete:", s.folder, stype)
}
if err != nil && err != leveldb.ErrNotFound {
return err
}
}
return nil
}
func (s *FolderStatisticsReference) GetStatistics() FolderStatistics {
return FolderStatistics{
LastFile: s.GetLastFile(),
}
}
type LastFile struct {
At time.Time
Filename string
}
func (f *LastFile) MarshalBinary() ([]byte, error) {
buf := make([]byte, 8+len(f.Filename))
binary.BigEndian.PutUint64(buf[:8], uint64(f.At.Unix()))
copy(buf[8:], []byte(f.Filename))
return buf, nil
func NewFolderStatisticsReference(ldb *leveldb.DB, folder string) *FolderStatisticsReference {
prefix := string(db.KeyTypeFolderStatistic) + folder
return &FolderStatisticsReference{
ns: db.NewNamespacedKV(ldb, prefix),
folder: folder,
}
}
func (f *LastFile) UnmarshalBinary(buf []byte) error {
f.At = time.Unix(int64(binary.BigEndian.Uint64(buf[:8])), 0)
f.Filename = string(buf[8:])
return nil
func (s *FolderStatisticsReference) GetLastFile() LastFile {
at, ok := s.ns.Time("lastFileAt")
if !ok {
return LastFile{}
}
file, ok := s.ns.String("lastFileName")
if !ok {
return LastFile{}
}
return LastFile{
At: at,
Filename: file,
}
}
func (s *FolderStatisticsReference) ReceivedFile(filename string) {
if debug {
l.Debugln("stats.FolderStatisticsReference.ReceivedFile:", s.folder, filename)
}
s.ns.PutTime("lastFileAt", time.Now())
s.ns.PutString("lastFileName", filename)
}
func (s *FolderStatisticsReference) GetStatistics() FolderStatistics {
return FolderStatistics{
LastFile: s.GetLastFile(),
}
}

22
internal/stats/leveldb.go
View File

@ -1,22 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This program is free software: you can redistribute it and/or modify it
// under the terms of the GNU General Public License as published by the Free
// Software Foundation, either version 3 of the License, or (at your option)
// any later version.
//
// This program is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
// more details.
//
// You should have received a copy of the GNU General Public License along
// with this program. If not, see <http://www.gnu.org/licenses/>.
package stats
// Same key space as files/leveldb.go keyType* constants
const (
keyTypeDeviceStatistic = iota + 30
keyTypeFolderStatistic
)