Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/db: Refactor: use a Lowlevel type underneath Instance (ref #5198) (#5212) 

This adds a thin type that holds the state associated with the
leveldb.DB, leaving the huge Instance type more or less stateless. Also
moves some keying stuff into the DB package so that other packages need
not know the keying specifics.

(This does not, yet, fix the cmd/stindex program, in order to keep the
diff size down. Hence the keying constants are still exported.)
This commit is contained in:
Jakob Borg 2018-10-10 11:34:24 +02:00 committed by GitHub
parent 8e645ab782
commit b50d57b7fd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 382 additions and 374 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/stindex/dump.go
View File

@ -16,7 +16,7 @@ import (
"github.com/syncthing/syncthing/lib/protocol"
)
func dump(ldb *db.Instance) {
func dump(ldb *db.Lowlevel) {
it := ldb.NewIterator(nil, nil)
for it.Next() {
key := it.Key()

2
cmd/stindex/dumpsize.go
View File

@ -37,7 +37,7 @@ func (h *ElementHeap) Pop() interface{} {
return x
}
func dumpsize(ldb *db.Instance) {
func dumpsize(ldb *db.Lowlevel) {
h := &ElementHeap{}
heap.Init(h)

10
cmd/syncthing/main.go
View File

@ -712,11 +712,13 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
if err != nil {
l.Fatalln("Error opening database:", err)
}
if err := db.UpdateSchema(ldb); err != nil {
l.Fatalln("Database schema:", err)
}
if runtimeOptions.resetDeltaIdxs {
l.Infoln("Reinitializing delta index IDs")
ldb.DropLocalDeltaIndexIDs()
ldb.DropRemoteDeltaIndexIDs()
db.DropDeltaIndexIDs(ldb)
}
protectedFiles := []string{
@ -737,7 +739,7 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
// Grab the previously running version string from the database.
miscDB := db.NewNamespacedKV(ldb, string(db.KeyTypeMiscData))
miscDB := db.NewMiscDataNamespace(ldb)
prevVersion, _ := miscDB.String("prevVersion")
// Strip away prerelease/beta stuff and just compare the release
@ -753,7 +755,7 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
// Drop delta indexes in case we've changed random stuff we
// shouldn't have. We will resend our index on next connect.
ldb.DropLocalDeltaIndexIDs()
db.DropDeltaIndexIDs(ldb)
// Remember the new version.
miscDB.PutString("prevVersion", Version)

2
lib/db/benchmark_test.go
View File

@ -45,7 +45,7 @@ func lazyInitBenchFileSet() {
replace(benchS, protocol.LocalDeviceID, firstHalf)
}
func tempDB() (*db.Instance, string) {
func tempDB() (*db.Lowlevel, string) {
dir, err := ioutil.TempDir("", "syncthing")
if err != nil {
panic(err)

10
lib/db/blockmap.go
View File

@ -22,14 +22,14 @@ var blockFinder *BlockFinder
const maxBatchSize = 1000
type BlockMap struct {
db *Instance
db *Lowlevel
folder uint32
}
func NewBlockMap(db *Instance, folder uint32) *BlockMap {
func NewBlockMap(db *Lowlevel, folder string) *BlockMap {
return &BlockMap{
db: db,
folder: folder,
folder: db.folderIdx.ID([]byte(folder)),
}
}
@ -139,10 +139,10 @@ func (m *BlockMap) blockKeyInto(o, hash []byte, file string) []byte {
}
type BlockFinder struct {
db *Instance
db *Lowlevel
}
func NewBlockFinder(db *Instance) *BlockFinder {
func NewBlockFinder(db *Lowlevel) *BlockFinder {
if blockFinder != nil {
return blockFinder
}

10
lib/db/blockmap_test.go
View File

@ -48,14 +48,14 @@ func init() {
}
}
func setup() (*Instance, *BlockFinder) {
func setup() (*Lowlevel, *BlockFinder) {
// Setup
db := OpenMemory()
return db, NewBlockFinder(db)
}
func dbEmpty(db *Instance) bool {
func dbEmpty(db *Lowlevel) bool {
iter := db.NewIterator(util.BytesPrefix([]byte{KeyTypeBlock}), nil)
defer iter.Release()
return !iter.Next()
@ -68,7 +68,7 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
t.Fatal("db not empty")
}
m := NewBlockMap(db, db.folderIdx.ID([]byte("folder1")))
m := NewBlockMap(db, "folder1")
f3.Type = protocol.FileInfoTypeDirectory
@ -152,8 +152,8 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
func TestBlockFinderLookup(t *testing.T) {
db, f := setup()
m1 := NewBlockMap(db, db.folderIdx.ID([]byte("folder1")))
m2 := NewBlockMap(db, db.folderIdx.ID([]byte("folder2")))
m1 := NewBlockMap(db, "folder1")
m2 := NewBlockMap(db, "folder2")
err := m1.Add([]protocol.FileInfo{f1})
if err != nil {

6
lib/db/keyer_test.go
View File

@ -16,7 +16,7 @@ func TestDeviceKey(t *testing.T) {
dev := []byte("device67890123456789012345678901")
name := []byte("name")
db := OpenMemory()
db := newInstance(OpenMemory())
key := db.keyer.GenerateDeviceFileKey(nil, fld, dev, name)
@ -44,7 +44,7 @@ func TestGlobalKey(t *testing.T) {
fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
name := []byte("name")
db := OpenMemory()
db := newInstance(OpenMemory())
key := db.keyer.GenerateGlobalVersionKey(nil, fld, name)
@ -69,7 +69,7 @@ func TestGlobalKey(t *testing.T) {
func TestSequenceKey(t *testing.T) {
fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
db := OpenMemory()
db := newInstance(OpenMemory())
const seq = 1234567890
key := db.keyer.GenerateSequenceKey(nil, fld, seq)

2
lib/db/leveldb.go
View File

@ -31,7 +31,7 @@ func (vl VersionList) String() string {
// update brings the VersionList up to date with file. It returns the updated
// VersionList, a potentially removed old FileVersion and its index, as well as
// the index where the new FileVersion was inserted.
func (vl VersionList) update(folder, device []byte, file protocol.FileInfo, db *Instance) (_ VersionList, removedFV FileVersion, removedAt int, insertedAt int) {
func (vl VersionList) update(folder, device []byte, file protocol.FileInfo, db *instance) (_ VersionList, removedFV FileVersion, removedAt int, insertedAt int) {
removedAt, insertedAt = -1, -1
for i, v := range vl.Versions {
if bytes.Equal(v.Device, device) {

186
lib/db/leveldb_dbinstance.go
View File

@ -10,87 +10,28 @@ import (
"bytes"
"encoding/binary"
"fmt"
"os"
"sort"
"strings"
"sync/atomic"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/errors"
"github.com/syndtr/goleveldb/leveldb/iterator"
"github.com/syndtr/goleveldb/leveldb/opt"
"github.com/syndtr/goleveldb/leveldb/storage"
"github.com/syndtr/goleveldb/leveldb/util"
)
type deletionHandler func(t readWriteTransaction, folder, device, name []byte, dbi iterator.Iterator)
type Instance struct {
committed int64 // this must be the first attribute in the struct to ensure 64 bit alignment on 32 bit plaforms
*leveldb.DB
location string
folderIdx *smallIndex
deviceIdx *smallIndex
keyer keyer
type instance struct {
*Lowlevel
keyer keyer
}
func Open(file string) (*Instance, error) {
opts := &opt.Options{
OpenFilesCacheCapacity: 100,
WriteBuffer: 4 << 20,
func newInstance(ll *Lowlevel) *instance {
return &instance{
Lowlevel: ll,
keyer: newDefaultKeyer(ll.folderIdx, ll.deviceIdx),
}
db, err := leveldb.OpenFile(file, opts)
if leveldbIsCorrupted(err) {
db, err = leveldb.RecoverFile(file, opts)
}
if leveldbIsCorrupted(err) {
// The database is corrupted, and we've tried to recover it but it
// didn't work. At this point there isn't much to do beyond dropping
// the database and reindexing...
l.Infoln("Database corruption detected, unable to recover. Reinitializing...")
if err := os.RemoveAll(file); err != nil {
return nil, errorSuggestion{err, "failed to delete corrupted database"}
}
db, err = leveldb.OpenFile(file, opts)
}
if err != nil {
return nil, errorSuggestion{err, "is another instance of Syncthing running?"}
}
return newDBInstance(db, file)
}
func OpenMemory() *Instance {
db, _ := leveldb.Open(storage.NewMemStorage(), nil)
ldb, _ := newDBInstance(db, "<memory>")
return ldb
}
func newDBInstance(db *leveldb.DB, location string) (*Instance, error) {
i := &Instance{
DB: db,
location: location,
folderIdx: newSmallIndex(db, []byte{KeyTypeFolderIdx}),
deviceIdx: newSmallIndex(db, []byte{KeyTypeDeviceIdx}),
}
i.keyer = newDefaultKeyer(i.folderIdx, i.deviceIdx)
err := i.updateSchema()
return i, err
}
// Committed returns the number of items committed to the database since startup
func (db *Instance) Committed() int64 {
return atomic.LoadInt64(&db.committed)
}
// Location returns the filesystem path where the database is stored
func (db *Instance) Location() string {
return db.location
}
func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, meta *metadataTracker) {
func (db *instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, meta *metadataTracker) {
t := db.newReadWriteTransaction()
defer t.close()
@ -131,7 +72,7 @@ func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, m
}
}
func (db *Instance) addSequences(folder []byte, fs []protocol.FileInfo) {
func (db *instance) addSequences(folder []byte, fs []protocol.FileInfo) {
t := db.newReadWriteTransaction()
defer t.close()
@ -146,7 +87,7 @@ func (db *Instance) addSequences(folder []byte, fs []protocol.FileInfo) {
}
}
func (db *Instance) removeSequences(folder []byte, fs []protocol.FileInfo) {
func (db *instance) removeSequences(folder []byte, fs []protocol.FileInfo) {
t := db.newReadWriteTransaction()
defer t.close()
@ -158,7 +99,7 @@ func (db *Instance) removeSequences(folder []byte, fs []protocol.FileInfo) {
}
}
func (db *Instance) withHave(folder, device, prefix []byte, truncate bool, fn Iterator) {
func (db *instance) withHave(folder, device, prefix []byte, truncate bool, fn Iterator) {
if len(prefix) > 0 {
unslashedPrefix := prefix
if bytes.HasSuffix(prefix, []byte{'/'}) {
@ -199,7 +140,7 @@ func (db *Instance) withHave(folder, device, prefix []byte, truncate bool, fn It
}
}
func (db *Instance) withHaveSequence(folder []byte, startSeq int64, fn Iterator) {
func (db *instance) withHaveSequence(folder []byte, startSeq int64, fn Iterator) {
t := db.newReadOnlyTransaction()
defer t.close()
@ -226,7 +167,7 @@ func (db *Instance) withHaveSequence(folder []byte, startSeq int64, fn Iterator)
}
}
func (db *Instance) withAllFolderTruncated(folder []byte, fn func(device []byte, f FileInfoTruncated) bool) {
func (db *instance) withAllFolderTruncated(folder []byte, fn func(device []byte, f FileInfoTruncated) bool) {
t := db.newReadWriteTransaction()
defer t.close()
@ -271,14 +212,14 @@ func (db *Instance) withAllFolderTruncated(folder []byte, fn func(device []byte,
}
}
func (db *Instance) getFile(key []byte) (protocol.FileInfo, bool) {
func (db *instance) getFile(key []byte) (protocol.FileInfo, bool) {
if f, ok := db.getFileTrunc(key, false); ok {
return f.(protocol.FileInfo), true
}
return protocol.FileInfo{}, false
}
func (db *Instance) getFileTrunc(key []byte, trunc bool) (FileIntf, bool) {
func (db *instance) getFileTrunc(key []byte, trunc bool) (FileIntf, bool) {
bs, err := db.Get(key, nil)
if err == leveldb.ErrNotFound {
return nil, false
@ -296,7 +237,7 @@ func (db *Instance) getFileTrunc(key []byte, trunc bool) (FileIntf, bool) {
return f, true
}
func (db *Instance) getGlobal(folder, file []byte, truncate bool) (FileIntf, bool) {
func (db *instance) getGlobal(folder, file []byte, truncate bool) (FileIntf, bool) {
t := db.newReadOnlyTransaction()
defer t.close()
@ -304,7 +245,7 @@ func (db *Instance) getGlobal(folder, file []byte, truncate bool) (FileIntf, boo
return f, ok
}
func (db *Instance) getGlobalInto(t readOnlyTransaction, gk, dk, folder, file []byte, truncate bool) ([]byte, []byte, FileIntf, bool) {
func (db *instance) getGlobalInto(t readOnlyTransaction, gk, dk, folder, file []byte, truncate bool) ([]byte, []byte, FileIntf, bool) {
gk = db.keyer.GenerateGlobalVersionKey(gk, folder, file)
bs, err := t.Get(gk, nil)
@ -325,7 +266,7 @@ func (db *Instance) getGlobalInto(t readOnlyTransaction, gk, dk, folder, file []
return gk, dk, nil, false
}
func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator) {
func (db *instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator) {
if len(prefix) > 0 {
unslashedPrefix := prefix
if bytes.HasSuffix(prefix, []byte{'/'}) {
@ -370,7 +311,7 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator
}
}
func (db *Instance) availability(folder, file []byte) []protocol.DeviceID {
func (db *instance) availability(folder, file []byte) []protocol.DeviceID {
k := db.keyer.GenerateGlobalVersionKey(nil, folder, file)
bs, err := db.Get(k, nil)
if err == leveldb.ErrNotFound {
@ -401,7 +342,7 @@ func (db *Instance) availability(folder, file []byte) []protocol.DeviceID {
return devices
}
func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator) {
func (db *instance) withNeed(folder, device []byte, truncate bool, fn Iterator) {
if bytes.Equal(device, protocol.LocalDeviceID[:]) {
db.withNeedLocal(folder, truncate, fn)
return
@ -473,7 +414,7 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator)
}
}
func (db *Instance) withNeedLocal(folder []byte, truncate bool, fn Iterator) {
func (db *instance) withNeedLocal(folder []byte, truncate bool, fn Iterator) {
t := db.newReadOnlyTransaction()
defer t.close()
@ -495,31 +436,7 @@ func (db *Instance) withNeedLocal(folder []byte, truncate bool, fn Iterator) {
}
}
func (db *Instance) ListFolders() []string {
t := db.newReadOnlyTransaction()
defer t.close()
dbi := t.NewIterator(util.BytesPrefix([]byte{KeyTypeGlobal}), nil)
defer dbi.Release()
folderExists := make(map[string]bool)
for dbi.Next() {
folder, ok := db.keyer.FolderFromGlobalVersionKey(dbi.Key())
if ok && !folderExists[string(folder)] {
folderExists[string(folder)] = true
}
}
folders := make([]string, 0, len(folderExists))
for k := range folderExists {
folders = append(folders, k)
}
sort.Strings(folders)
return folders
}
func (db *Instance) dropFolder(folder []byte) {
func (db *instance) dropFolder(folder []byte) {
t := db.newReadWriteTransaction()
defer t.close()
@ -537,7 +454,7 @@ func (db *Instance) dropFolder(folder []byte) {
}
}
func (db *Instance) dropDeviceFolder(device, folder []byte, meta *metadataTracker) {
func (db *instance) dropDeviceFolder(device, folder []byte, meta *metadataTracker) {
t := db.newReadWriteTransaction()
defer t.close()
@ -556,7 +473,7 @@ func (db *Instance) dropDeviceFolder(device, folder []byte, meta *metadataTracke
}
}
func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) {
func (db *instance) checkGlobals(folder []byte, meta *metadataTracker) {
t := db.newReadWriteTransaction()
defer t.close()
@ -604,7 +521,7 @@ func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) {
l.Debugf("db check completed for %q", folder)
}
func (db *Instance) getIndexID(device, folder []byte) protocol.IndexID {
func (db *instance) getIndexID(device, folder []byte) protocol.IndexID {
key := db.keyer.GenerateIndexIDKey(nil, device, folder)
cur, err := db.Get(key, nil)
if err != nil {
@ -619,7 +536,7 @@ func (db *Instance) getIndexID(device, folder []byte) protocol.IndexID {
return id
}
func (db *Instance) setIndexID(device, folder []byte, id protocol.IndexID) {
func (db *instance) setIndexID(device, folder []byte, id protocol.IndexID) {
key := db.keyer.GenerateIndexIDKey(nil, device, folder)
bs, _ := id.Marshal() // marshalling can't fail
if err := db.Put(key, bs, nil); err != nil {
@ -627,44 +544,15 @@ func (db *Instance) setIndexID(device, folder []byte, id protocol.IndexID) {
}
}
// DropLocalDeltaIndexIDs removes all index IDs for the local device ID from
// the database. This will cause a full index transmission on the next
// connection.
func (db *Instance) DropLocalDeltaIndexIDs() {
db.dropDeltaIndexIDs(true)
}
// DropRemoteDeltaIndexIDs removes all index IDs for the other devices than
// the local one from the database. This will cause them to send us a full
// index on the next connection.
func (db *Instance) DropRemoteDeltaIndexIDs() {
db.dropDeltaIndexIDs(false)
}
func (db *Instance) dropDeltaIndexIDs(local bool) {
t := db.newReadWriteTransaction()
defer t.close()
dbi := t.NewIterator(util.BytesPrefix([]byte{KeyTypeIndexID}), nil)
defer dbi.Release()
for dbi.Next() {
device, _ := db.keyer.DeviceFromIndexIDKey(dbi.Key())
if bytes.Equal(device, protocol.LocalDeviceID[:]) == local {
t.Delete(dbi.Key())
}
}
}
func (db *Instance) dropMtimes(folder []byte) {
func (db *instance) dropMtimes(folder []byte) {
db.dropPrefix(db.keyer.GenerateMtimesKey(nil, folder))
}
func (db *Instance) dropFolderMeta(folder []byte) {
func (db *instance) dropFolderMeta(folder []byte) {
db.dropPrefix(db.keyer.GenerateFolderMetaKey(nil, folder))
}
func (db *Instance) dropPrefix(prefix []byte) {
func (db *instance) dropPrefix(prefix []byte) {
t := db.newReadWriteTransaction()
defer t.close()
@ -701,22 +589,6 @@ func unmarshalVersionList(data []byte) (VersionList, bool) {
return vl, true
}
// A "better" version of leveldb's errors.IsCorrupted.
func leveldbIsCorrupted(err error) bool {
switch {
case err == nil:
return false
case errors.IsCorrupted(err):
return true
case strings.Contains(err.Error(), "corrupted"):
return true
}
return false
}
type errorSuggestion struct {
inner error
suggestion string

23
lib/db/leveldb_dbinstance_updateschema.go
View File

@ -38,8 +38,17 @@ func (e databaseDowngradeError) Error() string {
return fmt.Sprintf("Syncthing %s required", e.minSyncthingVersion)
}
func (db *Instance) updateSchema() error {
miscDB := NewNamespacedKV(db, string(KeyTypeMiscData))
func UpdateSchema(ll *Lowlevel) error {
updater := &schemaUpdater{newInstance(ll)}
return updater.updateSchema()
}
type schemaUpdater struct {
*instance
}
func (db *schemaUpdater) updateSchema() error {
miscDB := NewMiscDataNamespace(db.Lowlevel)
prevVersion, _ := miscDB.Int64("dbVersion")
if prevVersion > dbVersion {
@ -77,7 +86,7 @@ func (db *Instance) updateSchema() error {
return nil
}
func (db *Instance) updateSchema0to1() {
func (db *schemaUpdater) updateSchema0to1() {
t := db.newReadWriteTransaction()
defer t.close()
@ -159,7 +168,7 @@ func (db *Instance) updateSchema0to1() {
// updateSchema1to2 introduces a sequenceKey->deviceKey bucket for local items
// to allow iteration in sequence order (simplifies sending indexes).
func (db *Instance) updateSchema1to2() {
func (db *schemaUpdater) updateSchema1to2() {
t := db.newReadWriteTransaction()
defer t.close()
@ -178,7 +187,7 @@ func (db *Instance) updateSchema1to2() {
}
// updateSchema2to3 introduces a needKey->nil bucket for locally needed files.
func (db *Instance) updateSchema2to3() {
func (db *schemaUpdater) updateSchema2to3() {
t := db.newReadWriteTransaction()
defer t.close()
@ -209,7 +218,7 @@ func (db *Instance) updateSchema2to3() {
// release candidates (dbVersion 3 and 4)
// https://github.com/syncthing/syncthing/issues/5007
// https://github.com/syncthing/syncthing/issues/5053
func (db *Instance) updateSchemaTo5() {
func (db *schemaUpdater) updateSchemaTo5() {
t := db.newReadWriteTransaction()
var nk []byte
for _, folderStr := range db.ListFolders() {
@ -221,7 +230,7 @@ func (db *Instance) updateSchemaTo5() {
db.updateSchema2to3()
}
func (db *Instance) updateSchema5to6() {
func (db *schemaUpdater) updateSchema5to6() {
// For every local file with the Invalid bit set, clear the Invalid bit and
// set LocalFlags = FlagLocalIgnored.

122
lib/db/leveldb_test.go
View File

@ -7,107 +7,20 @@
package db
import (
"os"
"testing"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
)
func TestDropIndexIDs(t *testing.T) {
db := OpenMemory()
d1 := []byte("device67890123456789012345678901")
d2 := []byte("device12345678901234567890123456")
// Set some index IDs
db.setIndexID(protocol.LocalDeviceID[:], []byte("foo"), 1)
db.setIndexID(protocol.LocalDeviceID[:], []byte("bar"), 2)
db.setIndexID(d1, []byte("foo"), 3)
db.setIndexID(d1, []byte("bar"), 4)
db.setIndexID(d2, []byte("foo"), 5)
db.setIndexID(d2, []byte("bar"), 6)
// Verify them
if db.getIndexID(protocol.LocalDeviceID[:], []byte("foo")) != 1 {
t.Fatal("fail local 1")
}
if db.getIndexID(protocol.LocalDeviceID[:], []byte("bar")) != 2 {
t.Fatal("fail local 2")
}
if db.getIndexID(d1, []byte("foo")) != 3 {
t.Fatal("fail remote 1")
}
if db.getIndexID(d1, []byte("bar")) != 4 {
t.Fatal("fail remote 2")
}
if db.getIndexID(d2, []byte("foo")) != 5 {
t.Fatal("fail remote 3")
}
if db.getIndexID(d2, []byte("bar")) != 6 {
t.Fatal("fail remote 4")
}
// Drop the local ones, verify only they got dropped
db.DropLocalDeltaIndexIDs()
if db.getIndexID(protocol.LocalDeviceID[:], []byte("foo")) != 0 {
t.Fatal("fail local 1")
}
if db.getIndexID(protocol.LocalDeviceID[:], []byte("bar")) != 0 {
t.Fatal("fail local 2")
}
if db.getIndexID(d1, []byte("foo")) != 3 {
t.Fatal("fail remote 1")
}
if db.getIndexID(d1, []byte("bar")) != 4 {
t.Fatal("fail remote 2")
}
if db.getIndexID(d2, []byte("foo")) != 5 {
t.Fatal("fail remote 3")
}
if db.getIndexID(d2, []byte("bar")) != 6 {
t.Fatal("fail remote 4")
}
// Set local ones again
db.setIndexID(protocol.LocalDeviceID[:], []byte("foo"), 1)
db.setIndexID(protocol.LocalDeviceID[:], []byte("bar"), 2)
// Drop the remote ones, verify only they got dropped
db.DropRemoteDeltaIndexIDs()
if db.getIndexID(protocol.LocalDeviceID[:], []byte("foo")) != 1 {
t.Fatal("fail local 1")
}
if db.getIndexID(protocol.LocalDeviceID[:], []byte("bar")) != 2 {
t.Fatal("fail local 2")
}
if db.getIndexID(d1, []byte("foo")) != 0 {
t.Fatal("fail remote 1")
}
if db.getIndexID(d1, []byte("bar")) != 0 {
t.Fatal("fail remote 2")
}
if db.getIndexID(d2, []byte("foo")) != 0 {
t.Fatal("fail remote 3")
}
if db.getIndexID(d2, []byte("bar")) != 0 {
t.Fatal("fail remote 4")
}
}
func TestIgnoredFiles(t *testing.T) {
ldb, err := openJSONS("testdata/v0.14.48-ignoredfiles.db.jsons")
if err != nil {
t.Fatal(err)
}
db, _ := newDBInstance(ldb, "<memory>")
db := NewLowlevel(ldb, "<memory>")
UpdateSchema(db)
fs := NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db)
// The contents of the database are like this:
@ -228,11 +141,13 @@ func TestUpdate0to3(t *testing.T) {
if err != nil {
t.Fatal(err)
}
db, _ := newDBInstance(ldb, "<memory>")
db := newInstance(NewLowlevel(ldb, "<memory>"))
updater := schemaUpdater{db}
folder := []byte(update0to3Folder)
db.updateSchema0to1()
updater.updateSchema0to1()
if _, ok := db.getFile(db.keyer.GenerateDeviceFileKey(nil, folder, protocol.LocalDeviceID[:], []byte(slashPrefixed))); ok {
t.Error("File prefixed by '/' was not removed during transition to schema 1")
@ -242,7 +157,7 @@ func TestUpdate0to3(t *testing.T) {
t.Error("Invalid file wasn't added to global list")
}
db.updateSchema1to2()
updater.updateSchema1to2()
found := false
db.withHaveSequence(folder, 0, func(fi FileIntf) bool {
@ -263,7 +178,7 @@ func TestUpdate0to3(t *testing.T) {
t.Error("Local file wasn't added to sequence bucket", err)
}
db.updateSchema2to3()
updater.updateSchema2to3()
need := map[string]protocol.FileInfo{
haveUpdate0to3[remoteDevice0][0].Name: haveUpdate0to3[remoteDevice0][0],
@ -288,22 +203,17 @@ func TestUpdate0to3(t *testing.T) {
}
func TestDowngrade(t *testing.T) {
loc := "testdata/downgrade.db"
db, err := Open(loc)
if err != nil {
t.Fatal(err)
}
defer func() {
db.Close()
os.RemoveAll(loc)
}()
db := OpenMemory()
UpdateSchema(db) // sets the min version etc
miscDB := NewNamespacedKV(db, string(KeyTypeMiscData))
// Bump the database version to something newer than we actually support
miscDB := NewMiscDataNamespace(db)
miscDB.PutInt64("dbVersion", dbVersion+1)
l.Infoln(dbVersion)
db.Close()
db, err = Open(loc)
// Pretend we just opened the DB and attempt to update it again
err := UpdateSchema(db)
if err, ok := err.(databaseDowngradeError); !ok {
t.Fatal("Expected error due to database downgrade, got", err)
} else if err.minSyncthingVersion != dbMinSyncthingVersion {

8
lib/db/leveldb_transactions.go
View File

@ -8,7 +8,6 @@ package db
import (
"bytes"
"sync/atomic"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syndtr/goleveldb/leveldb"
@ -18,10 +17,10 @@ import (
// A readOnlyTransaction represents a database snapshot.
type readOnlyTransaction struct {
*leveldb.Snapshot
db *Instance
db *instance
}
func (db *Instance) newReadOnlyTransaction() readOnlyTransaction {
func (db *instance) newReadOnlyTransaction() readOnlyTransaction {
snap, err := db.GetSnapshot()
if err != nil {
panic(err)
@ -48,7 +47,7 @@ type readWriteTransaction struct {
*leveldb.Batch
}
func (db *Instance) newReadWriteTransaction() readWriteTransaction {
func (db *instance) newReadWriteTransaction() readWriteTransaction {
t := db.newReadOnlyTransaction()
return readWriteTransaction{
readOnlyTransaction: t,
@ -72,7 +71,6 @@ func (t readWriteTransaction) flush() {
if err := t.db.Write(t.Batch, nil); err != nil {
panic(err)
}
atomic.AddInt64(&t.db.committed, int64(t.Batch.Len()))
}
func (t readWriteTransaction) insertFile(fk, folder, device []byte, file protocol.FileInfo) {

122
lib/db/lowlevel.go Normal file
View File

@ -0,0 +1,122 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package db
import (
"os"
"strings"
"sync/atomic"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/errors"
"github.com/syndtr/goleveldb/leveldb/opt"
"github.com/syndtr/goleveldb/leveldb/storage"
)
const (
dbMaxOpenFiles = 100
dbWriteBuffer = 4 << 20
)
// Lowlevel is the lowest level database interface. It has a very simple
// purpose: hold the actual *leveldb.DB database, and the in-memory state
// that belong to that database. In the same way that a single on disk
// database can only be opened once, there should be only one Lowlevel for
// any given *leveldb.DB.
type Lowlevel struct {
committed int64 // atomic, must come first
*leveldb.DB
location string
folderIdx *smallIndex
deviceIdx *smallIndex
}
// Open attempts to open the database at the given location, and runs
// recovery on it if opening fails. Worst case, if recovery is not possible,
// the database is erased and created from scratch.
func Open(location string) (*Lowlevel, error) {
opts := &opt.Options{
OpenFilesCacheCapacity: dbMaxOpenFiles,
WriteBuffer: dbWriteBuffer,
}
db, err := leveldb.OpenFile(location, opts)
if leveldbIsCorrupted(err) {
db, err = leveldb.RecoverFile(location, opts)
}
if leveldbIsCorrupted(err) {
// The database is corrupted, and we've tried to recover it but it
// didn't work. At this point there isn't much to do beyond dropping
// the database and reindexing...
l.Infoln("Database corruption detected, unable to recover. Reinitializing...")
if err := os.RemoveAll(location); err != nil {
return nil, errorSuggestion{err, "failed to delete corrupted database"}
}
db, err = leveldb.OpenFile(location, opts)
}
if err != nil {
return nil, errorSuggestion{err, "is another instance of Syncthing running?"}
}
return NewLowlevel(db, location), nil
}
// OpenMemory returns a new Lowlevel referencing an in-memory database.
func OpenMemory() *Lowlevel {
db, _ := leveldb.Open(storage.NewMemStorage(), nil)
return NewLowlevel(db, "<memory>")
}
// Location returns the filesystem path where the database is stored
func (db *Lowlevel) Location() string {
return db.location
}
// ListFolders returns the list of folders currently in the database
func (db *Lowlevel) ListFolders() []string {
return db.folderIdx.Values()
}
// Committed returns the number of items committed to the database since startup
func (db *Lowlevel) Committed() int64 {
return atomic.LoadInt64(&db.committed)
}
func (db *Lowlevel) Put(key, val []byte, wo *opt.WriteOptions) error {
atomic.AddInt64(&db.committed, 1)
return db.DB.Put(key, val, wo)
}
func (db *Lowlevel) Delete(key []byte, wo *opt.WriteOptions) error {
atomic.AddInt64(&db.committed, 1)
return db.DB.Delete(key, wo)
}
// NewLowlevel wraps the given *leveldb.DB into a *lowlevel
func NewLowlevel(db *leveldb.DB, location string) *Lowlevel {
return &Lowlevel{
DB: db,
location: location,
folderIdx: newSmallIndex(db, []byte{KeyTypeFolderIdx}),
deviceIdx: newSmallIndex(db, []byte{KeyTypeDeviceIdx}),
}
}
// A "better" version of leveldb's errors.IsCorrupted.
func leveldbIsCorrupted(err error) bool {
switch {
case err == nil:
return false
case errors.IsCorrupted(err):
return true
case strings.Contains(err.Error(), "corrupted"):
return true
}
return false
}

25
lib/db/meta.go
View File

@ -20,6 +20,7 @@ type metadataTracker struct {
mut sync.RWMutex
counts CountsSet
indexes map[metaKey]int // device ID + local flags -> index in counts
dirty bool
}
type metaKey struct {
@ -55,18 +56,31 @@ func (m *metadataTracker) Marshal() ([]byte, error) {
// toDB saves the marshalled metadataTracker to the given db, under the key
// corresponding to the given folder
func (m *metadataTracker) toDB(db *Instance, folder []byte) error {
func (m *metadataTracker) toDB(db *instance, folder []byte) error {
key := db.keyer.GenerateFolderMetaKey(nil, folder)
m.mut.RLock()
defer m.mut.RUnlock()
if !m.dirty {
return nil
}
bs, err := m.Marshal()
if err != nil {
return err
}
return db.Put(key, bs, nil)
err = db.Put(key, bs, nil)
if err == nil {
m.dirty = false
}
return err
}
// fromDB initializes the metadataTracker from the marshalled data found in
// the database under the key corresponding to the given folder
func (m *metadataTracker) fromDB(db *Instance, folder []byte) error {
func (m *metadataTracker) fromDB(db *instance, folder []byte) error {
key := db.keyer.GenerateFolderMetaKey(nil, folder)
bs, err := db.Get(key, nil)
if err != nil {
@ -99,6 +113,7 @@ func (m *metadataTracker) addFile(dev protocol.DeviceID, f FileIntf) {
}
m.mut.Lock()
m.dirty = true
if flags := f.FileLocalFlags(); flags == 0 {
// Account regular files in the zero-flags bucket.
@ -141,6 +156,7 @@ func (m *metadataTracker) removeFile(dev protocol.DeviceID, f FileIntf) {
}
m.mut.Lock()
m.dirty = true
if flags := f.FileLocalFlags(); flags == 0 {
// Remove regular files from the zero-flags bucket
@ -194,6 +210,7 @@ func (m *metadataTracker) removeFileLocked(dev protocol.DeviceID, flags uint32,
// resetAll resets all metadata for the given device
func (m *metadataTracker) resetAll(dev protocol.DeviceID) {
m.mut.Lock()
m.dirty = true
for i, c := range m.counts.Counts {
if bytes.Equal(c.DeviceID, dev[:]) {
m.counts.Counts[i] = Counts{
@ -209,6 +226,7 @@ func (m *metadataTracker) resetAll(dev protocol.DeviceID) {
// sequence number
func (m *metadataTracker) resetCounts(dev protocol.DeviceID) {
m.mut.Lock()
m.dirty = true
for i, c := range m.counts.Counts {
if bytes.Equal(c.DeviceID, dev[:]) {
@ -285,6 +303,7 @@ func (m *metadataTracker) Created() time.Time {
func (m *metadataTracker) SetCreated() {
m.mut.Lock()
m.counts.Created = time.Now().UnixNano()
m.dirty = true
m.mut.Unlock()
}

24
lib/db/namespaced.go
View File

@ -17,13 +17,13 @@ import (
// NamespacedKV is a simple key-value store using a specific namespace within
// a leveldb.
type NamespacedKV struct {
db *Instance
db *Lowlevel
prefix []byte
}
// NewNamespacedKV returns a new NamespacedKV that lives in the namespace
// specified by the prefix.
func NewNamespacedKV(db *Instance, prefix string) *NamespacedKV {
func NewNamespacedKV(db *Lowlevel, prefix string) *NamespacedKV {
return &NamespacedKV{
db: db,
prefix: []byte(prefix),
@ -157,3 +157,23 @@ func (n NamespacedKV) Delete(key string) {
keyBs := append(n.prefix, []byte(key)...)
n.db.Delete(keyBs, nil)
}
// Well known namespaces that can be instantiated without knowing the key
// details.
// NewDeviceStatisticsNamespace creates a KV namespace for device statistics
// for the given device.
func NewDeviceStatisticsNamespace(db *Lowlevel, device string) *NamespacedKV {
return NewNamespacedKV(db, string(KeyTypeDeviceStatistic)+device)
}
// NewFolderStatisticsNamespace creates a KV namespace for folder statistics
// for the given folder.
func NewFolderStatisticsNamespace(db *Lowlevel, folder string) *NamespacedKV {
return NewNamespacedKV(db, string(KeyTypeFolderStatistic)+folder)
}
// NewMiscDateNamespace creates a KV namespace for miscellaneous metadata.
func NewMiscDataNamespace(db *Lowlevel) *NamespacedKV {
return NewNamespacedKV(db, string(KeyTypeMiscData))
}

32
lib/db/set.go
View File

@ -21,12 +21,13 @@ import (
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syndtr/goleveldb/leveldb/util"
)
type FileSet struct {
folder string
fs fs.Filesystem
db *Instance
db *instance
blockmap *BlockMap
meta *metadataTracker
@ -66,12 +67,14 @@ func init() {
}
}
func NewFileSet(folder string, fs fs.Filesystem, db *Instance) *FileSet {
func NewFileSet(folder string, fs fs.Filesystem, ll *Lowlevel) *FileSet {
db := newInstance(ll)
var s = FileSet{
folder: folder,
fs: fs,
db: db,
blockmap: NewBlockMap(db, db.folderIdx.ID([]byte(folder))),
blockmap: NewBlockMap(ll, folder),
meta: newMetadataTracker(),
updateMutex: sync.NewMutex(),
}
@ -310,7 +313,7 @@ func (s *FileSet) SetIndexID(device protocol.DeviceID, id protocol.IndexID) {
func (s *FileSet) MtimeFS() *fs.MtimeFS {
prefix := s.db.keyer.GenerateMtimesKey(nil, []byte(s.folder))
kv := NewNamespacedKV(s.db, string(prefix))
kv := NewNamespacedKV(s.db.Lowlevel, string(prefix))
return fs.NewMtimeFS(s.fs, kv)
}
@ -320,15 +323,26 @@ func (s *FileSet) ListDevices() []protocol.DeviceID {
// DropFolder clears out all information related to the given folder from the
// database.
func DropFolder(db *Instance, folder string) {
func DropFolder(ll *Lowlevel, folder string) {
db := newInstance(ll)
db.dropFolder([]byte(folder))
db.dropMtimes([]byte(folder))
db.dropFolderMeta([]byte(folder))
bm := &BlockMap{
db: db,
folder: db.folderIdx.ID([]byte(folder)),
}
bm := NewBlockMap(ll, folder)
bm.Drop()
// Also clean out the folder ID mapping.
db.folderIdx.Delete([]byte(folder))
}
// DropDeltaIndexIDs removes all delta index IDs from the database.
// This will cause a full index transmission on the next connection.
func DropDeltaIndexIDs(db *Lowlevel) {
dbi := db.NewIterator(util.BytesPrefix([]byte{KeyTypeIndexID}), nil)
defer dbi.Release()
for dbi.Next() {
db.Delete(dbi.Key(), nil)
}
}
func normalizeFilenames(fs []protocol.FileInfo) {

50
lib/db/smallindex.go
View File

@ -8,6 +8,7 @@ package db
import (
"encoding/binary"
"sort"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syndtr/goleveldb/leveldb"
@ -46,8 +47,11 @@ func (i *smallIndex) load() {
for it.Next() {
val := string(it.Value())
id := binary.BigEndian.Uint32(it.Key()[len(i.prefix):])
i.id2val[id] = val
i.val2id[val] = id
if val != "" {
// Empty value means the entry has been deleted.
i.id2val[id] = val
i.val2id[val] = id
}
if id >= i.nextID {
i.nextID = id + 1
}
@ -96,3 +100,45 @@ func (i *smallIndex) Val(id uint32) ([]byte, bool) {
return []byte(val), true
}
func (i *smallIndex) Delete(val []byte) {
i.mut.Lock()
defer i.mut.Unlock()
// Check the reverse mapping to get the ID for the value.
if id, ok := i.val2id[string(val)]; ok {
// Generate the corresponding database key.
key := make([]byte, len(i.prefix)+8) // prefix plus uint32 id
copy(key, i.prefix)
binary.BigEndian.PutUint32(key[len(i.prefix):], id)
// Put an empty value into the database. This indicates that the
// entry does not exist any more and prevents the ID from being
// reused in the future.
i.db.Put(key, []byte{}, nil)
// Delete reverse mapping.
delete(i.id2val, id)
}
// Delete forward mapping.
delete(i.val2id, string(val))
}
// Values returns the set of values in the index
func (i *smallIndex) Values() []string {
// In principle this method should return [][]byte because all the other
// methods deal in []byte keys. However, in practice, where it's used
// wants a []string and it's easier to just create that here rather than
// having to convert both here and there...
i.mut.Lock()
vals := make([]string, 0, len(i.val2id))
for val := range i.val2id {
vals = append(vals, val)
}
i.mut.Unlock()
sort.Strings(vals)
return vals
}

52
lib/db/smallindex_test.go Normal file
View File

@ -0,0 +1,52 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package db
import "testing"
func TestSmallIndex(t *testing.T) {
db := OpenMemory()
idx := newSmallIndex(db.DB, []byte{12, 34})
// ID zero should be unallocated
if val, ok := idx.Val(0); ok || val != nil {
t.Fatal("Unexpected return for nonexistent ID 0")
}
// A new key should get ID zero
if id := idx.ID([]byte("hello")); id != 0 {
t.Fatal("Expected 0, not", id)
}
// Looking up ID zero should work
if val, ok := idx.Val(0); !ok || string(val) != "hello" {
t.Fatalf(`Expected true, "hello", not %v, %q`, ok, val)
}
// Delete the key
idx.Delete([]byte("hello"))
// Next ID should be one
if id := idx.ID([]byte("key2")); id != 1 {
t.Fatal("Expected 1, not", id)
}
// Now lets create a new index instance based on what's actually serialized to the database.
idx = newSmallIndex(db.DB, []byte{12, 34})
// Status should be about the same as before.
if val, ok := idx.Val(0); ok || val != nil {
t.Fatal("Unexpected return for deleted ID 0")
}
if id := idx.ID([]byte("key2")); id != 1 {
t.Fatal("Expected 1, not", id)
}
// Setting "hello" again should get us ID 2, not 0 as it was originally.
if id := idx.ID([]byte("hello")); id != 2 {
t.Fatal("Expected 2, not", id)
}
}

4
lib/model/model.go
View File

@ -84,7 +84,7 @@ type Model struct {
*suture.Supervisor
cfg *config.Wrapper
db *db.Instance
db *db.Lowlevel
finder *db.BlockFinder
progressEmitter *ProgressEmitter
id protocol.DeviceID
@ -134,7 +134,7 @@ var (
// NewModel creates and starts a new model. The model starts in read-only mode,
// where it sends index information to connected peers and responds to requests
// for file data without altering the local folder in any way.
func NewModel(cfg *config.Wrapper, id protocol.DeviceID, clientName, clientVersion string, ldb *db.Instance, protectedFiles []string) *Model {
func NewModel(cfg *config.Wrapper, id protocol.DeviceID, clientName, clientVersion string, ldb *db.Lowlevel, protectedFiles []string) *Model {
m := &Model{
Supervisor: suture.New("model", suture.Spec{
Log: func(line string) {

54
lib/model/model_test.go
View File

@ -2618,60 +2618,6 @@ func TestIssue4357(t *testing.T) {
}
}
func TestScanNoDatabaseWrite(t *testing.T) {
// When scanning, nothing should be committed to database unless
// something actually changed.
db := db.OpenMemory()
m := NewModel(defaultCfgWrapper, protocol.LocalDeviceID, "syncthing", "dev", db, nil)
m.AddFolder(defaultFolderConfig)
m.StartFolder("default")
m.ServeBackground()
// Reach in and update the ignore matcher to one that always does
// reloads when asked to, instead of checking file mtimes. This is
// because we will be changing the files on disk often enough that the
// mtimes will be unreliable to determine change status.
m.fmut.Lock()
m.folderIgnores["default"] = ignore.New(defaultFs, ignore.WithCache(true), ignore.WithChangeDetector(newAlwaysChanged()))
m.fmut.Unlock()
m.SetIgnores("default", nil)
defer os.Remove("testdata/.stignore")
// Scan the folder twice. The second scan should be a no-op database wise
m.ScanFolder("default")
c0 := db.Committed()
m.ScanFolder("default")
c1 := db.Committed()
if c1 != c0 {
t.Errorf("scan should not commit data when nothing changed but %d != %d", c1, c0)
}
// Ignore a file we know exists. It'll be updated in the database.
m.SetIgnores("default", []string{"foo"})
m.ScanFolder("default")
c2 := db.Committed()
if c2 <= c1 {
t.Errorf("scan should commit data when something got ignored but %d <= %d", c2, c1)
}
// Scan again. Nothing should happen.
m.ScanFolder("default")
c3 := db.Committed()
if c3 != c2 {
t.Errorf("scan should not commit data when nothing changed (with ignores) but %d != %d", c3, c2)
}
}
func TestIssue2782(t *testing.T) {
// CheckHealth should accept a symlinked folder, when using tilde-expanded path.

5
lib/stats/device.go
View File

@ -21,10 +21,9 @@ type DeviceStatisticsReference struct {
device string
}
func NewDeviceStatisticsReference(ldb *db.Instance, device string) *DeviceStatisticsReference {
prefix := string(db.KeyTypeDeviceStatistic) + device
func NewDeviceStatisticsReference(ldb *db.Lowlevel, device string) *DeviceStatisticsReference {
return &DeviceStatisticsReference{
ns: db.NewNamespacedKV(ldb, prefix),
ns: db.NewDeviceStatisticsNamespace(ldb, device),
device: device,
}
}

5
lib/stats/folder.go
View File

@ -28,10 +28,9 @@ type LastFile struct {
Deleted bool `json:"deleted"`
}
func NewFolderStatisticsReference(ldb *db.Instance, folder string) *FolderStatisticsReference {
prefix := string(db.KeyTypeFolderStatistic) + folder
func NewFolderStatisticsReference(ldb *db.Lowlevel, folder string) *FolderStatisticsReference {
return &FolderStatisticsReference{
ns: db.NewNamespacedKV(ldb, prefix),
ns: db.NewFolderStatisticsNamespace(ldb, folder),
folder: folder,
}
}