Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/db: Move database schema migration into its own file (#4985)

This commit is contained in:
Simon Frei 2018-06-08 12:46:00 +02:00 committed by Jakob Borg
parent 4781e1d86f
commit 30056cd1ae
4 changed files with 156 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cmd/syncthing/main.go
View File

@ -751,9 +751,6 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
miscDB.PutString("prevVersion", Version)
}
// Potential database transitions
ldb.UpdateSchema()
m := model.NewModel(cfg, myID, "syncthing", Version, ldb, protectedFiles)
if t := os.Getenv("STDEADLOCKTIMEOUT"); t != "" {

2
lib/db/leveldb.go
View File

@ -13,8 +13,6 @@ import (
"github.com/syncthing/syncthing/lib/protocol"
)
const dbVersion = 3
const (
KeyTypeDevice = iota
KeyTypeGlobal

144
lib/db/leveldb_dbinstance.go
View File

@ -83,34 +83,10 @@ func newDBInstance(db *leveldb.DB, location string) *Instance {
}
i.folderIdx = newSmallIndex(i, []byte{KeyTypeFolderIdx})
i.deviceIdx = newSmallIndex(i, []byte{KeyTypeDeviceIdx})
i.updateSchema()
return i
}
// UpdateSchema does transitions to the current db version if necessary
func (db *Instance) UpdateSchema() {
miscDB := NewNamespacedKV(db, string(KeyTypeMiscData))
prevVersion, _ := miscDB.Int64("dbVersion")
if prevVersion >= dbVersion {
return
}
l.Infof("Updating database schema version from %v to %v...", prevVersion, dbVersion)
if prevVersion == 0 {
db.updateSchema0to1()
}
if prevVersion <= 1 {
db.updateSchema1to2()
}
if prevVersion <= 2 {
db.updateSchema2to3()
}
l.Infof("Finished updating database schema version from %v to %v", prevVersion, dbVersion)
miscDB.PutInt64("dbVersion", dbVersion)
}
// Committed returns the number of items committed to the database since startup
func (db *Instance) Committed() int64 {
return atomic.LoadInt64(&db.committed)
@ -620,124 +596,6 @@ func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) {
l.Debugf("db check completed for %q", folder)
}
func (db *Instance) updateSchema0to1() {
t := db.newReadWriteTransaction()
defer t.close()
dbi := t.NewIterator(util.BytesPrefix([]byte{KeyTypeDevice}), nil)
defer dbi.Release()
symlinkConv := 0
changedFolders := make(map[string]struct{})
ignAdded := 0
meta := newMetadataTracker() // dummy metadata tracker
var gk []byte
for dbi.Next() {
folder := db.deviceKeyFolder(dbi.Key())
device := db.deviceKeyDevice(dbi.Key())
name := db.deviceKeyName(dbi.Key())
// Remove files with absolute path (see #4799)
if strings.HasPrefix(string(name), "/") {
if _, ok := changedFolders[string(folder)]; !ok {
changedFolders[string(folder)] = struct{}{}
}
gk = db.globalKeyInto(gk, folder, name)
t.removeFromGlobal(gk, folder, device, nil, nil)
t.Delete(dbi.Key())
t.checkFlush()
continue
}
// Change SYMLINK_FILE and SYMLINK_DIRECTORY types to the current SYMLINK
// type (previously SYMLINK_UNKNOWN). It does this for all devices, both
// local and remote, and does not reset delta indexes. It shouldn't really
// matter what the symlink type is, but this cleans it up for a possible
// future when SYMLINK_FILE and SYMLINK_DIRECTORY are no longer understood.
var f protocol.FileInfo
if err := f.Unmarshal(dbi.Value()); err != nil {
// probably can't happen
continue
}
if f.Type == protocol.FileInfoTypeDeprecatedSymlinkDirectory || f.Type == protocol.FileInfoTypeDeprecatedSymlinkFile {
f.Type = protocol.FileInfoTypeSymlink
bs, err := f.Marshal()
if err != nil {
panic("can't happen: " + err.Error())
}
t.Put(dbi.Key(), bs)
t.checkFlush()
symlinkConv++
}
// Add invalid files to global list
if f.Invalid {
gk = db.globalKeyInto(gk, folder, name)
if t.updateGlobal(gk, folder, device, f, meta) {
if _, ok := changedFolders[string(folder)]; !ok {
changedFolders[string(folder)] = struct{}{}
}
ignAdded++
}
}
}
for folder := range changedFolders {
db.dropFolderMeta([]byte(folder))
}
l.Infof("Updated symlink type for %d index entries and added %d invalid files to global list", symlinkConv, ignAdded)
}
// updateSchema1to2 introduces a sequenceKey->deviceKey bucket for local items
// to allow iteration in sequence order (simplifies sending indexes).
func (db *Instance) updateSchema1to2() {
t := db.newReadWriteTransaction()
defer t.close()
var sk []byte
var dk []byte
for _, folderStr := range db.ListFolders() {
folder := []byte(folderStr)
db.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool {
sk = db.sequenceKeyInto(sk, folder, f.SequenceNo())
dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], []byte(f.FileName()))
t.Put(sk, dk)
t.checkFlush()
return true
})
}
}
// updateSchema2to3 introduces a needKey->nil bucket for locally needed files.
func (db *Instance) updateSchema2to3() {
t := db.newReadWriteTransaction()
defer t.close()
var nk []byte
var dk []byte
for _, folderStr := range db.ListFolders() {
folder := []byte(folderStr)
db.withGlobal(folder, nil, true, func(f FileIntf) bool {
name := []byte(f.FileName())
dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], name)
var v protocol.Vector
haveFile, ok := db.getFileTrunc(dk, true)
if ok {
v = haveFile.FileVersion()
}
if !need(f, ok, v) {
return true
}
nk = t.db.needKeyInto(nk, folder, []byte(f.FileName()))
t.Put(nk, nil)
t.checkFlush()
return true
})
}
}
// deviceKey returns a byte slice encoding the following information:
// keyTypeDevice (1 byte)
// folder (4 bytes)

155
lib/db/leveldb_dbinstance_updateschema.go Normal file
View File

@ -0,0 +1,155 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package db
import (
"strings"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syndtr/goleveldb/leveldb/util"
)
const dbVersion = 3
func (db *Instance) updateSchema() {
miscDB := NewNamespacedKV(db, string(KeyTypeMiscData))
prevVersion, _ := miscDB.Int64("dbVersion")
if prevVersion >= dbVersion {
return
}
l.Infof("Updating database schema version from %v to %v...", prevVersion, dbVersion)
if prevVersion < 1 {
db.updateSchema0to1()
}
if prevVersion < 2 {
db.updateSchema1to2()
}
if prevVersion < 3 {
db.updateSchema2to3()
}
miscDB.PutInt64("dbVersion", dbVersion)
}
func (db *Instance) updateSchema0to1() {
t := db.newReadWriteTransaction()
defer t.close()
dbi := t.NewIterator(util.BytesPrefix([]byte{KeyTypeDevice}), nil)
defer dbi.Release()
symlinkConv := 0
changedFolders := make(map[string]struct{})
ignAdded := 0
meta := newMetadataTracker() // dummy metadata tracker
var gk []byte
for dbi.Next() {
folder := db.deviceKeyFolder(dbi.Key())
device := db.deviceKeyDevice(dbi.Key())
name := db.deviceKeyName(dbi.Key())
// Remove files with absolute path (see #4799)
if strings.HasPrefix(string(name), "/") {
if _, ok := changedFolders[string(folder)]; !ok {
changedFolders[string(folder)] = struct{}{}
}
gk = db.globalKeyInto(gk, folder, name)
t.removeFromGlobal(gk, folder, device, nil, nil)
t.Delete(dbi.Key())
t.checkFlush()
continue
}
// Change SYMLINK_FILE and SYMLINK_DIRECTORY types to the current SYMLINK
// type (previously SYMLINK_UNKNOWN). It does this for all devices, both
// local and remote, and does not reset delta indexes. It shouldn't really
// matter what the symlink type is, but this cleans it up for a possible
// future when SYMLINK_FILE and SYMLINK_DIRECTORY are no longer understood.
var f protocol.FileInfo
if err := f.Unmarshal(dbi.Value()); err != nil {
// probably can't happen
continue
}
if f.Type == protocol.FileInfoTypeDeprecatedSymlinkDirectory || f.Type == protocol.FileInfoTypeDeprecatedSymlinkFile {
f.Type = protocol.FileInfoTypeSymlink
bs, err := f.Marshal()
if err != nil {
panic("can't happen: " + err.Error())
}
t.Put(dbi.Key(), bs)
t.checkFlush()
symlinkConv++
}
// Add invalid files to global list
if f.Invalid {
gk = db.globalKeyInto(gk, folder, name)
if t.updateGlobal(gk, folder, device, f, meta) {
if _, ok := changedFolders[string(folder)]; !ok {
changedFolders[string(folder)] = struct{}{}
}
ignAdded++
}
}
}
for folder := range changedFolders {
db.dropFolderMeta([]byte(folder))
}
}
// updateSchema1to2 introduces a sequenceKey->deviceKey bucket for local items
// to allow iteration in sequence order (simplifies sending indexes).
func (db *Instance) updateSchema1to2() {
t := db.newReadWriteTransaction()
defer t.close()
var sk []byte
var dk []byte
for _, folderStr := range db.ListFolders() {
folder := []byte(folderStr)
db.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool {
sk = db.sequenceKeyInto(sk, folder, f.SequenceNo())
dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], []byte(f.FileName()))
t.Put(sk, dk)
t.checkFlush()
return true
})
}
}
// updateSchema2to3 introduces a needKey->nil bucket for locally needed files.
func (db *Instance) updateSchema2to3() {
t := db.newReadWriteTransaction()
defer t.close()
var nk []byte
var dk []byte
for _, folderStr := range db.ListFolders() {
folder := []byte(folderStr)
db.withGlobal(folder, nil, true, func(f FileIntf) bool {
name := []byte(f.FileName())
dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], name)
var v protocol.Vector
haveFile, ok := db.getFileTrunc(dk, true)
if ok {
v = haveFile.FileVersion()
}
if !need(f, ok, v) {
return true
}
nk = t.db.needKeyInto(nk, folder, []byte(f.FileName()))
t.Put(nk, nil)
t.checkFlush()
return true
})
}
}