Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/db: Refactor key handling (ref #5198) (#5199) 

This breaks out the key generation stuff into a separate type. It's
cleaner on its own, and it prepares for future stuff.
This commit is contained in:
Jakob Borg 2018-09-18 10:41:06 +02:00 committed by GitHub
parent 797a999585
commit 6a87aac84f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 465 additions and 371 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

202
lib/db/keyer.go Normal file
View File

@ -0,0 +1,202 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package db
import (
"encoding/binary"
)
const (
keyPrefixLen = 1
keyFolderLen = 4 // indexed
keyDeviceLen = 4 // indexed
keySequenceLen = 8
keyHashLen = 32
maxInt64 int64 = 1<<63 - 1
)
const (
KeyTypeDevice = 0
KeyTypeGlobal = 1
KeyTypeBlock = 2
KeyTypeDeviceStatistic = 3
KeyTypeFolderStatistic = 4
KeyTypeVirtualMtime = 5
KeyTypeFolderIdx = 6
KeyTypeDeviceIdx = 7
KeyTypeIndexID = 8
KeyTypeFolderMeta = 9
KeyTypeMiscData = 10
KeyTypeSequence = 11
KeyTypeNeed = 12
)
type keyer interface {
// device file key stuff
GenerateDeviceFileKey(key, folder, device, name []byte) deviceFileKey
NameFromDeviceFileKey(key []byte) []byte
DeviceFromDeviceFileKey(key []byte) ([]byte, bool)
FolderFromDeviceFileKey(key []byte) ([]byte, bool)
// global version key stuff
GenerateGlobalVersionKey(key, folder, name []byte) globalVersionKey
NameFromGlobalVersionKey(key []byte) []byte
FolderFromGlobalVersionKey(key []byte) ([]byte, bool)
// file need index
GenerateNeedFileKey(key, folder, name []byte) needFileKey
// file sequence index
GenerateSequenceKey(key, folder []byte, seq int64) sequenceKey
SequenceFromSequenceKey(key []byte) int64
// index IDs
GenerateIndexIDKey(key, device, folder []byte) indexIDKey
DeviceFromIndexIDKey(key []byte) ([]byte, bool)
// Mtimes
GenerateMtimesKey(key, folder []byte) mtimesKey
// Folder metadata
GenerateFolderMetaKey(key, folder []byte) folderMetaKey
}
// defaultKeyer implements our key scheme. It needs folder and device
// indexes.
type defaultKeyer struct {
folderIdx *smallIndex
deviceIdx *smallIndex
}
func newDefaultKeyer(folderIdx, deviceIdx *smallIndex) defaultKeyer {
return defaultKeyer{
folderIdx: folderIdx,
deviceIdx: deviceIdx,
}
}
type deviceFileKey []byte
func (k deviceFileKey) WithoutName() []byte {
return k[:keyPrefixLen+keyFolderLen+keyDeviceLen]
}
func (k defaultKeyer) GenerateDeviceFileKey(key, folder, device, name []byte) deviceFileKey {
key = resize(key, keyPrefixLen+keyFolderLen+keyDeviceLen+len(name))
key[0] = KeyTypeDevice
binary.BigEndian.PutUint32(key[keyPrefixLen:], k.folderIdx.ID(folder))
binary.BigEndian.PutUint32(key[keyPrefixLen+keyFolderLen:], k.deviceIdx.ID(device))
copy(key[keyPrefixLen+keyFolderLen+keyDeviceLen:], name)
return key
}
func (k defaultKeyer) NameFromDeviceFileKey(key []byte) []byte {
return key[keyPrefixLen+keyFolderLen+keyDeviceLen:]
}
func (k defaultKeyer) DeviceFromDeviceFileKey(key []byte) ([]byte, bool) {
return k.deviceIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen+keyFolderLen:]))
}
func (k defaultKeyer) FolderFromDeviceFileKey(key []byte) ([]byte, bool) {
return k.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
}
type globalVersionKey []byte
func (k globalVersionKey) WithoutName() []byte {
return k[:keyPrefixLen+keyFolderLen]
}
func (k defaultKeyer) GenerateGlobalVersionKey(key, folder, name []byte) globalVersionKey {
key = resize(key, keyPrefixLen+keyFolderLen+len(name))
key[0] = KeyTypeGlobal
binary.BigEndian.PutUint32(key[keyPrefixLen:], k.folderIdx.ID(folder))
copy(key[keyPrefixLen+keyFolderLen:], name)
return key
}
func (k defaultKeyer) NameFromGlobalVersionKey(key []byte) []byte {
return key[keyPrefixLen+keyFolderLen:]
}
func (k defaultKeyer) FolderFromGlobalVersionKey(key []byte) ([]byte, bool) {
return k.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
}
type needFileKey []byte
func (k needFileKey) WithoutName() []byte {
return k[:keyPrefixLen+keyFolderLen]
}
func (k defaultKeyer) GenerateNeedFileKey(key, folder, name []byte) needFileKey {
key = resize(key, keyPrefixLen+keyFolderLen+len(name))
key[0] = KeyTypeNeed
binary.BigEndian.PutUint32(key[keyPrefixLen:], k.folderIdx.ID(folder))
copy(key[keyPrefixLen+keyFolderLen:], name)
return key
}
type sequenceKey []byte
func (k sequenceKey) WithoutSequence() []byte {
return k[:keyPrefixLen+keyFolderLen]
}
func (k defaultKeyer) GenerateSequenceKey(key, folder []byte, seq int64) sequenceKey {
key = resize(key, keyPrefixLen+keyFolderLen+keySequenceLen)
key[0] = KeyTypeSequence
binary.BigEndian.PutUint32(key[keyPrefixLen:], k.folderIdx.ID(folder))
binary.BigEndian.PutUint64(key[keyPrefixLen+keyFolderLen:], uint64(seq))
return key
}
func (k defaultKeyer) SequenceFromSequenceKey(key []byte) int64 {
return int64(binary.BigEndian.Uint64(key[keyPrefixLen+keyFolderLen:]))
}
type indexIDKey []byte
func (k defaultKeyer) GenerateIndexIDKey(key, device, folder []byte) indexIDKey {
key = resize(key, keyPrefixLen+keyDeviceLen+keyFolderLen)
key[0] = KeyTypeIndexID
binary.BigEndian.PutUint32(key[keyPrefixLen:], k.deviceIdx.ID(device))
binary.BigEndian.PutUint32(key[keyPrefixLen+keyDeviceLen:], k.folderIdx.ID(folder))
return key
}
func (k defaultKeyer) DeviceFromIndexIDKey(key []byte) ([]byte, bool) {
return k.deviceIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
}
type mtimesKey []byte
func (k defaultKeyer) GenerateMtimesKey(key, folder []byte) mtimesKey {
key = resize(key, keyPrefixLen+keyFolderLen)
key[0] = KeyTypeVirtualMtime
binary.BigEndian.PutUint32(key[keyPrefixLen:], k.folderIdx.ID(folder))
return key
}
type folderMetaKey []byte
func (k defaultKeyer) GenerateFolderMetaKey(key, folder []byte) folderMetaKey {
key = resize(key, keyPrefixLen+keyFolderLen)
key[0] = KeyTypeFolderMeta
binary.BigEndian.PutUint32(key[keyPrefixLen:], k.folderIdx.ID(folder))
return key
}
// resize returns a byte slice of the specified size, reusing bs if possible
func resize(bs []byte, size int) []byte {
if cap(bs) < size {
return make([]byte, size)
}
return bs[:size]
}

80
lib/db/keyer_test.go Normal file
View File

@ -0,0 +1,80 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package db
import (
"bytes"
"testing"
)
func TestDeviceKey(t *testing.T) {
fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
dev := []byte("device67890123456789012345678901")
name := []byte("name")
db := OpenMemory()
key := db.keyer.GenerateDeviceFileKey(nil, fld, dev, name)
fld2, ok := db.keyer.FolderFromDeviceFileKey(key)
if !ok {
t.Fatal("unexpectedly not found")
}
if !bytes.Equal(fld2, fld) {
t.Errorf("wrong folder %q != %q", fld2, fld)
}
dev2, ok := db.keyer.DeviceFromDeviceFileKey(key)
if !ok {
t.Fatal("unexpectedly not found")
}
if !bytes.Equal(dev2, dev) {
t.Errorf("wrong device %q != %q", dev2, dev)
}
name2 := db.keyer.NameFromDeviceFileKey(key)
if !bytes.Equal(name2, name) {
t.Errorf("wrong name %q != %q", name2, name)
}
}
func TestGlobalKey(t *testing.T) {
fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
name := []byte("name")
db := OpenMemory()
key := db.keyer.GenerateGlobalVersionKey(nil, fld, name)
fld2, ok := db.keyer.FolderFromGlobalVersionKey(key)
if !ok {
t.Error("should have been found")
}
if !bytes.Equal(fld2, fld) {
t.Errorf("wrong folder %q != %q", fld2, fld)
}
name2 := db.keyer.NameFromGlobalVersionKey(key)
if !bytes.Equal(name2, name) {
t.Errorf("wrong name %q != %q", name2, name)
}
_, ok = db.keyer.FolderFromGlobalVersionKey([]byte{1, 2, 3, 4, 5})
if ok {
t.Error("should not have been found")
}
}
func TestSequenceKey(t *testing.T) {
fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
db := OpenMemory()
const seq = 1234567890
key := db.keyer.GenerateSequenceKey(nil, fld, seq)
outSeq := db.keyer.SequenceFromSequenceKey(key)
if outSeq != seq {
t.Errorf("sequence number mangled, %d != %d", outSeq, seq)
}
}

18
lib/db/leveldb.go
View File

@ -13,22 +13,6 @@ import (
"github.com/syncthing/syncthing/lib/protocol"
)
const (
KeyTypeDevice = iota
KeyTypeGlobal
KeyTypeBlock
KeyTypeDeviceStatistic
KeyTypeFolderStatistic
KeyTypeVirtualMtime
KeyTypeFolderIdx
KeyTypeDeviceIdx
KeyTypeIndexID
KeyTypeFolderMeta
KeyTypeMiscData
KeyTypeSequence
KeyTypeNeed
)
func (vl VersionList) String() string {
var b bytes.Buffer
var id protocol.DeviceID
@ -86,7 +70,7 @@ func (vl VersionList) update(folder, device []byte, file protocol.FileInfo, db *
// to determine the winner.)
//
// A surprise missing file entry here is counted as a win for us.
if of, ok := db.getFile(db.deviceKey(folder, v.Device, []byte(file.Name))); !ok || file.WinsConflict(of) {
if of, ok := db.getFile(db.keyer.GenerateDeviceFileKey(nil, folder, v.Device, []byte(file.Name))); !ok || file.WinsConflict(of) {
vl = vl.insertAt(i, nv)
return vl, removedFV, removedAt, i
}

333
lib/db/leveldb_dbinstance.go
View File

@ -16,7 +16,6 @@ import (
"sync/atomic"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/errors"
"github.com/syndtr/goleveldb/leveldb/iterator"
@ -33,18 +32,9 @@ type Instance struct {
location string
folderIdx *smallIndex
deviceIdx *smallIndex
keyer keyer
}
const (
keyPrefixLen = 1
keyFolderLen = 4 // indexed
keyDeviceLen = 4 // indexed
keySequenceLen = 8
keyHashLen = 32
maxInt64 int64 = 1<<63 - 1
)
func Open(file string) (*Instance, error) {
opts := &opt.Options{
OpenFilesCacheCapacity: 100,
@ -80,11 +70,12 @@ func OpenMemory() *Instance {
func newDBInstance(db *leveldb.DB, location string) (*Instance, error) {
i := &Instance{
DB: db,
location: location,
DB: db,
location: location,
folderIdx: newSmallIndex(db, []byte{KeyTypeFolderIdx}),
deviceIdx: newSmallIndex(db, []byte{KeyTypeDeviceIdx}),
}
i.folderIdx = newSmallIndex(i, []byte{KeyTypeFolderIdx})
i.deviceIdx = newSmallIndex(i, []byte{KeyTypeDeviceIdx})
i.keyer = newDefaultKeyer(i.folderIdx, i.deviceIdx)
err := i.updateSchema()
return i, err
}
@ -107,7 +98,7 @@ func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, m
var gk []byte
for _, f := range fs {
name := []byte(f.Name)
fk = db.deviceKeyInto(fk, folder, device, name)
fk = db.keyer.GenerateDeviceFileKey(fk, folder, device, name)
// Get and unmarshal the file entry. If it doesn't exist or can't be
// unmarshalled we'll add it as a new entry.
@ -131,7 +122,7 @@ func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, m
t.insertFile(fk, folder, device, f)
gk = db.globalKeyInto(gk, folder, name)
gk = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
t.updateGlobal(gk, folder, device, f, meta)
// Write out and reuse the batch every few records, to avoid the batch
@ -147,8 +138,8 @@ func (db *Instance) addSequences(folder []byte, fs []protocol.FileInfo) {
var sk []byte
var dk []byte
for _, f := range fs {
sk = db.sequenceKeyInto(sk, folder, f.Sequence)
dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], []byte(f.Name))
sk = db.keyer.GenerateSequenceKey(sk, folder, f.Sequence)
dk = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], []byte(f.Name))
t.Put(sk, dk)
l.Debugf("adding sequence; folder=%q sequence=%v %v", folder, f.Sequence, f.Name)
t.checkFlush()
@ -161,7 +152,7 @@ func (db *Instance) removeSequences(folder []byte, fs []protocol.FileInfo) {
var sk []byte
for _, f := range fs {
t.Delete(db.sequenceKeyInto(sk, folder, f.Sequence))
t.Delete(db.keyer.GenerateSequenceKey(sk, folder, f.Sequence))
l.Debugf("removing sequence; folder=%q sequence=%v %v", folder, f.Sequence, f.Name)
t.checkFlush()
}
@ -176,7 +167,7 @@ func (db *Instance) withHave(folder, device, prefix []byte, truncate bool, fn It
prefix = append(prefix, '/')
}
if f, ok := db.getFileTrunc(db.deviceKey(folder, device, unslashedPrefix), true); ok && !fn(f) {
if f, ok := db.getFileTrunc(db.keyer.GenerateDeviceFileKey(nil, folder, device, unslashedPrefix), true); ok && !fn(f) {
return
}
}
@ -184,11 +175,11 @@ func (db *Instance) withHave(folder, device, prefix []byte, truncate bool, fn It
t := db.newReadOnlyTransaction()
defer t.close()
dbi := t.NewIterator(util.BytesPrefix(db.deviceKey(folder, device, prefix)[:keyPrefixLen+keyFolderLen+keyDeviceLen+len(prefix)]), nil)
dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateDeviceFileKey(nil, folder, device, prefix)), nil)
defer dbi.Release()
for dbi.Next() {
name := db.deviceKeyName(dbi.Key())
name := db.keyer.NameFromDeviceFileKey(dbi.Key())
if len(prefix) > 0 && !bytes.HasPrefix(name, prefix) {
return
}
@ -212,13 +203,13 @@ func (db *Instance) withHaveSequence(folder []byte, startSeq int64, fn Iterator)
t := db.newReadOnlyTransaction()
defer t.close()
dbi := t.NewIterator(&util.Range{Start: db.sequenceKey(folder, startSeq), Limit: db.sequenceKey(folder, maxInt64)}, nil)
dbi := t.NewIterator(&util.Range{Start: db.keyer.GenerateSequenceKey(nil, folder, startSeq), Limit: db.keyer.GenerateSequenceKey(nil, folder, maxInt64)}, nil)
defer dbi.Release()
for dbi.Next() {
f, ok := db.getFile(dbi.Value())
if !ok {
l.Debugln("missing file for sequence number", db.sequenceKeySequence(dbi.Key()))
l.Debugln("missing file for sequence number", db.keyer.SequenceFromSequenceKey(dbi.Key()))
continue
}
@ -239,13 +230,19 @@ func (db *Instance) withAllFolderTruncated(folder []byte, fn func(device []byte,
t := db.newReadWriteTransaction()
defer t.close()
dbi := t.NewIterator(util.BytesPrefix(db.deviceKey(folder, nil, nil)[:keyPrefixLen+keyFolderLen]), nil)
dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateDeviceFileKey(nil, folder, nil, nil).WithoutName()), nil)
defer dbi.Release()
var gk []byte
for dbi.Next() {
device := db.deviceKeyDevice(dbi.Key())
device, ok := db.keyer.DeviceFromDeviceFileKey(dbi.Key())
if !ok {
// Not having the device in the index is bad. Clear it.
t.Delete(dbi.Key())
t.checkFlush()
continue
}
var f FileInfoTruncated
// The iterator function may keep a reference to the unmarshalled
// struct, which in turn references the buffer it was unmarshalled
@ -261,7 +258,7 @@ func (db *Instance) withAllFolderTruncated(folder []byte, fn func(device []byte,
case "", ".", "..", "/": // A few obviously invalid filenames
l.Infof("Dropping invalid filename %q from database", f.Name)
name := []byte(f.Name)
gk = db.globalKeyInto(gk, folder, name)
gk = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
t.removeFromGlobal(gk, folder, device, name, nil)
t.Delete(dbi.Key())
t.checkFlush()
@ -308,7 +305,7 @@ func (db *Instance) getGlobal(folder, file []byte, truncate bool) (FileIntf, boo
}
func (db *Instance) getGlobalInto(t readOnlyTransaction, gk, dk, folder, file []byte, truncate bool) ([]byte, []byte, FileIntf, bool) {
gk = db.globalKeyInto(gk, folder, file)
gk = db.keyer.GenerateGlobalVersionKey(gk, folder, file)
bs, err := t.Get(gk, nil)
if err != nil {
@ -320,7 +317,7 @@ func (db *Instance) getGlobalInto(t readOnlyTransaction, gk, dk, folder, file []
return gk, dk, nil, false
}
dk = db.deviceKeyInto(dk, folder, vl.Versions[0].Device, file)
dk = db.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[0].Device, file)
if fi, ok := db.getFileTrunc(dk, truncate); ok {
return gk, dk, fi, true
}
@ -345,12 +342,12 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator
t := db.newReadOnlyTransaction()
defer t.close()
dbi := t.NewIterator(util.BytesPrefix(db.globalKey(folder, prefix)), nil)
dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateGlobalVersionKey(nil, folder, prefix)), nil)
defer dbi.Release()
var fk []byte
for dbi.Next() {
name := db.globalKeyName(dbi.Key())
name := db.keyer.NameFromGlobalVersionKey(dbi.Key())
if len(prefix) > 0 && !bytes.HasPrefix(name, prefix) {
return
}
@ -360,7 +357,7 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator
continue
}
fk = db.deviceKeyInto(fk, folder, vl.Versions[0].Device, name)
fk = db.keyer.GenerateDeviceFileKey(fk, folder, vl.Versions[0].Device, name)
f, ok := db.getFileTrunc(fk, truncate)
if !ok {
@ -374,7 +371,7 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator
}
func (db *Instance) availability(folder, file []byte) []protocol.DeviceID {
k := db.globalKey(folder, file)
k := db.keyer.GenerateGlobalVersionKey(nil, folder, file)
bs, err := db.Get(k, nil)
if err == leveldb.ErrNotFound {
return nil
@ -413,7 +410,7 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator)
t := db.newReadOnlyTransaction()
defer t.close()
dbi := t.NewIterator(util.BytesPrefix(db.globalKey(folder, nil)[:keyPrefixLen+keyFolderLen]), nil)
dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateGlobalVersionKey(nil, folder, nil).WithoutName()), nil)
defer dbi.Release()
var fk []byte
@ -431,7 +428,7 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator)
continue
}
name := db.globalKeyName(dbi.Key())
name := db.keyer.NameFromGlobalVersionKey(dbi.Key())
needVersion := vl.Versions[0].Version
needDevice := protocol.DeviceIDFromBytes(vl.Versions[0].Device)
@ -446,7 +443,7 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator)
continue
}
fk = db.deviceKeyInto(fk, folder, vl.Versions[i].Device, name)
fk = db.keyer.GenerateDeviceFileKey(fk, folder, vl.Versions[i].Device, name)
bs, err := t.Get(fk, nil)
if err != nil {
l.Debugln("surprise error:", err)
@ -480,7 +477,7 @@ func (db *Instance) withNeedLocal(folder []byte, truncate bool, fn Iterator) {
t := db.newReadOnlyTransaction()
defer t.close()
dbi := t.NewIterator(util.BytesPrefix(db.needKey(folder, nil)[:keyPrefixLen+keyFolderLen]), nil)
dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateNeedFileKey(nil, folder, nil).WithoutName()), nil)
defer dbi.Release()
var dk []byte
@ -488,7 +485,7 @@ func (db *Instance) withNeedLocal(folder []byte, truncate bool, fn Iterator) {
var f FileIntf
var ok bool
for dbi.Next() {
gk, dk, f, ok = db.getGlobalInto(t, gk, dk, folder, db.globalKeyName(dbi.Key()), truncate)
gk, dk, f, ok = db.getGlobalInto(t, gk, dk, folder, db.keyer.NameFromGlobalVersionKey(dbi.Key()), truncate)
if !ok {
continue
}
@ -507,7 +504,7 @@ func (db *Instance) ListFolders() []string {
folderExists := make(map[string]bool)
for dbi.Next() {
folder, ok := db.globalKeyFolder(dbi.Key())
folder, ok := db.keyer.FolderFromGlobalVersionKey(dbi.Key())
if ok && !folderExists[string(folder)] {
folderExists[string(folder)] = true
}
@ -528,13 +525,13 @@ func (db *Instance) dropFolder(folder []byte) {
for _, key := range [][]byte{
// Remove all items related to the given folder from the device->file bucket
db.deviceKey(folder, nil, nil)[:keyPrefixLen+keyFolderLen],
db.keyer.GenerateDeviceFileKey(nil, folder, nil, nil).WithoutName(),
// Remove all sequences related to the folder
db.sequenceKey([]byte(folder), 0)[:keyPrefixLen+keyFolderLen],
db.keyer.GenerateSequenceKey(nil, []byte(folder), 0).WithoutSequence(),
// Remove all items related to the given folder from the global bucket
db.globalKey(folder, nil)[:keyPrefixLen+keyFolderLen],
db.keyer.GenerateGlobalVersionKey(nil, folder, nil).WithoutName(),
// Remove all needs related to the folder
db.needKey(folder, nil)[:keyPrefixLen+keyFolderLen],
db.keyer.GenerateNeedFileKey(nil, folder, nil).WithoutName(),
} {
t.deleteKeyPrefix(key)
}
@ -544,15 +541,15 @@ func (db *Instance) dropDeviceFolder(device, folder []byte, meta *metadataTracke
t := db.newReadWriteTransaction()
defer t.close()
dbi := t.NewIterator(util.BytesPrefix(db.deviceKey(folder, device, nil)), nil)
dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateDeviceFileKey(nil, folder, device, nil)), nil)
defer dbi.Release()
var gk []byte
for dbi.Next() {
key := dbi.Key()
name := db.deviceKeyName(key)
gk = db.globalKeyInto(gk, folder, name)
name := db.keyer.NameFromDeviceFileKey(key)
gk = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
t.removeFromGlobal(gk, folder, device, name, meta)
t.Delete(key)
t.checkFlush()
@ -563,7 +560,7 @@ func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) {
t := db.newReadWriteTransaction()
defer t.close()
dbi := t.NewIterator(util.BytesPrefix(db.globalKey(folder, nil)[:keyPrefixLen+keyFolderLen]), nil)
dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateGlobalVersionKey(nil, folder, nil).WithoutName()), nil)
defer dbi.Release()
var fk []byte
@ -578,10 +575,10 @@ func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) {
// there are global entries pointing to no longer existing files. Here
// we find those and clear them out.
name := db.globalKeyName(dbi.Key())
name := db.keyer.NameFromGlobalVersionKey(dbi.Key())
var newVL VersionList
for i, version := range vl.Versions {
fk = db.deviceKeyInto(fk, folder, version.Device, name)
fk = db.keyer.GenerateDeviceFileKey(fk, folder, version.Device, name)
_, err := t.Get(fk, nil)
if err == leveldb.ErrNotFound {
continue
@ -607,110 +604,8 @@ func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) {
l.Debugf("db check completed for %q", folder)
}
// deviceKey returns a byte slice encoding the following information:
// keyTypeDevice (1 byte)
// folder (4 bytes)
// device (4 bytes)
// name (variable size)
func (db *Instance) deviceKey(folder, device, file []byte) []byte {
return db.deviceKeyInto(nil, folder, device, file)
}
func (db *Instance) deviceKeyInto(k, folder, device, file []byte) []byte {
reqLen := keyPrefixLen + keyFolderLen + keyDeviceLen + len(file)
k = resize(k, reqLen)
k[0] = KeyTypeDevice
binary.BigEndian.PutUint32(k[keyPrefixLen:], db.folderIdx.ID(folder))
binary.BigEndian.PutUint32(k[keyPrefixLen+keyFolderLen:], db.deviceIdx.ID(device))
copy(k[keyPrefixLen+keyFolderLen+keyDeviceLen:], file)
return k
}
// deviceKeyName returns the device ID from the key
func (db *Instance) deviceKeyName(key []byte) []byte {
return key[keyPrefixLen+keyFolderLen+keyDeviceLen:]
}
// deviceKeyFolder returns the folder name from the key
func (db *Instance) deviceKeyFolder(key []byte) []byte {
folder, ok := db.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
if !ok {
panic("bug: lookup of nonexistent folder ID")
}
return folder
}
// deviceKeyDevice returns the device ID from the key
func (db *Instance) deviceKeyDevice(key []byte) []byte {
device, ok := db.deviceIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen+keyFolderLen:]))
if !ok {
panic("bug: lookup of nonexistent device ID")
}
return device
}
// globalKey returns a byte slice encoding the following information:
// keyTypeGlobal (1 byte)
// folder (4 bytes)
// name (variable size)
func (db *Instance) globalKey(folder, file []byte) []byte {
return db.globalKeyInto(nil, folder, file)
}
func (db *Instance) globalKeyInto(gk, folder, file []byte) []byte {
reqLen := keyPrefixLen + keyFolderLen + len(file)
gk = resize(gk, reqLen)
gk[0] = KeyTypeGlobal
binary.BigEndian.PutUint32(gk[keyPrefixLen:], db.folderIdx.ID(folder))
copy(gk[keyPrefixLen+keyFolderLen:], file)
return gk
}
// globalKeyName returns the filename from the key
func (db *Instance) globalKeyName(key []byte) []byte {
return key[keyPrefixLen+keyFolderLen:]
}
// globalKeyFolder returns the folder name from the key
func (db *Instance) globalKeyFolder(key []byte) ([]byte, bool) {
return db.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
}
// needKey is a globalKey with a different prefix
func (db *Instance) needKey(folder, file []byte) []byte {
return db.needKeyInto(nil, folder, file)
}
func (db *Instance) needKeyInto(k, folder, file []byte) []byte {
k = db.globalKeyInto(k, folder, file)
k[0] = KeyTypeNeed
return k
}
// sequenceKey returns a byte slice encoding the following information:
// KeyTypeSequence (1 byte)
// folder (4 bytes)
// sequence number (8 bytes)
func (db *Instance) sequenceKey(folder []byte, seq int64) []byte {
return db.sequenceKeyInto(nil, folder, seq)
}
func (db *Instance) sequenceKeyInto(k []byte, folder []byte, seq int64) []byte {
reqLen := keyPrefixLen + keyFolderLen + keySequenceLen
k = resize(k, reqLen)
k[0] = KeyTypeSequence
binary.BigEndian.PutUint32(k[keyPrefixLen:], db.folderIdx.ID(folder))
binary.BigEndian.PutUint64(k[keyPrefixLen+keyFolderLen:], uint64(seq))
return k
}
// sequenceKeySequence returns the sequence number from the key
func (db *Instance) sequenceKeySequence(key []byte) int64 {
return int64(binary.BigEndian.Uint64(key[keyPrefixLen+keyFolderLen:]))
}
func (db *Instance) getIndexID(device, folder []byte) protocol.IndexID {
key := db.indexIDKey(device, folder)
key := db.keyer.GenerateIndexIDKey(nil, device, folder)
cur, err := db.Get(key, nil)
if err != nil {
return 0
@ -725,44 +620,13 @@ func (db *Instance) getIndexID(device, folder []byte) protocol.IndexID {
}
func (db *Instance) setIndexID(device, folder []byte, id protocol.IndexID) {
key := db.indexIDKey(device, folder)
key := db.keyer.GenerateIndexIDKey(nil, device, folder)
bs, _ := id.Marshal() // marshalling can't fail
if err := db.Put(key, bs, nil); err != nil {
panic("storing index ID: " + err.Error())
}
}
func (db *Instance) indexIDKey(device, folder []byte) []byte {
k := make([]byte, keyPrefixLen+keyDeviceLen+keyFolderLen)
k[0] = KeyTypeIndexID
binary.BigEndian.PutUint32(k[keyPrefixLen:], db.deviceIdx.ID(device))
binary.BigEndian.PutUint32(k[keyPrefixLen+keyDeviceLen:], db.folderIdx.ID(folder))
return k
}
func (db *Instance) indexIDDevice(key []byte) []byte {
device, ok := db.deviceIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
if !ok {
// uuh ...
return nil
}
return device
}
func (db *Instance) mtimesKey(folder []byte) []byte {
prefix := make([]byte, 5) // key type + 4 bytes folder idx number
prefix[0] = KeyTypeVirtualMtime
binary.BigEndian.PutUint32(prefix[1:], db.folderIdx.ID(folder))
return prefix
}
func (db *Instance) folderMetaKey(folder []byte) []byte {
prefix := make([]byte, 5) // key type + 4 bytes folder idx number
prefix[0] = KeyTypeFolderMeta
binary.BigEndian.PutUint32(prefix[1:], db.folderIdx.ID(folder))
return prefix
}
// DropLocalDeltaIndexIDs removes all index IDs for the local device ID from
// the database. This will cause a full index transmission on the next
// connection.
@ -785,7 +649,7 @@ func (db *Instance) dropDeltaIndexIDs(local bool) {
defer dbi.Release()
for dbi.Next() {
device := db.indexIDDevice(dbi.Key())
device, _ := db.keyer.DeviceFromIndexIDKey(dbi.Key())
if bytes.Equal(device, protocol.LocalDeviceID[:]) == local {
t.Delete(dbi.Key())
}
@ -793,11 +657,11 @@ func (db *Instance) dropDeltaIndexIDs(local bool) {
}
func (db *Instance) dropMtimes(folder []byte) {
db.dropPrefix(db.mtimesKey(folder))
db.dropPrefix(db.keyer.GenerateMtimesKey(nil, folder))
}
func (db *Instance) dropFolderMeta(folder []byte) {
db.dropPrefix(db.folderMetaKey(folder))
db.dropPrefix(db.keyer.GenerateFolderMetaKey(nil, folder))
}
func (db *Instance) dropPrefix(prefix []byte) {
@ -853,99 +717,6 @@ func leveldbIsCorrupted(err error) bool {
return false
}
// A smallIndex is an in memory bidirectional []byte to uint32 map. It gives
// fast lookups in both directions and persists to the database. Don't use for
// storing more items than fit comfortably in RAM.
type smallIndex struct {
db *Instance
prefix []byte
id2val map[uint32]string
val2id map[string]uint32
nextID uint32
mut sync.Mutex
}
func newSmallIndex(db *Instance, prefix []byte) *smallIndex {
idx := &smallIndex{
db: db,
prefix: prefix,
id2val: make(map[uint32]string),
val2id: make(map[string]uint32),
mut: sync.NewMutex(),
}
idx.load()
return idx
}
// load iterates over the prefix space in the database and populates the in
// memory maps.
func (i *smallIndex) load() {
tr := i.db.newReadOnlyTransaction()
it := tr.NewIterator(util.BytesPrefix(i.prefix), nil)
for it.Next() {
val := string(it.Value())
id := binary.BigEndian.Uint32(it.Key()[len(i.prefix):])
i.id2val[id] = val
i.val2id[val] = id
if id >= i.nextID {
i.nextID = id + 1
}
}
it.Release()
tr.close()
}
// ID returns the index number for the given byte slice, allocating a new one
// and persisting this to the database if necessary.
func (i *smallIndex) ID(val []byte) uint32 {
i.mut.Lock()
// intentionally avoiding defer here as we want this call to be as fast as
// possible in the general case (folder ID already exists). The map lookup
// with the conversion of []byte to string is compiler optimized to not
// copy the []byte, which is why we don't assign it to a temp variable
// here.
if id, ok := i.val2id[string(val)]; ok {
i.mut.Unlock()
return id
}
id := i.nextID
i.nextID++
valStr := string(val)
i.val2id[valStr] = id
i.id2val[id] = valStr
key := make([]byte, len(i.prefix)+8) // prefix plus uint32 id
copy(key, i.prefix)
binary.BigEndian.PutUint32(key[len(i.prefix):], id)
i.db.Put(key, val, nil)
i.mut.Unlock()
return id
}
// Val returns the value for the given index number, or (nil, false) if there
// is no such index number.
func (i *smallIndex) Val(id uint32) ([]byte, bool) {
i.mut.Lock()
val, ok := i.id2val[id]
i.mut.Unlock()
if !ok {
return nil, false
}
return []byte(val), true
}
// resize returns a byte array of length reqLen, reusing k if possible
func resize(k []byte, reqLen int) []byte {
if cap(k) < reqLen {
return make([]byte, reqLen)
}
return k[:reqLen]
}
type errorSuggestion struct {
inner error
suggestion string

34
lib/db/leveldb_dbinstance_updateschema.go
View File

@ -91,16 +91,28 @@ func (db *Instance) updateSchema0to1() {
var gk []byte
for dbi.Next() {
folder := db.deviceKeyFolder(dbi.Key())
device := db.deviceKeyDevice(dbi.Key())
name := db.deviceKeyName(dbi.Key())
folder, ok := db.keyer.FolderFromDeviceFileKey(dbi.Key())
if !ok {
// not having the folder in the index is bad; delete and continue
t.Delete(dbi.Key())
t.checkFlush()
continue
}
device, ok := db.keyer.DeviceFromDeviceFileKey(dbi.Key())
if !ok {
// not having the device in the index is bad; delete and continue
t.Delete(dbi.Key())
t.checkFlush()
continue
}
name := db.keyer.NameFromDeviceFileKey(dbi.Key())
// Remove files with absolute path (see #4799)
if strings.HasPrefix(string(name), "/") {
if _, ok := changedFolders[string(folder)]; !ok {
changedFolders[string(folder)] = struct{}{}
}
gk = db.globalKeyInto(gk, folder, name)
gk = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
t.removeFromGlobal(gk, folder, device, nil, nil)
t.Delete(dbi.Key())
t.checkFlush()
@ -130,7 +142,7 @@ func (db *Instance) updateSchema0to1() {
// Add invalid files to global list
if f.IsInvalid() {
gk = db.globalKeyInto(gk, folder, name)
gk = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
if t.updateGlobal(gk, folder, device, f, meta) {
if _, ok := changedFolders[string(folder)]; !ok {
changedFolders[string(folder)] = struct{}{}
@ -156,8 +168,8 @@ func (db *Instance) updateSchema1to2() {
for _, folderStr := range db.ListFolders() {
folder := []byte(folderStr)
db.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool {
sk = db.sequenceKeyInto(sk, folder, f.SequenceNo())
dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], []byte(f.FileName()))
sk = db.keyer.GenerateSequenceKey(sk, folder, f.SequenceNo())
dk = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], []byte(f.FileName()))
t.Put(sk, dk)
t.checkFlush()
return true
@ -176,7 +188,7 @@ func (db *Instance) updateSchema2to3() {
folder := []byte(folderStr)
db.withGlobal(folder, nil, true, func(f FileIntf) bool {
name := []byte(f.FileName())
dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], name)
dk = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], name)
var v protocol.Vector
haveFile, ok := db.getFileTrunc(dk, true)
if ok {
@ -185,7 +197,7 @@ func (db *Instance) updateSchema2to3() {
if !need(f, ok, v) {
return true
}
nk = t.db.needKeyInto(nk, folder, []byte(f.FileName()))
nk = t.db.keyer.GenerateNeedFileKey(nk, folder, []byte(f.FileName()))
t.Put(nk, nil)
t.checkFlush()
return true
@ -201,7 +213,7 @@ func (db *Instance) updateSchemaTo5() {
t := db.newReadWriteTransaction()
var nk []byte
for _, folderStr := range db.ListFolders() {
nk = db.needKeyInto(nk, []byte(folderStr), nil)
nk = db.keyer.GenerateNeedFileKey(nk, []byte(folderStr), nil)
t.deleteKeyPrefix(nk[:keyPrefixLen+keyFolderLen])
}
t.close()
@ -230,7 +242,7 @@ func (db *Instance) updateSchema5to6() {
fi.LocalFlags = protocol.FlagLocalIgnored
bs, _ := fi.Marshal()
dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], []byte(fi.Name))
dk = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], []byte(fi.Name))
t.Put(dk, bs)
t.checkFlush()

57
lib/db/leveldb_test.go
View File

@ -7,7 +7,6 @@
package db
import (
"bytes"
"os"
"testing"
@ -15,58 +14,6 @@ import (
"github.com/syncthing/syncthing/lib/protocol"
)
func TestDeviceKey(t *testing.T) {
fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
dev := []byte("device67890123456789012345678901")
name := []byte("name")
db := OpenMemory()
db.folderIdx.ID(fld)
db.deviceIdx.ID(dev)
key := db.deviceKey(fld, dev, name)
fld2 := db.deviceKeyFolder(key)
if !bytes.Equal(fld2, fld) {
t.Errorf("wrong folder %q != %q", fld2, fld)
}
dev2 := db.deviceKeyDevice(key)
if !bytes.Equal(dev2, dev) {
t.Errorf("wrong device %q != %q", dev2, dev)
}
name2 := db.deviceKeyName(key)
if !bytes.Equal(name2, name) {
t.Errorf("wrong name %q != %q", name2, name)
}
}
func TestGlobalKey(t *testing.T) {
fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
name := []byte("name")
db := OpenMemory()
db.folderIdx.ID(fld)
key := db.globalKey(fld, name)
fld2, ok := db.globalKeyFolder(key)
if !ok {
t.Error("should have been found")
}
if !bytes.Equal(fld2, fld) {
t.Errorf("wrong folder %q != %q", fld2, fld)
}
name2 := db.globalKeyName(key)
if !bytes.Equal(name2, name) {
t.Errorf("wrong name %q != %q", name2, name)
}
_, ok = db.globalKeyFolder([]byte{1, 2, 3, 4, 5})
if ok {
t.Error("should not have been found")
}
}
func TestDropIndexIDs(t *testing.T) {
db := OpenMemory()
@ -287,11 +234,11 @@ func TestUpdate0to3(t *testing.T) {
db.updateSchema0to1()
if _, ok := db.getFile(db.deviceKey(folder, protocol.LocalDeviceID[:], []byte(slashPrefixed))); ok {
if _, ok := db.getFile(db.keyer.GenerateDeviceFileKey(nil, folder, protocol.LocalDeviceID[:], []byte(slashPrefixed))); ok {
t.Error("File prefixed by '/' was not removed during transition to schema 1")
}
if _, err := db.Get(db.globalKey(folder, []byte(invalid)), nil); err != nil {
if _, err := db.Get(db.keyer.GenerateGlobalVersionKey(nil, folder, []byte(invalid)), nil); err != nil {
t.Error("Invalid file wasn't added to global list")
}

4
lib/db/leveldb_transactions.go
View File

@ -37,7 +37,7 @@ func (t readOnlyTransaction) close() {
}
func (t readOnlyTransaction) getFile(folder, device, file []byte) (protocol.FileInfo, bool) {
return t.db.getFile(t.db.deviceKey(folder, device, file))
return t.db.getFile(t.db.keyer.GenerateDeviceFileKey(nil, folder, device, file))
}
// A readWriteTransaction is a readOnlyTransaction plus a batch for writes.
@ -111,7 +111,7 @@ func (t readWriteTransaction) updateGlobal(gk, folder, device []byte, file proto
}
// Fixup the list of files we need.
nk := t.db.needKey(folder, name)
nk := t.db.keyer.GenerateNeedFileKey(nil, folder, name)
hasNeeded, _ := t.db.Has(nk, nil)
if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); need(newGlobal, haveLocalFV, localFV.Version) {
if !hasNeeded {

4
lib/db/meta.go
View File

@ -56,7 +56,7 @@ func (m *metadataTracker) Marshal() ([]byte, error) {
// toDB saves the marshalled metadataTracker to the given db, under the key
// corresponding to the given folder
func (m *metadataTracker) toDB(db *Instance, folder []byte) error {
key := db.folderMetaKey(folder)
key := db.keyer.GenerateFolderMetaKey(nil, folder)
bs, err := m.Marshal()
if err != nil {
return err
@ -67,7 +67,7 @@ func (m *metadataTracker) toDB(db *Instance, folder []byte) error {
// fromDB initializes the metadataTracker from the marshalled data found in
// the database under the key corresponding to the given folder
func (m *metadataTracker) fromDB(db *Instance, folder []byte) error {
key := db.folderMetaKey(folder)
key := db.keyer.GenerateFolderMetaKey(nil, folder)
bs, err := db.Get(key, nil)
if err != nil {
return err

6
lib/db/set.go
View File

@ -160,7 +160,7 @@ func (s *FileSet) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
var dk []byte
folder := []byte(s.folder)
for _, nf := range oldFs {
dk = s.db.deviceKeyInto(dk, folder, device[:], []byte(osutil.NormalizedFilename(nf.Name)))
dk = s.db.keyer.GenerateDeviceFileKey(dk, folder, device[:], []byte(osutil.NormalizedFilename(nf.Name)))
ef, ok := s.db.getFile(dk)
if ok && ef.Version.Equal(nf.Version) && ef.IsInvalid() == nf.IsInvalid() {
continue
@ -242,7 +242,7 @@ func (s *FileSet) WithPrefixedGlobalTruncated(prefix string, fn Iterator) {
}
func (s *FileSet) Get(device protocol.DeviceID, file string) (protocol.FileInfo, bool) {
f, ok := s.db.getFile(s.db.deviceKey([]byte(s.folder), device[:], []byte(osutil.NormalizedFilename(file))))
f, ok := s.db.getFile(s.db.keyer.GenerateDeviceFileKey(nil, []byte(s.folder), device[:], []byte(osutil.NormalizedFilename(file))))
f.Name = osutil.NativeFilename(f.Name)
return f, ok
}
@ -309,7 +309,7 @@ func (s *FileSet) SetIndexID(device protocol.DeviceID, id protocol.IndexID) {
}
func (s *FileSet) MtimeFS() *fs.MtimeFS {
prefix := s.db.mtimesKey([]byte(s.folder))
prefix := s.db.keyer.GenerateMtimesKey(nil, []byte(s.folder))
kv := NewNamespacedKV(s.db, string(prefix))
return fs.NewMtimeFS(s.fs, kv)
}

98
lib/db/smallindex.go Normal file
View File

@ -0,0 +1,98 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package db
import (
"encoding/binary"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/util"
)
// A smallIndex is an in memory bidirectional []byte to uint32 map. It gives
// fast lookups in both directions and persists to the database. Don't use for
// storing more items than fit comfortably in RAM.
type smallIndex struct {
db *leveldb.DB
prefix []byte
id2val map[uint32]string
val2id map[string]uint32
nextID uint32
mut sync.Mutex
}
func newSmallIndex(db *leveldb.DB, prefix []byte) *smallIndex {
idx := &smallIndex{
db: db,
prefix: prefix,
id2val: make(map[uint32]string),
val2id: make(map[string]uint32),
mut: sync.NewMutex(),
}
idx.load()
return idx
}
// load iterates over the prefix space in the database and populates the in
// memory maps.
func (i *smallIndex) load() {
it := i.db.NewIterator(util.BytesPrefix(i.prefix), nil)
defer it.Release()
for it.Next() {
val := string(it.Value())
id := binary.BigEndian.Uint32(it.Key()[len(i.prefix):])
i.id2val[id] = val
i.val2id[val] = id
if id >= i.nextID {
i.nextID = id + 1
}
}
}
// ID returns the index number for the given byte slice, allocating a new one
// and persisting this to the database if necessary.
func (i *smallIndex) ID(val []byte) uint32 {
i.mut.Lock()
// intentionally avoiding defer here as we want this call to be as fast as
// possible in the general case (folder ID already exists). The map lookup
// with the conversion of []byte to string is compiler optimized to not
// copy the []byte, which is why we don't assign it to a temp variable
// here.
if id, ok := i.val2id[string(val)]; ok {
i.mut.Unlock()
return id
}
id := i.nextID
i.nextID++
valStr := string(val)
i.val2id[valStr] = id
i.id2val[id] = valStr
key := make([]byte, len(i.prefix)+8) // prefix plus uint32 id
copy(key, i.prefix)
binary.BigEndian.PutUint32(key[len(i.prefix):], id)
i.db.Put(key, val, nil)
i.mut.Unlock()
return id
}
// Val returns the value for the given index number, or (nil, false) if there
// is no such index number.
func (i *smallIndex) Val(id uint32) ([]byte, bool) {
i.mut.Lock()
val, ok := i.id2val[id]
i.mut.Unlock()
if !ok {
return nil, false
}
return []byte(val), true
}