Martchus
/
lmdb-safe
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
lmdb-safe/lmdb-typed.hh

717 lines
19 KiB
C++
Raw Blame History

#pragma once
#include "./lmdb-safe.hh"
namespace LMDBSafe {
/*
Open issues:
What is an error? What is an exception?
could id=0 be magic? ('no such id')
yes
Perhaps use the separate index concept from multi_index
perhaps get eiter to be of same type so for(auto& a : x) works
make it more value "like" with unique_ptr
*/
/** Return the highest ID used in a database. Returns 0 for an empty DB.
This makes us start everything at ID=1, which might make it possible to
treat id 0 as special
*/
LMDB_SAFE_EXPORT unsigned int MDBGetMaxID(MDBRWTransaction& txn, MDBDbi& dbi);
/** This is the serialization interface.
You need to define your these functions for the types you'd like to store.
*/
template<typename T>
std::string serToString(const T& t);
template<typename T>
void serFromString(string_view str, T& ret);
/** This is the serialization interface for keys.
You need to define your these functions for the types you'd like to use as keys.
*/
template <class T, class Enable>
inline std::string keyConv(const T& t);
template <class T, typename std::enable_if<std::is_arithmetic<T>::value,T>::type* = nullptr>
inline string_view keyConv(const T& t)
{
return string_view(reinterpret_cast<const char *>(&t), sizeof(t));
}
template<class T, typename std::enable_if<std::is_same<T, std::string>::value,T>::type* = nullptr>
inline string_view keyConv(const T& t)
{
return t;
}
template<class T, typename std::enable_if<std::is_same<T, string_view>::value,T>::type* = nullptr>
inline string_view keyConv(string_view t)
{
return t;
}
/*!
* \brief The LMDBIndexOps struct implements index operations, but only the operations that
* are broadcast to all indexes.
*
* Specifically, to deal with databases with less than the maximum number of interfaces, this
* only includes calls that should be ignored for empty indexes.
*
* This class only needs methods that must happen for all indexes at once. So specifically, *not*
* size<t> or get<t>. People ask for those themselves, and should no do that on indexes that
* don't exist.
*/
template<class Class,typename Type, typename Parent>
struct LMDB_SAFE_EXPORT LMDBIndexOps
{
explicit LMDBIndexOps(Parent* parent) : d_parent(parent){}
void put(MDBRWTransaction& txn, const Class& t, uint32_t id, unsigned int flags=0)
{
txn->put(d_idx, keyConv(d_parent->getMember(t)), id, flags);
}
void del(MDBRWTransaction& txn, const Class& t, uint32_t id)
{
if(const auto rc = txn->del(d_idx, keyConv(d_parent->getMember(t)), id)) {
throw LMDBError("Error deleting from index: ", rc);
}
}
void clear(MDBRWTransaction& txn)
{
if (const auto rc = mdb_drop(*txn, d_idx, 0)) {
throw LMDBError("Error clearing index: ", rc);
}
}
void openDB(std::shared_ptr<MDBEnv>& env, string_view str, unsigned int flags)
{
d_idx = env->openDB(str, flags);
}
MDBDbi d_idx;
Parent* d_parent;
};
/** This is an index on a field in a struct, it derives from the LMDBIndexOps */
template<class Class,typename Type,Type Class::*PtrToMember>
struct index_on : LMDBIndexOps<Class, Type, index_on<Class, Type, PtrToMember>>
{
index_on() : LMDBIndexOps<Class, Type, index_on<Class, Type, PtrToMember>>(this)
{}
static Type getMember(const Class& c)
{
return c.*PtrToMember;
}
typedef Type type;
};
/** This is a calculated index */
template<class Class, typename Type, class Func>
struct index_on_function : LMDBIndexOps<Class, Type, index_on_function<Class, Type, Func> >
{
index_on_function() : LMDBIndexOps<Class, Type, index_on_function<Class, Type, Func> >(this)
{}
static Type getMember(const Class& c)
{
Func f;
return f(c);
}
typedef Type type;
};
/** nop index, so we can fill our N indexes, even if you don't use them all */
struct nullindex_t
{
template<typename Class>
void put(MDBRWTransaction& txn, const Class& t, uint32_t id, unsigned int flags=0)
{
(void)txn;
(void)t;
(void)id;
(void)flags;
}
template<typename Class>
void del(MDBRWTransaction& txn, const Class& t, uint32_t id)
{
(void)txn;
(void)t;
(void)id;
}
template<typename Class>
void clear(Class& txn)
{
(void)txn;
}
void openDB(std::shared_ptr<MDBEnv>& env, string_view str, unsigned int flags)
{
(void)env;
(void)str;
(void)flags;
}
typedef uint32_t type; // dummy
};
/** The main class. Templatized only on the indexes and typename right now */
template<typename T, class I1=nullindex_t, class I2=nullindex_t, class I3 = nullindex_t, class I4 = nullindex_t>
class LMDB_SAFE_EXPORT TypedDBI
{
public:
TypedDBI(std::shared_ptr<MDBEnv> env, string_view name)
: d_env(env), d_name(name)
{
d_main = d_env->openDB(name, MDB_CREATE | MDB_INTEGERKEY);
// now you might be tempted to go all MPL on this so we can get rid of the
// ugly macro. I'm not very receptive to that idea since it will make things
// EVEN uglier.
#define openMacro(N) std::get<N>(d_tuple).openDB(d_env, std::string(name)+"_"#N, MDB_CREATE | MDB_DUPFIXED | MDB_DUPSORT);
openMacro(0);
openMacro(1);
openMacro(2);
openMacro(3);
#undef openMacro
}
// we get a lot of our smarts from this tuple, it enables get<0> etc
typedef std::tuple<I1, I2, I3, I4> tuple_t;
tuple_t d_tuple;
// We support readonly and rw transactions. Here we put the Readonly operations
// which get sourced by both kinds of transactions
template<class Parent>
struct ReadonlyOperations
{
ReadonlyOperations(Parent& parent) : d_parent(parent)
{}
//! Number of entries in main database
size_t size()
{
MDB_stat stat;
mdb_stat(**d_parent.d_txn, d_parent.d_parent->d_main, &stat);
return stat.ms_entries;
}
//! Number of entries in the various indexes - should be the same
template<int N>
size_t size()
{
MDB_stat stat;
mdb_stat(**d_parent.d_txn, std::get<N>(d_parent.d_parent->d_tuple).d_idx, &stat);
return stat.ms_entries;
}
//! Get item with id, from main table directly
bool get(uint32_t id, T& t)
{
MDBOutVal data;
if((*d_parent.d_txn)->get(d_parent.d_parent->d_main, id, data))
return false;
serFromString(data.get<string_view>(), t);
return true;
}
//! Get item through index N, then via the main database
template<std::size_t N>
uint32_t get(const typename std::tuple_element<N, tuple_t>::type::type& key, T& out)
{
MDBOutVal id;
if(!(*d_parent.d_txn)->get(std::get<N>(d_parent.d_parent->d_tuple).d_idx, keyConv(key), id)) {
if(get(id.get<uint32_t>(), out))
return id.get<uint32_t>();
}
return 0;
}
//! Cardinality of index N
template<std::size_t N>
uint32_t cardinality()
{
auto cursor = (*d_parent.d_txn)->getCursor(std::get<N>(d_parent.d_parent->d_tuple).d_idx);
bool first = true;
MDBOutVal key, data;
uint32_t count = 0;
while(!cursor.get(key, data, first ? MDB_FIRST : MDB_NEXT_NODUP)) {
++count;
first=false;
}
return count;
}
//! End iderator type
struct eiter_t
{};
// can be on main, or on an index
// when on main, return data directly
// when on index, indirect
// we can be limited to one key, or iterate over entire database
// iter requires you to put the cursor in the right place first!
struct iter_t
{
explicit iter_t(Parent* parent, typename Parent::cursor_t&& cursor, bool on_index, bool one_key, bool end=false) :
d_parent(parent),
d_cursor(std::move(cursor)),
d_on_index(on_index), // is this an iterator on main database or on index?
d_one_key(one_key), // should we stop at end of key? (equal range)
d_end(end)
{
if(d_end)
return;
d_prefix.clear();
if(d_cursor.get(d_key, d_id, MDB_GET_CURRENT)) {
d_end = true;
return;
}
if(d_on_index) {
if((*d_parent->d_txn)->get(d_parent->d_parent->d_main, d_id, d_data))
throw LMDBError("Missing id in constructor");
serFromString(d_data.get<string_view>(), d_t);
}
else
serFromString(d_id.get<string_view>(), d_t);
}
explicit iter_t(Parent* parent, typename Parent::cursor_t&& cursor, string_view prefix) :
d_parent(parent),
d_cursor(std::move(cursor)),
d_on_index(true), // is this an iterator on main database or on index?
d_one_key(false),
d_prefix(prefix),
d_end(false)
{
if(d_end)
return;
if(d_cursor.get(d_key, d_id, MDB_GET_CURRENT)) {
d_end = true;
return;
}
if(d_on_index) {
if((*d_parent->d_txn)->get(d_parent->d_parent->d_main, d_id, d_data))
throw LMDBError("Missing id in constructor");
serFromString(d_data.get<string_view>(), d_t);
}
else
serFromString(d_id.get<string_view>(), d_t);
}
std::function<bool(const MDBOutVal&)> filter;
void del()
{
d_cursor.del();
}
bool operator!=(const eiter_t&) const
{
return !d_end;
}
bool operator==(const eiter_t&) const
{
return d_end;
}
const T& operator*()
{
return d_t;
}
const T* operator->()
{
return &d_t;
}
T &value()
{
return d_t;
}
// implements generic ++ or --
iter_t& genoperator(MDB_cursor_op dupop, MDB_cursor_op op)
{
MDBOutVal data;
next:;
const auto rc = d_cursor.get(d_key, d_id, d_one_key ? dupop : op);
if(rc == MDB_NOTFOUND) {
d_end = true;
}
else if(rc) {
throw LMDBError("Unable to get in genoperator: ", rc);
}
else if(!d_prefix.empty() && d_key.get<std::string>().rfind(d_prefix, 0)!=0) {
d_end = true;
}
else {
if(d_on_index) {
if((*d_parent->d_txn)->get(d_parent->d_parent->d_main, d_id, data))
throw LMDBError("Missing id field in genoperator");
if(filter && !filter(data))
goto next;
serFromString(data.get<string_view>(), d_t);
}
else {
if(filter && !filter(data))
goto next;
serFromString(d_id.get<string_view>(), d_t);
}
}
return *this;
}
iter_t& operator++()
{
return genoperator(MDB_NEXT_DUP, MDB_NEXT);
}
iter_t& operator--()
{
return genoperator(MDB_PREV_DUP, MDB_PREV);
}
// get ID this iterator points to
uint32_t getID()
{
if(d_on_index)
return d_id.get<uint32_t>();
else
return d_key.get<uint32_t>();
}
const MDBOutVal& getKey()
{
return d_key;
}
// transaction we are part of
Parent* d_parent;
typename Parent::cursor_t d_cursor;
// gcc complains if I don't zero-init these, which is worrying XXX
MDBOutVal d_key{{0,0}}, d_data{{0,0}}, d_id{{0,0}};
bool d_on_index;
bool d_one_key;
std::string d_prefix;
bool d_end{false};
T d_t;
};
template<int N>
iter_t genbegin(MDB_cursor_op op)
{
typename Parent::cursor_t cursor = (*d_parent.d_txn)->getCursor(std::get<N>(d_parent.d_parent->d_tuple).d_idx);
MDBOutVal out, id;
if(cursor.get(out, id, op)) {
// on_index, one_key, end
return iter_t{&d_parent, std::move(cursor), true, false, true};
}
return iter_t{&d_parent, std::move(cursor), true, false};
};
template<int N>
iter_t begin()
{
return genbegin<N>(MDB_FIRST);
}
template<int N>
iter_t rbegin()
{
return genbegin<N>(MDB_LAST);
}
iter_t begin()
{
typename Parent::cursor_t cursor = (*d_parent.d_txn)->getCursor(d_parent.d_parent->d_main);
MDBOutVal out, id;
if(cursor.get(out, id, MDB_FIRST)) {
// on_index, one_key, end
return iter_t{&d_parent, std::move(cursor), false, false, true};
}
return iter_t{&d_parent, std::move(cursor), false, false};
};
eiter_t end()
{
return eiter_t();
}
// basis for find, lower_bound
template<std::size_t N>
iter_t genfind(const typename std::tuple_element<N, tuple_t>::type::type& key, MDB_cursor_op op)
{
typename Parent::cursor_t cursor = (*d_parent.d_txn)->getCursor(std::get<N>(d_parent.d_parent->d_tuple).d_idx);
const auto keystr = keyConv(key);
MDBInVal in(keystr);
MDBOutVal out, id;
out.d_mdbval = in.d_mdbval;
if(cursor.get(out, id, op)) {
// on_index, one_key, end
return iter_t{&d_parent, std::move(cursor), true, false, true};
}
return iter_t{&d_parent, std::move(cursor), true, false};
};
template<std::size_t N>
iter_t find(const typename std::tuple_element<N, tuple_t>::type::type& key)
{
return genfind<N>(key, MDB_SET);
}
template<std::size_t N>
iter_t lower_bound(const typename std::tuple_element<N, tuple_t>::type::type& key)
{
return genfind<N>(key, MDB_SET_RANGE);
}
//! equal range - could possibly be expressed through genfind
template<std::size_t N>
std::pair<iter_t,eiter_t> equal_range(const typename std::tuple_element<N, tuple_t>::type::type& key)
{
typename Parent::cursor_t cursor = (*d_parent.d_txn)->getCursor(std::get<N>(d_parent.d_parent->d_tuple).d_idx);
const auto keyString=keyConv(key);
MDBInVal in(keyString);
MDBOutVal out, id;
out.d_mdbval = in.d_mdbval;
if(cursor.get(out, id, MDB_SET)) {
// on_index, one_key, end
return {iter_t{&d_parent, std::move(cursor), true, true, true}, eiter_t()};
}
return {iter_t{&d_parent, std::move(cursor), true, true}, eiter_t()};
};
//! equal range - could possibly be expressed through genfind
template<std::size_t N>
std::pair<iter_t,eiter_t> prefix_range(const typename std::tuple_element<N, tuple_t>::type::type& key)
{
typename Parent::cursor_t cursor = (*d_parent.d_txn)->getCursor(std::get<N>(d_parent.d_parent->d_tuple).d_idx);
const auto keyString=keyConv(key);
MDBInVal in(keyString);
MDBOutVal out, id;
out.d_mdbval = in.d_mdbval;
if(cursor.get(out, id, MDB_SET_RANGE)) {
// on_index, one_key, end
return {iter_t{&d_parent, std::move(cursor), true, true, true}, eiter_t()};
}
return {iter_t(&d_parent, std::move(cursor), keyString), eiter_t()};
};
Parent& d_parent;
};
class LMDB_SAFE_EXPORT ROTransaction : public ReadonlyOperations<ROTransaction>
{
public:
explicit ROTransaction(TypedDBI* parent) : ReadonlyOperations<ROTransaction>(*this), d_parent(parent), d_txn(std::make_shared<MDBROTransaction>(d_parent->d_env->getROTransaction()))
{
}
explicit ROTransaction(TypedDBI* parent, std::shared_ptr<MDBROTransaction> txn) : ReadonlyOperations<ROTransaction>(*this), d_parent(parent), d_txn(txn)
{
}
ROTransaction(ROTransaction&& rhs) :
ReadonlyOperations<ROTransaction>(*this), d_parent(rhs.d_parent),d_txn(std::move(rhs.d_txn))
{
rhs.d_parent = 0;
}
std::shared_ptr<MDBROTransaction> getTransactionHandle()
{
return d_txn;
}
typedef MDBROCursor cursor_t;
TypedDBI* d_parent;
std::shared_ptr<MDBROTransaction> d_txn;
};
class LMDB_SAFE_EXPORT RWTransaction : public ReadonlyOperations<RWTransaction>
{
public:
explicit RWTransaction(TypedDBI* parent) : ReadonlyOperations<RWTransaction>(*this), d_parent(parent)
{
d_txn = std::make_shared<MDBRWTransaction>(d_parent->d_env->getRWTransaction());
}
explicit RWTransaction(TypedDBI* parent, std::shared_ptr<MDBRWTransaction> txn) : ReadonlyOperations<RWTransaction>(*this), d_parent(parent), d_txn(txn)
{
}
RWTransaction(RWTransaction&& rhs) :
ReadonlyOperations<RWTransaction>(*this),
d_parent(rhs.d_parent), d_txn(std::move(rhs.d_txn))
{
rhs.d_parent = 0;
}
// insert something, with possibly a specific id
uint32_t put(const T& t, uint32_t id=0)
{
unsigned int flags = 0;
if(!id) {
id = MDBGetMaxID(*d_txn, d_parent->d_main) + 1;
flags = MDB_APPEND;
}
(*d_txn)->put(d_parent->d_main, id, serToString(t), flags);
#define insertMacro(N) std::get<N>(d_parent->d_tuple).put(*d_txn, t, id);
insertMacro(0);
insertMacro(1);
insertMacro(2);
insertMacro(3);
#undef insertMacro
return id;
}
// modify an item 'in place', plus update indexes
void modify(uint32_t id, std::function<void(T&)> func)
{
T t;
if(!this->get(id, t))
throw LMDBError("Could not modify id " + std::to_string(id));
func(t);
del(id); // this is the lazy way. We could test for changed index fields
put(t, id);
}
//! delete an item, and from indexes
void del(uint32_t id)
{
T t;
if(!this->get(id, t))
return;
(*d_txn)->del(d_parent->d_main, id);
clearIndex(id, t);
}
//! clear database & indexes
void clear()
{
if (const auto rc = mdb_drop(**d_txn, d_parent->d_main, 0)) {
throw LMDBError("Error database: ", rc);
}
#define clearMacro(N) std::get<N>(d_parent->d_tuple).clear(*d_txn);
clearMacro(0);
clearMacro(1);
clearMacro(2);
clearMacro(3);
#undef clearMacro
}
//! commit this transaction
void commit()
{
(*d_txn)->commit();
}
//! abort this transaction
void abort()
{
(*d_txn)->abort();
}
typedef MDBRWCursor cursor_t;
std::shared_ptr<MDBRWTransaction> getTransactionHandle()
{
return d_txn;
}
private:
// clear this ID from all indexes
void clearIndex(uint32_t id, const T& t)
{
#define clearMacro(N) std::get<N>(d_parent->d_tuple).del(*d_txn, t, id);
clearMacro(0);
clearMacro(1);
clearMacro(2);
clearMacro(3);
#undef clearMacro
}
public:
TypedDBI* d_parent;
std::shared_ptr<MDBRWTransaction> d_txn;
};
//! Get an RW transaction
RWTransaction getRWTransaction()
{
return RWTransaction(this);
}
//! Get an RO transaction
ROTransaction getROTransaction()
{
return ROTransaction(this);
}
//! Get an RW transaction
RWTransaction getRWTransaction(std::shared_ptr<MDBRWTransaction> txn)
{
return RWTransaction(this, txn);
}
//! Get an RO transaction
ROTransaction getROTransaction(std::shared_ptr<MDBROTransaction> txn)
{
return ROTransaction(this, txn);
}
std::shared_ptr<MDBEnv> getEnv()
{
return d_env;
}
private:
std::shared_ptr<MDBEnv> d_env;
MDBDbi d_main;
std::string d_name;
};
}
Reference in New Issue View Git Blame Copy Permalink