Martchus
/
lmdb-safe
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

lots of changes

This commit is contained in:
bert hubert 2018-12-27 17:49:41 +01:00
parent 654985e981
commit 59b3b602fa
6 changed files with 104 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
Makefile
View File

@ -1,16 +1,14 @@
#LIBS=lmdb-LMDB_0.9.22/libraries/liblmdb/liblmdb.a
#INCLUDES=-Ilmdb-LMDB_0.9.22/libraries/liblmdb
#LIBS=lmdb-LMDB_0.9.23/libraries/liblmdb/liblmdb.a
#INCLUDES=-Ilmdb-LMDB_0.9.23/libraries/liblmdb
LIBS=-llmdb
CXXVERSIONFLAG= -std=gnu++11
CXXFLAGS:= $(CXXVERSIONFLAG) -Wall -O2 -MMD -MP -ggdb -pthread $(INCLUDES) # -fsanitize=address -fno-omit-frame-pointer
CXXFLAGS:= $(CXXVERSIONFLAG) -Wall -O2 -MMD -MP -ggdb -pthread $(INCLUDES) -Iext # -fsanitize=address -fno-omit-frame-pointer
CFLAGS:= -Wall -O2 -MMD -MP -ggdb
PROGRAMS = lmdb-test basic-example scale-example multi-example rel-example \
resize-example typed-example
PROGRAMS = lmdb-various basic-example scale-example multi-example rel-example \
resize-example typed-example test-basic lmdb-view
all: $(PROGRAMS)
@ -19,9 +17,14 @@ clean:
-include *.d
test-basic: test-basic.o lmdb-safe.o
g++ $(CXXVERSIONFLAG) $^ -o $@ -pthread $(LIBS)
lmdb-test: lmdb-test.o lmdb-safe.o
g++ $(CXXVERSIONFLAG) $^ -o $@ -pthread $(LIBS) #-lasan
lmdb-various: lmdb-various.o lmdb-safe.o
g++ $(CXXVERSIONFLAG) $^ -o $@ -pthread $(LIBS)
lmdb-view: lmdb-view.o lmdb-safe.o
g++ $(CXXVERSIONFLAG) $^ -o $@ $(LIBS)
basic-example: basic-example.o lmdb-safe.o
g++ $(CXXVERSIONFLAG) $^ -o $@ -pthread $(LIBS)

14
lmdb-safe.cc
View File

@ -17,7 +17,7 @@ MDBDbi::MDBDbi(MDB_env* env, MDB_txn* txn, const string_view dbname, int flags)
{
// A transaction that uses this function must finish (either commit or abort) before any other transaction in the process may use this function.
int rc = mdb_dbi_open(txn, &dbname[0], flags, &d_dbi);
int rc = mdb_dbi_open(txn, dbname.empty() ? 0 : &dbname[0], flags, &d_dbi);
if(rc)
throw std::runtime_error("Unable to open named database: " + MDBError(rc));
@ -139,13 +139,17 @@ MDBDbi MDBEnv::openDB(const string_view dbname, int flags)
if(!(envflags & MDB_RDONLY)) {
auto rwt = getRWTransaction();
MDBDbi ret = rwt.openDB(&dbname[0], flags);
MDBDbi ret = rwt.openDB(dbname, flags);
rwt.commit();
return ret;
}
auto rwt = getROTransaction();
return rwt.openDB(dbname, flags);
MDBDbi ret;
{
auto rwt = getROTransaction();
ret = rwt.openDB(dbname, flags);
}
return ret;
}
MDBRWTransaction::MDBRWTransaction(MDBEnv* parent, int flags) : d_parent(parent)

4
lmdb-safe.hh
View File

@ -259,7 +259,7 @@ public:
// this is something you can do, readonly
MDBDbi openDB(string_view dbname, int flags)
{
return MDBDbi(d_parent->d_env, d_txn, dbname, flags);
return MDBDbi( d_parent->d_env, d_txn, dbname, flags);
}
MDBROCursor getCursor(const MDBDbi&);
@ -444,7 +444,7 @@ public:
return rc;
}
MDBDbi openDB(const char* dbname, int flags)
MDBDbi openDB(string_view dbname, int flags)
{
return MDBDbi(d_parent->d_env, d_txn, dbname, flags);
}

1
lmdb-typed.cc
View File

@ -1,6 +1,5 @@
#include "lmdb-typed.hh"
unsigned int MDBGetMaxID(MDBRWTransaction& txn, MDBDbi& dbi)
{
auto cursor = txn.getCursor(dbi);

101
lmdb-typed.hh
View File

@ -53,11 +53,41 @@ template<typename T>
void serFromString(const std::string& str, T& ret)
{
ret = T();
boost::iostreams::array_source source(str.c_str(), str.size());
boost::iostreams::stream<boost::iostreams::array_source> stream(source);
boost::archive::binary_iarchive in_archive(stream, boost::archive::no_header|boost::archive::no_codecvt);
in_archive >> ret;
/*
std::istringstream istr{str};
boost::archive::binary_iarchive oi(istr,boost::archive::no_header|boost::archive::no_codecvt );
oi >> ret;
*/
}
template<typename KeyType>
std::string keyConv(const KeyType& t);
template<>
inline std::string keyConv(const std::string& t)
{
return t;
}
template<>
inline std::string keyConv(const uint32_t& t)
{
return std::string((char*)&t, sizeof(4));
}
template<>
inline std::string keyConv(const int32_t& t)
{
return std::string((char*)&t, sizeof(4));
}
/** This is a struct that implements index operations, but
only the operations that are broadcast to all indexes.
Specifically, to deal with databases with less than the maximum
@ -72,14 +102,16 @@ template<class Class,typename Type, typename Parent>
struct LMDBIndexOps
{
explicit LMDBIndexOps(Parent* parent) : d_parent(parent){}
void put(MDBRWTransaction& txn, const Class& t, uint32_t id)
void put(MDBRWTransaction& txn, const Class& t, uint32_t id, int flags=0)
{
txn.put(d_idx, d_parent->getMember(t), id);
txn.put(d_idx, keyConv(d_parent->getMember(t)), id, flags);
}
void del(MDBRWTransaction& txn, const Class& t, uint32_t id)
{
txn.del(d_idx, d_parent->getMember(t), id);
if(int rc = txn.del(d_idx, keyConv(d_parent->getMember(t)), id)) {
throw std::runtime_error("Error deleting from index: " + std::string(mdb_strerror(rc)));
}
}
void openDB(std::shared_ptr<MDBEnv>& env, string_view str, int flags)
@ -124,7 +156,7 @@ struct index_on_function : LMDBIndexOps<Class, Type, index_on_function<Class, Ty
struct nullindex_t
{
template<typename Class>
void put(MDBRWTransaction& txn, const Class& t, uint32_t id)
void put(MDBRWTransaction& txn, const Class& t, uint32_t id, int flags=0)
{}
template<typename Class>
void del(MDBRWTransaction& txn, const Class& t, uint32_t id)
@ -137,6 +169,7 @@ struct nullindex_t
typedef uint32_t type; // dummy
};
/** The main class. Templatized only on the indexes and typename right now */
template<typename T, class I1=nullindex_t, class I2=nullindex_t, class I3 = nullindex_t, class I4 = nullindex_t>
class TypedDBI
@ -156,9 +189,9 @@ public:
openMacro(2);
openMacro(3);
#undef openMacro
}
// we get a lot of our smarts from this tuple, it enables get<0> etc
typedef std::tuple<I1, I2, I3, I4> tuple_t;
tuple_t d_tuple;
@ -204,7 +237,7 @@ public:
uint32_t get(const typename std::tuple_element<N, tuple_t>::type::type& key, T& out)
{
MDBOutVal id;
if(!d_parent.d_txn->get(std::get<N>(d_parent.d_parent->d_tuple).d_idx, key, id)) {
if(!d_parent.d_txn->get(std::get<N>(d_parent.d_parent->d_tuple).d_idx, keyConv(key), id)) {
if(get(id.get<uint32_t>(), out))
return id.get<uint32_t>();
}
@ -261,7 +294,7 @@ public:
serFromString(d_id.get<std::string>(), d_t);
}
std::function<bool(const MDBOutVal&)> filter;
void del()
{
d_cursor.del();
@ -291,19 +324,30 @@ public:
iter_t& genoperator(MDB_cursor_op dupop, MDB_cursor_op op)
{
MDBOutVal data;
int rc = d_cursor.get(d_key, d_id, d_one_key ? dupop : op);
int rc;
next:;
rc = d_cursor.get(d_key, d_id, d_one_key ? dupop : op);
if(rc == MDB_NOTFOUND) {
d_end = true;
}
else if(rc) {
throw std::runtime_error("in genoperator, " + std::string(mdb_strerror(rc)));
}
else {
if(d_on_index) {
if(d_parent->d_txn->get(d_parent->d_parent->d_main, d_id, data))
throw std::runtime_error("Missing id field");
if(filter && !filter(data))
goto next;
serFromString(data.get<std::string>(), d_t);
}
else
else {
if(filter && !filter(data))
goto next;
serFromString(d_id.get<std::string>(), d_t);
}
}
return *this;
}
@ -339,20 +383,32 @@ public:
};
template<int N>
iter_t begin()
iter_t genbegin(MDB_cursor_op op)
{
typename Parent::cursor_t cursor = d_parent.d_txn->getCursor(std::get<N>(d_parent.d_parent->d_tuple).d_idx);
MDBOutVal out, id;
if(cursor.get(out, id, MDB_FIRST))
{ // on_index, one_key, end
if(cursor.get(out, id, op)) {
// on_index, one_key, end
return iter_t{&d_parent, std::move(cursor), true, false, true};
}
return iter_t{&d_parent, std::move(cursor), true, false};
};
template<int N>
iter_t begin()
{
return genbegin<N>(MDB_FIRST);
}
template<int N>
iter_t rbegin()
{
return genbegin<N>(MDB_LAST);
}
iter_t begin()
{
typename Parent::cursor_t cursor = d_parent.d_txn->getCursor(d_parent.d_parent->d_main);
@ -377,8 +433,9 @@ public:
iter_t genfind(const typename std::tuple_element<N, tuple_t>::type::type& key, MDB_cursor_op op)
{
typename Parent::cursor_t cursor = d_parent.d_txn->getCursor(std::get<N>(d_parent.d_parent->d_tuple).d_idx);
MDBInVal in(key);
std::string keystr = keyConv(key);
MDBInVal in(keystr);
MDBOutVal out, id;
out.d_mdbval = in.d_mdbval;
@ -408,8 +465,9 @@ public:
std::pair<iter_t,eiter_t> equal_range(const typename std::tuple_element<N, tuple_t>::type::type& key)
{
typename Parent::cursor_t cursor = d_parent.d_txn->getCursor(std::get<N>(d_parent.d_parent->d_tuple).d_idx);
MDBInVal in(key);
std::string keyString=keyConv(key);
MDBInVal in(keyString);
MDBOutVal out, id;
out.d_mdbval = in.d_mdbval;
@ -478,9 +536,12 @@ public:
// insert something, with possibly a specific id
uint32_t put(const T& t, uint32_t id=0)
{
if(!id)
int flags = 0;
if(!id) {
id = MDBGetMaxID(*d_txn, d_parent->d_main) + 1;
d_txn->put(d_parent->d_main, id, serToString(t));
flags = MDB_APPEND;
}
d_txn->put(d_parent->d_main, id, serToString(t), flags);
#define insertMacro(N) std::get<N>(d_parent->d_tuple).put(*d_txn, t, id);
insertMacro(0);
@ -496,8 +557,8 @@ public:
void modify(uint32_t id, std::function<void(T&)> func)
{
T t;
if(!this->get(id, t))
return; // XXX should be exception
if(!this->get(id, t))
throw std::runtime_error("Could not modify id "+std::to_string(id));
func(t);
del(id); // this is the lazy way. We could test for changed index fields

0
lmdb-test.cc → lmdb-various.cc
View File