Martchus
/
lmdb-safe
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

big mdbval redo

This commit is contained in:
bert hubert 2018-12-10 14:51:02 +01:00
parent ac9da997a0
commit 0a22c24024
8 changed files with 183 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
Makefile
View File

@ -1,13 +1,15 @@
LIBS=lmdb-LMDB_0.9.22/libraries/liblmdb/liblmdb.a
INCLUDES=-Ilmdb-LMDB_0.9.22/libraries/liblmdb
#LIBS=lmdb-LMDB_0.9.22/libraries/liblmdb/liblmdb.a
#INCLUDES=-Ilmdb-LMDB_0.9.22/libraries/liblmdb
# LIBS=-llmdb
LIBS=-llmdb
CXXFLAGS:=-std=gnu++17 -Wall -O2 -MMD -MP -ggdb -pthread $(INCLUDES) # -fsanitize=address -fno-omit-frame-pointer
CFLAGS:= -Wall -O2 -MMD -MP -ggdb
CXXVERSIONFLAG= -std=gnu++17
PROGRAMS = lmdb-test basic-example scale-example multi-example
PROGRAMS = lmdb-test basic-example scale-example multi-example rel-example \
resize-example
all: $(PROGRAMS)
@ -18,13 +20,19 @@ clean:
lmdb-test: lmdb-test.o lmdb-safe.o
g++ -std=gnu++17 $^ -o $@ -pthread $(LIBS) #-lasan
g++ $(CXXVERSIONGLAG) $^ -o $@ -pthread $(LIBS) #-lasan
basic-example: basic-example.o lmdb-safe.o
g++ -std=gnu++17 $^ -o $@ -pthread $(LIBS)
g++ $(CXXVERSIONGLAG) $^ -o $@ -pthread $(LIBS)
scale-example: scale-example.o lmdb-safe.o
g++ -std=gnu++17 $^ -o $@ -pthread $(LIBS)
g++ $(CXXVERSIONGLAG) $^ -o $@ -pthread $(LIBS)
multi-example: multi-example.o lmdb-safe.o
g++ -std=gnu++17 $^ -o $@ -pthread $(LIBS)
g++ $(CXXVERSIONGLAG) $^ -o $@ -pthread $(LIBS)
rel-example: rel-example.o lmdb-safe.o
g++ $(CXXVERSIONGLAG) $^ -o $@ -pthread $(LIBS) -lboost_serialization
resize-example: resize-example.o lmdb-safe.o
g++ $(CXXVERSIONGLAG) $^ -o $@ -pthread $(LIBS)

8
basic-example.cc
View File

@ -3,9 +3,9 @@
void checkLMDB(MDBEnv* env, MDBDbi dbi)
{
auto rotxn = env->getROTransaction();
string_view data;
MDBOutVal data;
if(!rotxn.get(dbi, "lmdb", data)) {
cout<< "Outside RW transaction, found that lmdb = " << data <<endl;
cout<< "Outside RW transaction, found that lmdb = " << data.get<string_view>() <<endl;
}
else
cout<<"Outside RW transaction, found nothing" << endl;
@ -20,9 +20,9 @@ int main()
mdb_drop(txn, dbi, 0);
txn.put(dbi, "lmdb", "great");
string_view data;
MDBOutVal data;
if(!txn.get(dbi, "lmdb", data)) {
cout<< "Within RW transaction, found that lmdb = " << data <<endl;
cout<< "Within RW transaction, found that lmdb = " << data.get<string_view>() <<endl;
}
else
cout<<"Found nothing" << endl;

15
lmdb-safe.cc
View File

@ -231,18 +231,3 @@ void MDBRWTransaction::put(MDB_dbi dbi, string_view key, string_view val, int fl
put(dbi, MDB_val{key.size(), (void*)&key[0]}, MDB_val{val.size(), (void*)&val[0]}, flags);
}
int MDBRWTransaction::get(MDB_dbi dbi, string_view key, string_view& val)
{
MDB_val res;
int rc = get(dbi, MDB_val{key.size(), (void*)&key[0]}, res);
val=string_view((char*)res.mv_data, res.mv_size);
return rc;
}
int MDBROTransaction::get(MDB_dbi dbi, string_view key, string_view& val)
{
MDB_val res;
int rc = get(dbi, MDB_val{key.size(), (void*)&key[0]}, res);
val=string_view((char*)res.mv_data, res.mv_size);
return rc;
}

147
lmdb-safe.hh
View File

@ -5,6 +5,8 @@
#include <map>
#include <thread>
#include <memory>
#include <string>
#include <string.h>
#include <mutex>
using namespace std;
@ -87,6 +89,109 @@ private:
std::shared_ptr<MDBEnv> getMDBEnv(const char* fname, int flags, int mode);
struct MDBOutVal
{
operator MDB_val&()
{
return d_mdbval;
}
template <class T,
typename std::enable_if<std::is_arithmetic<T>::value,
T>::type* = nullptr>
T get()
{
T ret;
if(d_mdbval.mv_size != sizeof(T))
throw runtime_error("MDB data has wrong length for type");
memcpy(&ret, d_mdbval.mv_data, sizeof(T));
return ret;
}
template <class T,
typename std::enable_if<std::is_class<T>::value,T>::type* = nullptr>
T get();
template<class T>
T get_struct()
{
T ret;
if(d_mdbval.mv_size != sizeof(T))
throw runtime_error("MDB data has wrong length for type");
memcpy(&ret, d_mdbval.mv_data, sizeof(T));
return ret;
}
MDB_val d_mdbval;
};
template<> inline std::string MDBOutVal::get<std::string>()
{
return std::string((char*)d_mdbval.mv_data, d_mdbval.mv_size);
}
template<> inline std::string_view MDBOutVal::get<std::string_view>()
{
return std::string_view((char*)d_mdbval.mv_data, d_mdbval.mv_size);
}
class MDBInVal
{
public:
MDBInVal(const MDBOutVal& rhs)
{
d_mdbval = rhs.d_mdbval;
}
template <class T,
typename std::enable_if<std::is_arithmetic<T>::value,
T>::type* = nullptr>
MDBInVal(T i)
{
memcpy(&d_memory[0], &i, sizeof(i));
d_mdbval.mv_size = sizeof(T);
d_mdbval.mv_data = d_memory;;
}
MDBInVal(const char* s)
{
d_mdbval.mv_size = strlen(s);
d_mdbval.mv_data = (void*)s;
}
MDBInVal(const string_view& v)
{
d_mdbval.mv_size = v.size();
d_mdbval.mv_data = (void*)&v[0];
}
template<typename T>
static MDBInVal fromStruct(const T& t)
{
MDBInVal ret;
ret.d_mdbval.mv_size = sizeof(T);
ret.d_mdbval.mv_data = (void*)&t;
return ret;
}
operator MDB_val&()
{
return d_mdbval;
}
MDB_val d_mdbval;
private:
MDBInVal(){}
char d_memory[sizeof(double)];
};
class MDBROCursor;
class MDBROTransaction
@ -119,20 +224,29 @@ public:
}
int get(MDB_dbi dbi, const MDB_val& key, MDB_val& val)
int get(MDB_dbi dbi, const MDBInVal& key, MDBOutVal& val)
{
if(!d_txn)
throw std::runtime_error("Attempt to use a closed RO transaction for get");
int rc = mdb_get(d_txn, dbi, (MDB_val*)&key, &val);
int rc = mdb_get(d_txn, dbi, const_cast<MDB_val*>(&key.d_mdbval),
const_cast<MDB_val*>(&val.d_mdbval));
if(rc && rc != MDB_NOTFOUND)
throw std::runtime_error("getting data: " + std::string(mdb_strerror(rc)));
return rc;
}
int get(MDB_dbi dbi, string_view key, string_view& val);
int get(MDB_dbi dbi, const MDBInVal& key, string_view& val)
{
MDBOutVal out;
int rc = get(dbi, key, out);
if(!rc)
val = out.get<std::string_view>();
return rc;
}
// this is something you can do, readonly
MDBDbi openDB(const char* dbname, int flags)
{
@ -192,9 +306,9 @@ public:
mdb_cursor_close(d_cursor);
}
int get(MDB_val& key, MDB_val& data, MDB_cursor_op op)
int get(MDBOutVal& key, MDBOutVal& data, MDB_cursor_op op)
{
return mdb_cursor_get(d_cursor, &key, &data, op);
return mdb_cursor_get(d_cursor, &key.d_mdbval, &data.d_mdbval, op);
}
MDB_cursor* d_cursor;
@ -283,20 +397,27 @@ public:
return rc;
}
int get(MDB_dbi dbi, const MDB_val& key, MDB_val& val)
int get(MDB_dbi dbi, const MDBInVal& key, MDBOutVal& val)
{
if(!d_txn)
throw std::runtime_error("Attempt to use a closed RW transaction for get");
int rc = mdb_get(d_txn, dbi, (MDB_val*)&key, &val);
int rc = mdb_get(d_txn, dbi, const_cast<MDB_val*>(&key.d_mdbval),
const_cast<MDB_val*>(&val.d_mdbval));
if(rc && rc != MDB_NOTFOUND)
throw std::runtime_error("getting data: " + std::string(mdb_strerror(rc)));
return rc;
}
int get(MDB_dbi dbi, string_view key, string_view& val);
int get(MDB_dbi dbi, const MDBInVal& key, string_view& val)
{
MDBOutVal out;
int rc = get(dbi, key, out);
if(!rc)
val = out.get<std::string_view>();
return rc;
}
MDBDbi openDB(const char* dbname, int flags)
{
@ -369,9 +490,9 @@ public:
d_parent->unreportCursor(this);
}
int get(MDB_val& key, MDB_val& data, MDB_cursor_op op)
int get(MDBOutVal& key, MDBOutVal& data, MDB_cursor_op op)
{
int rc = mdb_cursor_get(d_cursor, &key, &data, op);
int rc = mdb_cursor_get(d_cursor, &key.d_mdbval, &data.d_mdbval, op);
if(rc && rc != MDB_NOTFOUND)
throw std::runtime_error("mdb_cursor_get: " + string(mdb_strerror(rc)));
return rc;

29
lmdb-test.cc
View File

@ -18,7 +18,7 @@ static void closeTest()
auto txn = env->getROTransaction();
for(auto& d : {&main, &dbi, &hyc}) {
auto rocursor = txn.getCursor(*d);
MDB_val key{0,0}, data{0,0};
MDBOutVal key, data;
if(rocursor.get(key, data, MDB_FIRST))
continue;
int count=0;
@ -61,9 +61,8 @@ try
for(int n=0; n < 15; ++n) {
auto txn = env->getROTransaction();
int val = n + 1000*tid;
MDB_val res;
if(txn.get(dbi, {sizeof(val), (char*)&val},
res)) {
MDBOutVal res;
if(txn.get(dbi, val, res)) {
throw std::runtime_error("no record");
}
@ -77,22 +76,6 @@ catch(std::exception& e)
throw;
}
struct MDBVal
{
MDBVal(unsigned int v) : d_v(v)
{
d_mdbval.mv_size=sizeof(v);
d_mdbval.mv_data = &d_v;
}
operator const MDB_val&()
{
return d_mdbval;
}
unsigned int d_v;
MDB_val d_mdbval;
};
void doFill()
{
auto env = getMDBEnv("./database", 0, 0600);
@ -101,7 +84,7 @@ void doFill()
for(int n = 0; n < 20; ++n) {
auto txn = env->getRWTransaction();
for(int j=0; j < 1000000; ++j) {
MDBVal mv(n*1000000+j);
MDBInVal mv(n*1000000+j);
txn.put(dbi, mv, mv, 0);
}
txn.commit();
@ -119,8 +102,8 @@ void doMeasure()
auto txn = env->getROTransaction();
unsigned int count=0;
for(int j=0; j < 1000000; ++j) {
MDBVal mv(n*1000000+j);
MDB_val res;
MDBInVal mv(n*1000000+j);
MDBOutVal res;
if(!txn.get(dbi, mv, res))
++count;
}

12
multi-example.cc
View File

@ -12,6 +12,7 @@ int main()
txn.put(dbi, "lmdb", "hot");
txn.put(dbi, "lmdb", "fast");
txn.put(dbi, "lmdb", "zooms");
txn.put(dbi, "lmdb", "c");
txn.put(dbi, "mdb", "old name");
std::string_view v1;
@ -26,13 +27,14 @@ int main()
txn = env->getRWTransaction();
auto cursor = txn.getCursor(dbi);
int count=0;
MDB_val key{4, (char*)"lmdb"};
MDB_val data{0,0};
MDBOutVal key, data;
MDBInVal start("lmdb");
key.d_mdbval = start.d_mdbval;
int rc=0;
while(!(rc=cursor.get(key, data, count ? MDB_NEXT_DUP : MDB_SET))) {
std::string_view k((char*)key.mv_data, key.mv_size);
std::string_view v((char*)data.mv_data, data.mv_size);
cout << k << " = " << v <<endl;
cout << key.get<string_view>() << " = " << data.get<string_view>() <<endl;
++count;
}
}

24
rel-example.cc
View File

@ -50,10 +50,10 @@ struct MDBVal
static unsigned int getMaxID(MDBRWTransaction& txn, MDBDbi& dbi)
{
auto cursor = txn.getCursor(dbi);
MDB_val maxidval, maxcontent;
MDBOutVal maxidval, maxcontent;
unsigned int maxid{0};
if(!cursor.get(maxidval, maxcontent, MDB_LAST)) {
memcpy(&maxid, maxidval.mv_data, 4);
maxid = maxidval.get<unsigned int>();
}
return maxid;
}
@ -94,7 +94,11 @@ int main(int argc, char** argv)
string prefix(argv[1]);
auto lim=atoi(argv[2]);
for(int n=0; n < lim; ++n) {
string domain(prefix+std::to_string(n)+".com");
string domain;
if(n)
domain.assign(prefix+std::to_string(n)+".com");
else
domain="powerdns.com";
Record r;
r.id=++maxid;
r.domain_id = ++maxdomainid;
@ -147,17 +151,19 @@ int main(int argc, char** argv)
auto rocursor = rotxn.getCursor(nameidx);
MDB_val data;
MDBOutVal data;
int count = 0;
while(!rocursor.get(MDBVal("www.powerdns.com"), data, count ? MDB_NEXT_DUP : MDB_SET)) {
unsigned int id;
memcpy(&id, data.mv_data, 4);
MDBOutVal key;
MDBInVal tmp("www.powerdns.com");
key.d_mdbval = tmp.d_mdbval;
while(!rocursor.get(key, data, count ? MDB_NEXT_DUP : MDB_SET)) {
unsigned int id = data.get<unsigned int>();
cout<<"Got something: id="<<id<<endl;
MDB_val record;
MDBOutVal record;
if(!rotxn.get(records, data, record)) {
Record test;
stringstream istr{std::string((char*)record.mv_data, record.mv_size)};
stringstream istr{record.get<string>()};
boost::archive::binary_iarchive oi(istr,boost::archive::no_header );
oi >> test;
cout <<"Record: "<<test.name<<" "<<test.type <<" " <<test.ttl<<" "<<test.content<<endl;

2
scale-example.cc
View File

@ -29,7 +29,7 @@ int main(int argc, char** argv)
cout<<"Counting records.. "; cout.flush();
auto cursor=txn.getCursor(dbi);
MDB_val key, data;
MDBOutVal key, data;
int count=0;
while(!cursor.get(key, data, count ? MDB_NEXT : MDB_FIRST)) {
count++;