eradicate some MDB_dbi use, more string_view

This commit is contained in:
bert hubert 2018-12-14 23:00:44 +01:00
parent ac4876876e
commit d98d657765
5 changed files with 464 additions and 19 deletions

View File

@ -10,7 +10,7 @@ CFLAGS:= -Wall -O2 -MMD -MP -ggdb
PROGRAMS = lmdb-test basic-example scale-example multi-example rel-example \
resize-example
resize-example lmdb-typed
all: $(PROGRAMS)
@ -21,19 +21,22 @@ clean:
lmdb-test: lmdb-test.o lmdb-safe.o
g++ $(CXXVERSIONGLAG) $^ -o $@ -pthread $(LIBS) #-lasan
g++ $(CXXVERSIONFLAG) $^ -o $@ -pthread $(LIBS) #-lasan
basic-example: basic-example.o lmdb-safe.o
g++ $(CXXVERSIONGLAG) $^ -o $@ -pthread $(LIBS)
g++ $(CXXVERSIONFLAG) $^ -o $@ -pthread $(LIBS)
scale-example: scale-example.o lmdb-safe.o
g++ $(CXXVERSIONGLAG) $^ -o $@ -pthread $(LIBS)
g++ $(CXXVERSIONFLAG) $^ -o $@ -pthread $(LIBS)
multi-example: multi-example.o lmdb-safe.o
g++ $(CXXVERSIONGLAG) $^ -o $@ -pthread $(LIBS)
g++ $(CXXVERSIONFLAG) $^ -o $@ -pthread $(LIBS)
rel-example: rel-example.o lmdb-safe.o
g++ $(CXXVERSIONGLAG) $^ -o $@ -pthread $(LIBS) -lboost_serialization
g++ $(CXXVERSIONFLAG) $^ -o $@ -pthread $(LIBS) -lboost_serialization
resize-example: resize-example.o lmdb-safe.o
g++ $(CXXVERSIONGLAG) $^ -o $@ -pthread $(LIBS)
g++ $(CXXVERSIONFLAG) $^ -o $@ -pthread $(LIBS)
lmdb-typed: lmdb-typed.o lmdb-safe.o
g++ $(CXXVERSIONFLAG) $^ -o $@ -pthread $(LIBS) -lboost_serialization

View File

@ -13,11 +13,11 @@ static string MDBError(int rc)
return mdb_strerror(rc);
}
MDBDbi::MDBDbi(MDB_env* env, MDB_txn* txn, const char* dbname, int flags)
MDBDbi::MDBDbi(MDB_env* env, MDB_txn* txn, const string_view dbname, int flags)
{
// A transaction that uses this function must finish (either commit or abort) before any other transaction in the process may use this function.
int rc = mdb_dbi_open(txn, dbname, flags, &d_dbi);
int rc = mdb_dbi_open(txn, &dbname[0], flags, &d_dbi);
if(rc)
throw std::runtime_error("Unable to open named database: " + MDBError(rc));
@ -27,7 +27,7 @@ MDBDbi::MDBDbi(MDB_env* env, MDB_txn* txn, const char* dbname, int flags)
MDBEnv::MDBEnv(const char* fname, int flags, int mode)
{
mdb_env_create(&d_env);
if(mdb_env_set_mapsize(d_env, 4ULL*4096*244140ULL)) // 4GB
if(mdb_env_set_mapsize(d_env, 16ULL*4096*244140ULL)) // 4GB
throw std::runtime_error("setting map size");
/*
Various other options may also need to be set before opening the handle, e.g. mdb_env_set_mapsize(), mdb_env_set_maxreaders(), mdb_env_set_maxdbs(),
@ -128,7 +128,7 @@ std::shared_ptr<MDBEnv> getMDBEnv(const char* fname, int flags, int mode)
}
MDBDbi MDBEnv::openDB(const char* dbname, int flags)
MDBDbi MDBEnv::openDB(const string_view dbname, int flags)
{
unsigned int envflags;
mdb_env_get_flags(d_env, &envflags);
@ -139,7 +139,7 @@ MDBDbi MDBEnv::openDB(const char* dbname, int flags)
if(!(envflags & MDB_RDONLY)) {
auto rwt = getRWTransaction();
MDBDbi ret = rwt.openDB(dbname, flags);
MDBDbi ret = rwt.openDB(&dbname[0], flags);
rwt.commit();
return ret;
}

View File

@ -40,7 +40,7 @@ public:
{
d_dbi = -1;
}
explicit MDBDbi(MDB_env* env, MDB_txn* txn, const char* dbname, int flags);
explicit MDBDbi(MDB_env* env, MDB_txn* txn, string_view dbname, int flags);
operator const MDB_dbi&() const
{
@ -65,7 +65,7 @@ public:
// but, elsewhere, docs say database handles do not need to be closed?
}
MDBDbi openDB(const char* dbname, int flags);
MDBDbi openDB(const string_view dbname, int flags);
MDBRWTransaction getRWTransaction();
MDBROTransaction getROTransaction();
@ -257,7 +257,7 @@ public:
// this is something you can do, readonly
MDBDbi openDB(const char* dbname, int flags)
MDBDbi openDB(string_view dbname, int flags)
{
return MDBDbi(d_parent->d_env, d_txn, dbname, flags);
}
@ -408,7 +408,7 @@ public:
}
int del(MDB_dbi dbi, const MDBInVal& key, const MDBInVal& val)
int del(MDBDbi& dbi, const MDBInVal& key, const MDBInVal& val)
{
int rc;
rc=mdb_del(d_txn, dbi, (MDB_val*)&key.d_mdbval, (MDB_val*)&val.d_mdbval);
@ -417,7 +417,7 @@ public:
return rc;
}
int del(MDB_dbi dbi, const MDBInVal& key)
int del(MDBDbi& dbi, const MDBInVal& key)
{
int rc;
rc=mdb_del(d_txn, dbi, (MDB_val*)&key.d_mdbval, 0);
@ -427,7 +427,7 @@ public:
}
int get(MDB_dbi dbi, const MDBInVal& key, MDBOutVal& val)
int get(MDBDbi& dbi, const MDBInVal& key, MDBOutVal& val)
{
if(!d_txn)
throw std::runtime_error("Attempt to use a closed RW transaction for get");
@ -439,7 +439,7 @@ public:
return rc;
}
int get(MDB_dbi dbi, const MDBInVal& key, string_view& val)
int get(MDBDbi& dbi, const MDBInVal& key, string_view& val)
{
MDBOutVal out;
int rc = get(dbi, key, out);

112
lmdb-typed.cc Normal file
View File

@ -0,0 +1,112 @@
#include "lmdb-typed.hh"
#include <string>
unsigned int getMaxID(MDBRWTransaction& txn, MDBDbi& dbi)
{
auto cursor = txn.getCursor(dbi);
MDBOutVal maxidval, maxcontent;
unsigned int maxid{0};
if(!cursor.get(maxidval, maxcontent, MDB_LAST)) {
maxid = maxidval.get<unsigned int>();
}
return maxid;
}
using namespace std;
struct DNSResourceRecord
{
string qname; // index
uint16_t qtype{0};
uint32_t domain_id{0}; // index
string content;
uint32_t ttl{0};
string ordername; // index
bool auth{true};
};
template<class Archive>
void serialize(Archive & ar, DNSResourceRecord& g, const unsigned int version)
{
ar & g.qtype;
ar & g.qname;
ar & g.content;
ar & g.ttl;
ar & g.domain_id;
ar & g.ordername;
ar & g.auth;
}
int main()
{
TypedDBI<DNSResourceRecord,
index_on<DNSResourceRecord, string, &DNSResourceRecord::qname>,
index_on<DNSResourceRecord, uint32_t, &DNSResourceRecord::domain_id>,
index_on<DNSResourceRecord, string, &DNSResourceRecord::ordername>
> tdbi(getMDBEnv("./typed.lmdb", MDB_NOSUBDIR, 0600), "records");
auto txn = tdbi.getRWTransaction();
#if 0
cout<<"Going to iterate over powerdns.com!"<<endl;
for(auto iter = txn.find1("powerdns.com"); iter != txn.end(); ++iter) {
cout << iter->qname << " " << iter->qtype << " " << iter->content <<endl;
}
cout<<"Done iterating"<<endl;
#endif
DNSResourceRecord rr;
rr.domain_id=0;
rr.qtype = 5;
rr.ttl = 3600;
rr.qname = "www.powerdns.com";
rr.ordername = "www";
rr.content = "powerdns.com";
auto id = txn.insert(rr);
cout<<"Inserted as id "<<id<<endl;
rr.qname = "powerdns.com";
rr.qtype = 1;
rr.ordername=" ";
rr.content = "1.2.3.4";
id = txn.insert(rr);
cout<<"Inserted as id "<<id<<endl;
rr.qtype = 2;
rr.content = "ns1.powerdns.com";
rr.ordername = "ns1";
id = txn.insert(rr);
cout<<"Inserted as id "<<id<<endl;
rr.content = "ns2.powerdns.com";
rr.ordername = "ns2";
id = txn.insert(rr);
cout<<"Inserted as id "<<id<<endl;
DNSResourceRecord rr2;
id = txn.get1("www.powerdns.com", rr2);
cout<<"Retrieved id "<< id <<", content: "<<rr2.content<<endl;
id = txn.get1("powerdns.com", rr2);
cout<<"Retrieved id "<< id <<", content: "<<rr2.content<<endl;
DNSResourceRecord rr3;
id = txn.get1("powerdns.com", rr3);
cout<< id << endl;
txn.del(1);
DNSResourceRecord rr4;
id = txn.get3("ns1", rr4);
cout<<"Found "<<id<<": " << rr4.content <<endl;
txn.commit();
}

330
lmdb-typed.hh Normal file
View File

@ -0,0 +1,330 @@
#pragma once
#include <iostream>
#include "lmdb-safe.hh"
#include <boost/archive/binary_oarchive.hpp>
#include <boost/archive/binary_iarchive.hpp>
#include <boost/serialization/vector.hpp>
#include <boost/serialization/string.hpp>
#include <boost/serialization/utility.hpp>
#include <boost/iostreams/stream.hpp>
#include <boost/iostreams/stream_buffer.hpp>
#include <boost/iostreams/device/back_inserter.hpp>
#include <sstream>
using std::cout;
using std::endl;
unsigned int getMaxID(MDBRWTransaction& txn, MDBDbi& dbi);
template<typename T>
std::string serToString(const T& t)
{
std::string serial_str;
boost::iostreams::back_insert_device<std::string> inserter(serial_str);
boost::iostreams::stream<boost::iostreams::back_insert_device<std::string> > s(inserter);
boost::archive::binary_oarchive oa(s, boost::archive::no_header | boost::archive::no_codecvt);
oa << t;
return serial_str;
}
template<typename T>
void serFromString(const std::string& str, T& ret)
{
ret = T();
std::istringstream istr{str};
boost::archive::binary_iarchive oi(istr,boost::archive::no_header|boost::archive::no_codecvt );
oi >> ret;
}
/* This is for storing a struct that has to be found using several
of its fields.
We want a typed lmdb database that we can only:
* insert such structs
* remove them
* mutate them
All while maintaining indexes on insert, removal and mutation.
struct DNSResourceRecord
{
string qname; // index
string qtype;
uint32_t domain_id; // index
string content;
string ordername; // index
bool auth;
}
TypedDBI<DNSResourceRecord,
DNSResourceRecord::qname,
DNSResourceRecord::domain_id,
DNSResourceRecord::ordername> tdbi;
DNSResourceRecord rr;
uint32_t id = tdbi.insert(rr); // inserts, creates three index items
tdbi.modify(id, [](auto& rr) { rr.auth=false; });
DNSResourceRecord retrr;
uint32_t id = tdbi.get<1>(qname, retrr);
// this checks for changes and updates indexes if need be
tdbi.modify(id, [](auto& rr) { rr.ordername="blah"; });
*/
template<class Class,typename Type,Type Class::*PtrToMember>
struct index_on
{
static Type getMember(const Class& c)
{
return c.*PtrToMember;
}
void put(MDBRWTransaction& txn, const Class& t, uint32_t id)
{
txn.put(d_idx, getMember(t), id);
}
void del(MDBRWTransaction& txn, const Class& t, uint32_t id)
{
txn.del(d_idx, getMember(t), id);
}
void openDB(std::shared_ptr<MDBEnv>& env, string_view str, int flags)
{
d_idx = env->openDB(str, flags);
}
typedef Type type;
MDBDbi d_idx;
};
struct nullindex_t
{
template<typename Class>
void put(MDBRWTransaction& txn, const Class& t, uint32_t id)
{}
template<typename Class>
void del(MDBRWTransaction& txn, const Class& t, uint32_t id)
{}
void openDB(std::shared_ptr<MDBEnv>& env, string_view str, int flags)
{
}
typedef uint32_t type; // dummy
};
template<typename T, class I1=nullindex_t, class I2=nullindex_t, class I3 = nullindex_t, class I4 = nullindex_t>
class TypedDBI
{
public:
TypedDBI(std::shared_ptr<MDBEnv> env, string_view name)
: d_env(env), d_name(name)
{
d_main = d_env->openDB(name, MDB_CREATE | MDB_INTEGERKEY);
d_i1.openDB(d_env, std::string(name)+"_1", MDB_CREATE | MDB_DUPFIXED | MDB_DUPSORT);
d_i2.openDB(d_env, std::string(name)+"_2", MDB_CREATE | MDB_DUPFIXED | MDB_DUPSORT);
d_i3.openDB(d_env, std::string(name)+"_3", MDB_CREATE | MDB_DUPFIXED | MDB_DUPSORT);
d_i4.openDB(d_env, std::string(name)+"_4", MDB_CREATE | MDB_DUPFIXED | MDB_DUPSORT);
}
I1 d_i1;
I2 d_i2;
I3 d_i3;
I4 d_i4;
class RWTransaction
{
public:
explicit RWTransaction(TypedDBI* parent) : d_parent(parent), d_txn(d_parent->d_env->getRWTransaction())
{
}
RWTransaction(RWTransaction&& rhs) :
d_parent(rhs.d_parent), d_txn(std::move(rhs.d_txn))
{
rhs.d_parent = 0;
}
uint32_t insert(const T& t)
{
uint32_t id = getMaxID(d_txn, d_parent->d_main) + 1;
d_txn.put(d_parent->d_main, id, serToString(t));
d_parent->d_i1.put(d_txn, t, id);
d_parent->d_i2.put(d_txn, t, id);
d_parent->d_i3.put(d_txn, t, id);
d_parent->d_i4.put(d_txn, t, id);
return id;
}
bool get(uint32_t id, T& t)
{
MDBOutVal data;
if(d_txn.get(d_parent->d_main, id, data))
return false;
serFromString(data.get<std::string>(), t);
return true;
}
void del(uint32_t id)
{
T t;
if(!get(id, t))
return;
d_txn.del(d_parent->d_main, id);
d_parent->d_i1.del(d_txn, t, id);
d_parent->d_i2.del(d_txn, t, id);
d_parent->d_i3.del(d_txn, t, id);
d_parent->d_i4.del(d_txn, t, id);
}
uint32_t get1(const typename I1::type& key, T& out)
{
MDBOutVal id;
if(!d_txn.get(d_parent->d_i1.d_idx, key, id))
return get(id.get<uint32_t>(), out);
return 0;
}
uint32_t get2(const typename I2::type& key, T& out)
{
MDBOutVal id;
if(!d_txn.get(d_parent->d_i2.d_idx, key, id))
return get(id.get<uint32_t>(), out);
return 0;
}
uint32_t get3(const typename I3::type& key, T& out)
{
MDBOutVal id;
if(!d_txn.get(d_parent->d_i3.d_idx, key, id))
return get(id.get<uint32_t>(), out);
return 0;
}
uint32_t get4(const typename I4::type& key, T& out)
{
MDBOutVal id;
if(!d_txn.get(d_parent->d_i4.d_idx, key, id))
return get(id.get<uint32_t>(), out);
return 0;
}
void commit()
{
d_txn.commit();
}
void abort()
{
d_txn.abort();
}
private:
TypedDBI* d_parent;
MDBRWTransaction d_txn;
};
RWTransaction getRWTransaction()
{
return RWTransaction(this);
}
private:
std::shared_ptr<MDBEnv> d_env;
MDBDbi d_main;
std::string d_name;
};
#if 0
struct eiter1_t
{};
struct iter1_t
{
explicit iter1_t(MDBROTransaction && txn, const MDBDbi& dbi, const MDBDbi& main, const typename I1::type& key) : d_txn(std::move(txn)), d_cursor(d_txn.getCursor(dbi)), d_in(key), d_main(main)
{
d_key.d_mdbval = d_in.d_mdbval;
MDBOutVal id, data;
if(d_cursor.get(d_key, id, MDB_SET)) {
d_end = true;
return;
}
if(d_txn.get(d_main, id, data))
throw std::runtime_error("Missing id in constructor");
serFromString(data.get<std::string>(), d_t);
}
bool operator!=(const eiter1_t& rhs)
{
return !d_end;
}
bool operator==(const eiter1_t& rhs)
{
return d_end;
}
const T& operator*()
{
return d_t;
}
const T* operator->()
{
return &d_t;
}
iter1_t& operator++()
{
MDBOutVal id, data;
int rc = d_cursor.get(d_key, id, MDB_NEXT_DUP);
if(rc == MDB_NOTFOUND) {
d_end = true;
}
else {
if(d_txn.get(d_main, id, data))
throw std::runtime_error("Missing id field");
serFromString(data.get<std::string>(), d_t);
}
return *this;
}
MDBROTransaction d_txn;
MDBROCursor d_cursor;
MDBOutVal d_key, d_data;
MDBInVal d_in;
bool d_end{false};
T d_t;
MDBDbi d_main;
};
iter1_t find1(const typename I1::type& key)
{
iter1_t ret{std::move(d_env->getROTransaction()), d_idx1.d_ix1 d_main, key};
return ret;
};
eiter1_t end()
{
return eiter1_t();
}
#endif