#pragma once #include #include "lmdb-safe.hh" #include #include #include #include #include #include #include #include #include using std::cout; using std::endl; /* Open issues: Everything should go into a namespace What is an error? What is an exception? could id=0 be magic? ('no such id') Composite keys (powerdns needs them) functional keys Insert? Put? Naming matters Is boost the best serializer? rename find to equal_range Perhaps use the separate index concept from multi_index A dump function would be nice (typed) */ unsigned int getMaxID(MDBRWTransaction& txn, MDBDbi& dbi); template std::string serToString(const T& t) { std::string serial_str; boost::iostreams::back_insert_device inserter(serial_str); boost::iostreams::stream > s(inserter); boost::archive::binary_oarchive oa(s, boost::archive::no_header | boost::archive::no_codecvt); oa << t; return serial_str; } template void serFromString(const std::string& str, T& ret) { ret = T(); std::istringstream istr{str}; boost::archive::binary_iarchive oi(istr,boost::archive::no_header|boost::archive::no_codecvt ); oi >> ret; } /* This is for storing a struct that has to be found using several of its fields. We want a typed lmdb database that we can only: * insert such structs * remove them * mutate them All while maintaining indexes on insert, removal and mutation. struct DNSResourceRecord { string qname; // index string qtype; uint32_t domain_id; // index string content; string ordername; // index bool auth; } TypedDBI tdbi; DNSResourceRecord rr; uint32_t id = tdbi.insert(rr); // inserts, creates three index items tdbi.modify(id, [](auto& rr) { rr.auth=false; }); DNSResourceRecord retrr; uint32_t id = tdbi.get<1>(qname, retrr); // this checks for changes and updates indexes if need be tdbi.modify(id, [](auto& rr) { rr.ordername="blah"; }); */ // this only needs methods that must happen for all indexes at once // so specifically, not size or get, people ask for those themselves, and // should no do that on indexes that don't exist template struct LMDBIndexOps { explicit LMDBIndexOps(Parent* parent) : d_parent(parent){} void put(MDBRWTransaction& txn, const Class& t, uint32_t id) { txn.put(d_idx, d_parent->getMember(t), id); } void del(MDBRWTransaction& txn, const Class& t, uint32_t id) { txn.del(d_idx, d_parent->getMember(t), id); } void openDB(std::shared_ptr& env, string_view str, int flags) { d_idx = env->openDB(str, flags); } MDBDbi d_idx; Parent* d_parent; }; template struct index_on : LMDBIndexOps> { index_on() : LMDBIndexOps>(this) {} static Type getMember(const Class& c) { return c.*PtrToMember; } typedef Type type; }; template struct index_on_function : LMDBIndexOps > { index_on_function() : LMDBIndexOps >(this) {} static Type getMember(const Class& c) { Func f; return f(c); } typedef Type type; }; struct nullindex_t { template void put(MDBRWTransaction& txn, const Class& t, uint32_t id) {} template void del(MDBRWTransaction& txn, const Class& t, uint32_t id) {} void openDB(std::shared_ptr& env, string_view str, int flags) { } typedef uint32_t type; // dummy }; template class TypedDBI { public: TypedDBI(std::shared_ptr env, string_view name) : d_env(env), d_name(name) { d_main = d_env->openDB(name, MDB_CREATE | MDB_INTEGERKEY); #define openMacro(N) std::get(d_tuple).openDB(d_env, std::string(name)+"_"#N, MDB_CREATE | MDB_DUPFIXED | MDB_DUPSORT); openMacro(0); openMacro(1); openMacro(2); openMacro(3); #undef openMacro } typedef std::tuple tuple_t; tuple_t d_tuple; template struct ReadonlyOperations { ReadonlyOperations(Parent& parent) : d_parent(parent) {} uint32_t size() { MDB_stat stat; mdb_stat(d_parent.d_txn, d_parent.d_parent->d_main, &stat); return stat.ms_entries; } template uint32_t size() { MDB_stat stat; mdb_stat(d_parent.d_txn, std::get(d_parent.d_parent->d_tuple).d_idx, &stat); return stat.ms_entries; } bool get(uint32_t id, T& t) { MDBOutVal data; if(d_parent.d_txn.get(d_parent.d_parent->d_main, id, data)) return false; serFromString(data.get(), t); return true; } template uint32_t get(const typename std::tuple_element::type::type& key, T& out) { MDBOutVal id; if(!d_parent.d_txn.get(std::get(d_parent.d_parent->d_tuple).d_idx, key, id)) return get(id.get(), out); return 0; } template uint32_t cardinality() { auto cursor = d_parent.d_txn.getCursor(std::get(d_parent.d_parent->d_tuple).d_idx); bool first = true; MDBOutVal key, data; uint32_t count = 0; while(!cursor.get(key, data, first ? MDB_FIRST : MDB_NEXT_NODUP)) { ++count; first=false; } return count; } struct eiter_t {}; // can be on main, or on an index // when on main, return data directly // when on index, indirect // we can be limited to one key, or iterate over entire table // struct iter_t { explicit iter_t(Parent* parent, typename Parent::cursor_t&& cursor, bool on_index, bool one_key, bool end=false) : d_parent(parent), d_cursor(std::move(cursor)), d_on_index(on_index), d_one_key(one_key), d_end(end) { if(d_end) return; if(d_cursor.get(d_key, d_id, MDB_GET_CURRENT)) { d_end = true; return; } if(d_on_index) { if(d_parent->d_txn.get(d_parent->d_parent->d_main, d_id, d_data)) throw std::runtime_error("Missing id in constructor"); serFromString(d_data.get(), d_t); } else serFromString(d_id.get(), d_t); } bool operator!=(const eiter_t& rhs) { return !d_end; } bool operator==(const eiter_t& rhs) { return d_end; } const T& operator*() { return d_t; } const T* operator->() { return &d_t; } iter_t& genoperator(MDB_cursor_op dupop, MDB_cursor_op op) { MDBOutVal data; int rc = d_cursor.get(d_key, d_id, d_one_key ? dupop : op); if(rc == MDB_NOTFOUND) { d_end = true; } else { if(d_on_index) { if(d_parent->d_txn.get(d_parent->d_parent->d_main, d_id, data)) throw std::runtime_error("Missing id field"); serFromString(data.get(), d_t); } else serFromString(d_id.get(), d_t); } return *this; } iter_t& operator++() { return genoperator(MDB_NEXT_DUP, MDB_NEXT); } iter_t& operator--() { return genoperator(MDB_PREV_DUP, MDB_PREV); } uint32_t getID() { if(d_on_index) return d_id.get(); else return d_key.get(); } Parent* d_parent; typename Parent::cursor_t d_cursor; MDBOutVal d_key{0,0}, d_data{0,0}, d_id{0,0}; bool d_on_index; bool d_one_key; bool d_end{false}; T d_t; }; template iter_t begin() { typename Parent::cursor_t cursor = d_parent.d_txn.getCursor(std::get(d_parent.d_parent->d_tuple).d_idx); MDBOutVal out, id; if(cursor.get(out, id, MDB_FIRST)) { // on_index, one_key, end return iter_t{&d_parent, std::move(cursor), true, false, true}; } return iter_t{&d_parent, std::move(cursor), true, false}; }; iter_t begin() { typename Parent::cursor_t cursor = d_parent.d_txn.getCursor(d_parent.d_parent->d_main); MDBOutVal out, id; if(cursor.get(out, id, MDB_FIRST)) { // on_index, one_key, end return iter_t{&d_parent, std::move(cursor), false, false, true}; } return iter_t{&d_parent, std::move(cursor), false, false}; }; template iter_t find(const typename std::tuple_element::type::type& key) { typename Parent::cursor_t cursor = d_parent.d_txn.getCursor(std::get(d_parent.d_parent->d_tuple).d_idx); MDBInVal in(key); MDBOutVal out, id; out.d_mdbval = in.d_mdbval; if(cursor.get(out, id, MDB_SET)) { // on_index, one_key, end return iter_t{&d_parent, std::move(cursor), true, false, true}; } return iter_t{&d_parent, std::move(cursor), true, false}; }; template std::pair equal_range(const typename std::tuple_element::type::type& key) { typename Parent::cursor_t cursor = d_parent.d_txn.getCursor(std::get(d_parent.d_parent->d_tuple).d_idx); MDBInVal in(key); MDBOutVal out, id; out.d_mdbval = in.d_mdbval; if(cursor.get(out, id, MDB_SET)) { // on_index, one_key, end return {iter_t{&d_parent, std::move(cursor), true, true, true}, eiter_t()}; } return {iter_t{&d_parent, std::move(cursor), true, true}, eiter_t()}; }; eiter_t end() { return eiter_t(); } Parent& d_parent; }; class ROTransaction : public ReadonlyOperations { public: explicit ROTransaction(TypedDBI* parent) : ReadonlyOperations(*this), d_parent(parent), d_txn(d_parent->d_env->getROTransaction()) { } ROTransaction(ROTransaction&& rhs) : ReadonlyOperations(*this), d_parent(rhs.d_parent),d_txn(std::move(rhs.d_txn)) { rhs.d_parent = 0; } typedef MDBROCursor cursor_t; TypedDBI* d_parent; MDBROTransaction d_txn; }; class RWTransaction : public ReadonlyOperations { public: explicit RWTransaction(TypedDBI* parent) : ReadonlyOperations(*this), d_parent(parent), d_txn(d_parent->d_env->getRWTransaction()) { } RWTransaction(RWTransaction&& rhs) : ReadonlyOperations(*this), d_parent(rhs.d_parent), d_txn(std::move(rhs.d_txn)) { rhs.d_parent = 0; } uint32_t insert(const T& t, uint32_t id=0) { if(!id) id = getMaxID(d_txn, d_parent->d_main) + 1; d_txn.put(d_parent->d_main, id, serToString(t)); #define insertMacro(N) std::get(d_parent->d_tuple).put(d_txn, t, id); insertMacro(0); insertMacro(1); insertMacro(2); insertMacro(3); #undef insertMacro return id; } void modify(uint32_t id, std::function func) { T t; if(!this->get(id, t)) return; // XXX should be exception func(t); del(id); // this is the lazy way. We could test for changed index fields insert(t, id); } void del(uint32_t id) { T t; if(!this->get(id, t)) return; d_txn.del(d_parent->d_main, id); clearIndex(id, t); } void clear() { auto cursor = d_txn.getCursor(d_parent->d_main); bool first = true; MDBOutVal key, data; while(!cursor.get(key, data, first ? MDB_FIRST : MDB_NEXT)) { first = false; T t; serFromString(data.get(), t); clearIndex(key.get(), t); cursor.del(); } } void commit() { d_txn.commit(); } void abort() { d_txn.abort(); } typedef MDBRWCursor cursor_t; private: void clearIndex(uint32_t id, const T& t) { #define clearMacro(N) std::get(d_parent->d_tuple).del(d_txn, t, id); clearMacro(0); clearMacro(1); clearMacro(2); clearMacro(3); #undef clearMacro } public: TypedDBI* d_parent; MDBRWTransaction d_txn; }; RWTransaction getRWTransaction() { return RWTransaction(this); } ROTransaction getROTransaction() { return ROTransaction(this); } private: std::shared_ptr d_env; MDBDbi d_main; std::string d_name; };