diff --git a/lmdb-typed.cc b/lmdb-typed.cc index 6e99952..81f074d 100644 --- a/lmdb-typed.cc +++ b/lmdb-typed.cc @@ -1,4 +1,6 @@ #include "lmdb-typed.hh" +#include + #include unsigned int getMaxID(MDBRWTransaction& txn, MDBDbi& dbi) @@ -44,7 +46,8 @@ struct compound std::string operator()(const DNSResourceRecord& rr) { std::string ret; - ret.assign((char*)&rr.domain_id, 4); + uint32_t id = htonl(rr.domain_id); + ret.assign((char*)&id, 4); ret.append(rr.ordername); return ret; } @@ -69,7 +72,7 @@ int main() } cout<<"Iterating over name 'powerdns.com': "<("powerdns.com"); - for(auto& iter = range.first; iter != range.second; ++iter) + for(auto iter = std::move(range.first); iter != range.second; ++iter) { cout << iter->qname << " " << iter->qtype << " " <content <(c(rr3)); - cout <<"Found using compound index: "<qname<< " # " <ordername<(c(rr3)); + cout <<"Found for '"<qname<< " # '" <ordername<< "'"<qname<< " # " <ordername<qname<< " # '" <ordername<<"'"< std::string serToString(const T& t) { - std::string serial_str; boost::iostreams::back_insert_device inserter(serial_str); boost::iostreams::stream > s(inserter); @@ -54,47 +57,15 @@ void serFromString(const std::string& str, T& ret) oi >> ret; } +/** This is a struct that implements index operations, but + only the operations that are broadcast to all indexes. + Specifically, to deal with databases with less than the maximum + number of interfaces, this only includes calls that should be + ignored for empty indexes. -/* This is for storing a struct that has to be found using several - of its fields. - - We want a typed lmdb database that we can only: - * insert such structs - * remove them - * mutate them - - All while maintaining indexes on insert, removal and mutation. - - struct DNSResourceRecord - { - string qname; // index - string qtype; - uint32_t domain_id; // index - string content; - string ordername; // index - bool auth; - } - - TypedDBI tdbi; - - DNSResourceRecord rr; - uint32_t id = tdbi.insert(rr); // inserts, creates three index items - - tdbi.modify(id, [](auto& rr) { rr.auth=false; }); - - DNSResourceRecord retrr; - uint32_t id = tdbi.get<1>(qname, retrr); - - // this checks for changes and updates indexes if need be - tdbi.modify(id, [](auto& rr) { rr.ordername="blah"; }); -*/ - -// this only needs methods that must happen for all indexes at once -// so specifically, not size or get, people ask for those themselves, and -// should no do that on indexes that don't exist + this only needs methods that must happen for all indexes at once + so specifically, not size or get, people ask for those themselves, and + should no do that on indexes that don't exist */ template struct LMDBIndexOps @@ -118,6 +89,8 @@ struct LMDBIndexOps Parent* d_parent; }; +/** This is an index on a field in a struct, it derives from the LMDBIndexOps */ + template struct index_on : LMDBIndexOps> { @@ -131,6 +104,7 @@ struct index_on : LMDBIndexOps> typedef Type type; }; +/** This is a calculated index */ template struct index_on_function : LMDBIndexOps > { @@ -145,6 +119,7 @@ struct index_on_function : LMDBIndexOps @@ -161,6 +136,7 @@ struct nullindex_t typedef uint32_t type; // dummy }; +/** The main class. Templatized only on the indexes and typename right now */ template class TypedDBI { @@ -169,7 +145,10 @@ public: : d_env(env), d_name(name) { d_main = d_env->openDB(name, MDB_CREATE | MDB_INTEGERKEY); - + + // now you might be tempted to go all MPL on this so we can get rid of the + // ugly macro. I'm not very receptive to that idea since it will make things + // EVEN uglier. #define openMacro(N) std::get(d_tuple).openDB(d_env, std::string(name)+"_"#N, MDB_CREATE | MDB_DUPFIXED | MDB_DUPSORT); openMacro(0); openMacro(1); @@ -179,15 +158,19 @@ public: } + // we get a lot of our smarts from this tuple, it enables get<0> etc typedef std::tuple tuple_t; tuple_t d_tuple; + // We support readonly and rw transactions. Here we put the Readonly operations + // which get sourced by both kinds of transactions template struct ReadonlyOperations { ReadonlyOperations(Parent& parent) : d_parent(parent) {} + //! Number of entries in main database uint32_t size() { MDB_stat stat; @@ -195,6 +178,7 @@ public: return stat.ms_entries; } + //! Number of entries in the various indexes - should be the same template uint32_t size() { @@ -202,7 +186,8 @@ public: mdb_stat(d_parent.d_txn, std::get(d_parent.d_parent->d_tuple).d_idx, &stat); return stat.ms_entries; } - + + //! Get item with id, from main table directly bool get(uint32_t id, T& t) { MDBOutVal data; @@ -213,6 +198,7 @@ public: return true; } + //! Get item through index N, then via the main database template uint32_t get(const typename std::tuple_element::type::type& key, T& out) { @@ -222,7 +208,7 @@ public: return 0; } - + //! Cardinality of index N template uint32_t cardinality() { @@ -237,21 +223,22 @@ public: return count; } + //! End iderator type struct eiter_t {}; // can be on main, or on an index // when on main, return data directly // when on index, indirect - // we can be limited to one key, or iterate over entire table - // + // we can be limited to one key, or iterate over entire database + // iter requires you to put the cursor in the right place first! struct iter_t { explicit iter_t(Parent* parent, typename Parent::cursor_t&& cursor, bool on_index, bool one_key, bool end=false) : d_parent(parent), d_cursor(std::move(cursor)), - d_on_index(on_index), - d_one_key(one_key), + d_on_index(on_index), // is this an iterator on main database or on index? + d_one_key(one_key), // should we stop at end of key? (equal range) d_end(end) { if(d_end) @@ -291,7 +278,8 @@ public: { return &d_t; } - + + // implements generic ++ or -- iter_t& genoperator(MDB_cursor_op dupop, MDB_cursor_op op) { MDBOutVal data; @@ -320,7 +308,8 @@ public: { return genoperator(MDB_PREV_DUP, MDB_PREV); } - + + // get ID this iterator points to uint32_t getID() { if(d_on_index) @@ -328,9 +317,12 @@ public: else return d_key.get(); } - + + // transaction we are part of Parent* d_parent; typename Parent::cursor_t d_cursor; + + // gcc complains if I don't zero-init these, which is worrying XXX MDBOutVal d_key{0,0}, d_data{0,0}, d_id{0,0}; bool d_on_index; bool d_one_key; @@ -367,9 +359,14 @@ public: return iter_t{&d_parent, std::move(cursor), false, false}; }; - + eiter_t end() + { + return eiter_t(); + } + + // basis for find, lower_bound template - iter_t find(const typename std::tuple_element::type::type& key) + iter_t genfind(const typename std::tuple_element::type::type& key, MDB_cursor_op op) { typename Parent::cursor_t cursor = d_parent.d_txn.getCursor(std::get(d_parent.d_parent->d_tuple).d_idx); @@ -377,7 +374,7 @@ public: MDBOutVal out, id; out.d_mdbval = in.d_mdbval; - if(cursor.get(out, id, MDB_SET)) { + if(cursor.get(out, id, op)) { // on_index, one_key, end return iter_t{&d_parent, std::move(cursor), true, false, true}; } @@ -385,6 +382,20 @@ public: return iter_t{&d_parent, std::move(cursor), true, false}; }; + template + iter_t find(const typename std::tuple_element::type::type& key) + { + return genfind(key, MDB_SET); + } + + template + iter_t lower_bound(const typename std::tuple_element::type::type& key) + { + return genfind(key, MDB_SET_RANGE); + } + + + //! equal range - could possibly be expressed through genfind template std::pair equal_range(const typename std::tuple_element::type::type& key) { @@ -402,12 +413,6 @@ public: return {iter_t{&d_parent, std::move(cursor), true, true}, eiter_t()}; }; - - eiter_t end() - { - return eiter_t(); - } - Parent& d_parent; }; @@ -446,7 +451,7 @@ public: rhs.d_parent = 0; } - + // insert something, with possibly a specific id uint32_t insert(const T& t, uint32_t id=0) { if(!id) @@ -463,6 +468,7 @@ public: return id; } + // modify an item 'in place', plus update indexes void modify(uint32_t id, std::function func) { T t; @@ -474,7 +480,7 @@ public: insert(t, id); } - + //! delete an item, and from indexes void del(uint32_t id) { T t; @@ -485,6 +491,7 @@ public: clearIndex(id, t); } + //! clear database & indexes (by hand!) void clear() { auto cursor = d_txn.getCursor(d_parent->d_main); @@ -498,22 +505,23 @@ public: cursor.del(); } } - + + //! commit this transaction void commit() { d_txn.commit(); } + //! abort this transaction void abort() { d_txn.abort(); } - typedef MDBRWCursor cursor_t; - private: + // clear this ID from all indexes void clearIndex(uint32_t id, const T& t) { #define clearMacro(N) std::get(d_parent->d_tuple).del(d_txn, t, id); @@ -528,12 +536,14 @@ public: TypedDBI* d_parent; MDBRWTransaction d_txn; }; - + + //! Get an RW transaction RWTransaction getRWTransaction() { return RWTransaction(this); } + //! Get an RO transaction ROTransaction getROTransaction() { return ROTransaction(this);