diff --git a/Makefile b/Makefile index 35fe343..b412a6b 100644 --- a/Makefile +++ b/Makefile @@ -1,16 +1,14 @@ -#LIBS=lmdb-LMDB_0.9.22/libraries/liblmdb/liblmdb.a -#INCLUDES=-Ilmdb-LMDB_0.9.22/libraries/liblmdb +#LIBS=lmdb-LMDB_0.9.23/libraries/liblmdb/liblmdb.a +#INCLUDES=-Ilmdb-LMDB_0.9.23/libraries/liblmdb LIBS=-llmdb CXXVERSIONFLAG= -std=gnu++11 -CXXFLAGS:= $(CXXVERSIONFLAG) -Wall -O2 -MMD -MP -ggdb -pthread $(INCLUDES) # -fsanitize=address -fno-omit-frame-pointer +CXXFLAGS:= $(CXXVERSIONFLAG) -Wall -O2 -MMD -MP -ggdb -pthread $(INCLUDES) -Iext # -fsanitize=address -fno-omit-frame-pointer CFLAGS:= -Wall -O2 -MMD -MP -ggdb - - -PROGRAMS = lmdb-test basic-example scale-example multi-example rel-example \ - resize-example typed-example +PROGRAMS = lmdb-various basic-example scale-example multi-example rel-example \ + resize-example typed-example test-basic lmdb-view all: $(PROGRAMS) @@ -19,9 +17,14 @@ clean: -include *.d +test-basic: test-basic.o lmdb-safe.o + g++ $(CXXVERSIONFLAG) $^ -o $@ -pthread $(LIBS) -lmdb-test: lmdb-test.o lmdb-safe.o - g++ $(CXXVERSIONFLAG) $^ -o $@ -pthread $(LIBS) #-lasan +lmdb-various: lmdb-various.o lmdb-safe.o + g++ $(CXXVERSIONFLAG) $^ -o $@ -pthread $(LIBS) + +lmdb-view: lmdb-view.o lmdb-safe.o + g++ $(CXXVERSIONFLAG) $^ -o $@ $(LIBS) basic-example: basic-example.o lmdb-safe.o g++ $(CXXVERSIONFLAG) $^ -o $@ -pthread $(LIBS) diff --git a/lmdb-safe.cc b/lmdb-safe.cc index 6c427e5..eea5864 100644 --- a/lmdb-safe.cc +++ b/lmdb-safe.cc @@ -17,7 +17,7 @@ MDBDbi::MDBDbi(MDB_env* env, MDB_txn* txn, const string_view dbname, int flags) { // A transaction that uses this function must finish (either commit or abort) before any other transaction in the process may use this function. - int rc = mdb_dbi_open(txn, &dbname[0], flags, &d_dbi); + int rc = mdb_dbi_open(txn, dbname.empty() ? 0 : &dbname[0], flags, &d_dbi); if(rc) throw std::runtime_error("Unable to open named database: " + MDBError(rc)); @@ -139,13 +139,17 @@ MDBDbi MDBEnv::openDB(const string_view dbname, int flags) if(!(envflags & MDB_RDONLY)) { auto rwt = getRWTransaction(); - MDBDbi ret = rwt.openDB(&dbname[0], flags); + MDBDbi ret = rwt.openDB(dbname, flags); rwt.commit(); return ret; } - - auto rwt = getROTransaction(); - return rwt.openDB(dbname, flags); + + MDBDbi ret; + { + auto rwt = getROTransaction(); + ret = rwt.openDB(dbname, flags); + } + return ret; } MDBRWTransaction::MDBRWTransaction(MDBEnv* parent, int flags) : d_parent(parent) diff --git a/lmdb-safe.hh b/lmdb-safe.hh index 35b1eb9..448e73d 100644 --- a/lmdb-safe.hh +++ b/lmdb-safe.hh @@ -259,7 +259,7 @@ public: // this is something you can do, readonly MDBDbi openDB(string_view dbname, int flags) { - return MDBDbi(d_parent->d_env, d_txn, dbname, flags); + return MDBDbi( d_parent->d_env, d_txn, dbname, flags); } MDBROCursor getCursor(const MDBDbi&); @@ -444,7 +444,7 @@ public: return rc; } - MDBDbi openDB(const char* dbname, int flags) + MDBDbi openDB(string_view dbname, int flags) { return MDBDbi(d_parent->d_env, d_txn, dbname, flags); } diff --git a/lmdb-typed.cc b/lmdb-typed.cc index 28849f1..95206e3 100644 --- a/lmdb-typed.cc +++ b/lmdb-typed.cc @@ -1,6 +1,5 @@ #include "lmdb-typed.hh" - unsigned int MDBGetMaxID(MDBRWTransaction& txn, MDBDbi& dbi) { auto cursor = txn.getCursor(dbi); diff --git a/lmdb-typed.hh b/lmdb-typed.hh index 6fd768f..8870bfa 100644 --- a/lmdb-typed.hh +++ b/lmdb-typed.hh @@ -53,11 +53,41 @@ template void serFromString(const std::string& str, T& ret) { ret = T(); + + boost::iostreams::array_source source(str.c_str(), str.size()); + boost::iostreams::stream stream(source); + boost::archive::binary_iarchive in_archive(stream, boost::archive::no_header|boost::archive::no_codecvt); + in_archive >> ret; + + /* std::istringstream istr{str}; boost::archive::binary_iarchive oi(istr,boost::archive::no_header|boost::archive::no_codecvt ); oi >> ret; + */ } +template +std::string keyConv(const KeyType& t); + + +template<> +inline std::string keyConv(const std::string& t) +{ + return t; +} + +template<> +inline std::string keyConv(const uint32_t& t) +{ + return std::string((char*)&t, sizeof(4)); +} +template<> +inline std::string keyConv(const int32_t& t) +{ + return std::string((char*)&t, sizeof(4)); +} + + /** This is a struct that implements index operations, but only the operations that are broadcast to all indexes. Specifically, to deal with databases with less than the maximum @@ -72,14 +102,16 @@ template struct LMDBIndexOps { explicit LMDBIndexOps(Parent* parent) : d_parent(parent){} - void put(MDBRWTransaction& txn, const Class& t, uint32_t id) + void put(MDBRWTransaction& txn, const Class& t, uint32_t id, int flags=0) { - txn.put(d_idx, d_parent->getMember(t), id); + txn.put(d_idx, keyConv(d_parent->getMember(t)), id, flags); } void del(MDBRWTransaction& txn, const Class& t, uint32_t id) { - txn.del(d_idx, d_parent->getMember(t), id); + if(int rc = txn.del(d_idx, keyConv(d_parent->getMember(t)), id)) { + throw std::runtime_error("Error deleting from index: " + std::string(mdb_strerror(rc))); + } } void openDB(std::shared_ptr& env, string_view str, int flags) @@ -124,7 +156,7 @@ struct index_on_function : LMDBIndexOps - void put(MDBRWTransaction& txn, const Class& t, uint32_t id) + void put(MDBRWTransaction& txn, const Class& t, uint32_t id, int flags=0) {} template void del(MDBRWTransaction& txn, const Class& t, uint32_t id) @@ -137,6 +169,7 @@ struct nullindex_t typedef uint32_t type; // dummy }; + /** The main class. Templatized only on the indexes and typename right now */ template class TypedDBI @@ -156,9 +189,9 @@ public: openMacro(2); openMacro(3); #undef openMacro - } + // we get a lot of our smarts from this tuple, it enables get<0> etc typedef std::tuple tuple_t; tuple_t d_tuple; @@ -204,7 +237,7 @@ public: uint32_t get(const typename std::tuple_element::type::type& key, T& out) { MDBOutVal id; - if(!d_parent.d_txn->get(std::get(d_parent.d_parent->d_tuple).d_idx, key, id)) { + if(!d_parent.d_txn->get(std::get(d_parent.d_parent->d_tuple).d_idx, keyConv(key), id)) { if(get(id.get(), out)) return id.get(); } @@ -261,7 +294,7 @@ public: serFromString(d_id.get(), d_t); } - + std::function filter; void del() { d_cursor.del(); @@ -291,19 +324,30 @@ public: iter_t& genoperator(MDB_cursor_op dupop, MDB_cursor_op op) { MDBOutVal data; - int rc = d_cursor.get(d_key, d_id, d_one_key ? dupop : op); + int rc; + next:; + rc = d_cursor.get(d_key, d_id, d_one_key ? dupop : op); if(rc == MDB_NOTFOUND) { d_end = true; } + else if(rc) { + throw std::runtime_error("in genoperator, " + std::string(mdb_strerror(rc))); + } else { if(d_on_index) { if(d_parent->d_txn->get(d_parent->d_parent->d_main, d_id, data)) throw std::runtime_error("Missing id field"); + if(filter && !filter(data)) + goto next; serFromString(data.get(), d_t); } - else + else { + if(filter && !filter(data)) + goto next; + serFromString(d_id.get(), d_t); + } } return *this; } @@ -339,20 +383,32 @@ public: }; template - iter_t begin() + iter_t genbegin(MDB_cursor_op op) { typename Parent::cursor_t cursor = d_parent.d_txn->getCursor(std::get(d_parent.d_parent->d_tuple).d_idx); MDBOutVal out, id; - if(cursor.get(out, id, MDB_FIRST)) -{ // on_index, one_key, end + if(cursor.get(out, id, op)) { + // on_index, one_key, end return iter_t{&d_parent, std::move(cursor), true, false, true}; } return iter_t{&d_parent, std::move(cursor), true, false}; }; + template + iter_t begin() + { + return genbegin(MDB_FIRST); + } + + template + iter_t rbegin() + { + return genbegin(MDB_LAST); + } + iter_t begin() { typename Parent::cursor_t cursor = d_parent.d_txn->getCursor(d_parent.d_parent->d_main); @@ -377,8 +433,9 @@ public: iter_t genfind(const typename std::tuple_element::type::type& key, MDB_cursor_op op) { typename Parent::cursor_t cursor = d_parent.d_txn->getCursor(std::get(d_parent.d_parent->d_tuple).d_idx); - - MDBInVal in(key); + + std::string keystr = keyConv(key); + MDBInVal in(keystr); MDBOutVal out, id; out.d_mdbval = in.d_mdbval; @@ -408,8 +465,9 @@ public: std::pair equal_range(const typename std::tuple_element::type::type& key) { typename Parent::cursor_t cursor = d_parent.d_txn->getCursor(std::get(d_parent.d_parent->d_tuple).d_idx); - - MDBInVal in(key); + + std::string keyString=keyConv(key); + MDBInVal in(keyString); MDBOutVal out, id; out.d_mdbval = in.d_mdbval; @@ -478,9 +536,12 @@ public: // insert something, with possibly a specific id uint32_t put(const T& t, uint32_t id=0) { - if(!id) + int flags = 0; + if(!id) { id = MDBGetMaxID(*d_txn, d_parent->d_main) + 1; - d_txn->put(d_parent->d_main, id, serToString(t)); + flags = MDB_APPEND; + } + d_txn->put(d_parent->d_main, id, serToString(t), flags); #define insertMacro(N) std::get(d_parent->d_tuple).put(*d_txn, t, id); insertMacro(0); @@ -496,8 +557,8 @@ public: void modify(uint32_t id, std::function func) { T t; - if(!this->get(id, t)) - return; // XXX should be exception + if(!this->get(id, t)) + throw std::runtime_error("Could not modify id "+std::to_string(id)); func(t); del(id); // this is the lazy way. We could test for changed index fields diff --git a/lmdb-test.cc b/lmdb-various.cc similarity index 100% rename from lmdb-test.cc rename to lmdb-various.cc