2018-12-07 13:52:17 +01:00
|
|
|
#include <lmdb.h>
|
|
|
|
#include <iostream>
|
|
|
|
#include <fstream>
|
|
|
|
#include <set>
|
2018-12-07 18:17:03 +01:00
|
|
|
#include <map>
|
|
|
|
#include <thread>
|
|
|
|
#include <memory>
|
2018-12-08 14:08:26 +01:00
|
|
|
#include <mutex>
|
2018-12-07 13:52:17 +01:00
|
|
|
|
|
|
|
using namespace std;
|
|
|
|
|
|
|
|
/* open issues:
|
|
|
|
*
|
|
|
|
* - missing convenience functions (string_view, string)
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
The error strategy. Anything that "should never happen" turns into an exception. But things like 'duplicate entry' or 'no such key' are for you to deal with.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
Thread safety: we are as safe as lmdb. You can talk to MDBEnv from as many threads as you want
|
|
|
|
*/
|
|
|
|
|
2018-12-08 14:08:26 +01:00
|
|
|
/** MDBDbi is our only 'value type' object, as 1) a dbi is actually an integer
|
|
|
|
and 2) per LMDB documentation, we never close it. */
|
2018-12-07 13:52:17 +01:00
|
|
|
class MDBDbi
|
|
|
|
{
|
|
|
|
public:
|
2018-12-08 14:08:26 +01:00
|
|
|
explicit MDBDbi(MDB_env* env, MDB_txn* txn, const char* dbname, int flags);
|
2018-12-07 13:52:17 +01:00
|
|
|
|
|
|
|
operator const MDB_dbi&() const
|
|
|
|
{
|
|
|
|
return d_dbi;
|
|
|
|
}
|
|
|
|
|
|
|
|
MDB_dbi d_dbi;
|
|
|
|
};
|
|
|
|
|
|
|
|
class MDBRWTransaction;
|
|
|
|
class MDBROTransaction;
|
|
|
|
|
|
|
|
class MDBEnv
|
|
|
|
{
|
|
|
|
public:
|
2018-12-10 10:27:07 +01:00
|
|
|
MDBEnv(const char* fname, int flags, int mode);
|
2018-12-07 13:52:17 +01:00
|
|
|
|
|
|
|
~MDBEnv()
|
|
|
|
{
|
2018-12-10 10:27:07 +01:00
|
|
|
for(auto& a : d_RWtransactionsOut) {
|
|
|
|
if(a.second)
|
|
|
|
cout << "thread " <<a.first<<" had "<<a.second<<" RW transactions open"<<endl;
|
|
|
|
}
|
|
|
|
for(auto& a : d_ROtransactionsOut) {
|
|
|
|
if(a.second)
|
|
|
|
cout << "thread " <<a.first<<" had "<<a.second<<" RO transactions open"<<endl;
|
|
|
|
}
|
|
|
|
|
2018-12-07 13:52:17 +01:00
|
|
|
// Only a single thread may call this function. All transactions, databases, and cursors must already be closed before calling this function
|
|
|
|
mdb_env_close(d_env);
|
|
|
|
// but, elsewhere, docs say database handles do not need to be closed?
|
|
|
|
}
|
|
|
|
|
|
|
|
MDBDbi openDB(const char* dbname, int flags);
|
|
|
|
|
|
|
|
MDBRWTransaction getRWTransaction();
|
|
|
|
MDBROTransaction getROTransaction();
|
|
|
|
|
|
|
|
operator MDB_env*& ()
|
|
|
|
{
|
|
|
|
return d_env;
|
|
|
|
}
|
|
|
|
MDB_env* d_env;
|
2018-12-08 14:08:26 +01:00
|
|
|
|
|
|
|
int getRWTX();
|
|
|
|
void incRWTX();
|
|
|
|
void decRWTX();
|
|
|
|
int getROTX();
|
|
|
|
void incROTX();
|
|
|
|
void decROTX();
|
|
|
|
private:
|
2018-12-08 20:58:19 +01:00
|
|
|
std::mutex d_openmut;
|
|
|
|
std::mutex d_countmutex;
|
2018-12-07 18:17:03 +01:00
|
|
|
std::map<std::thread::id, int> d_RWtransactionsOut;
|
|
|
|
std::map<std::thread::id, int> d_ROtransactionsOut;
|
2018-12-07 13:52:17 +01:00
|
|
|
};
|
|
|
|
|
2018-12-10 10:27:07 +01:00
|
|
|
std::shared_ptr<MDBEnv> getMDBEnv(const char* fname, int flags, int mode);
|
2018-12-07 18:17:03 +01:00
|
|
|
|
2018-12-07 13:52:17 +01:00
|
|
|
class MDBROCursor;
|
|
|
|
|
|
|
|
class MDBROTransaction
|
|
|
|
{
|
|
|
|
public:
|
2018-12-10 10:27:07 +01:00
|
|
|
explicit MDBROTransaction(MDBEnv* parent, int flags=0);
|
2018-12-07 13:52:17 +01:00
|
|
|
|
|
|
|
MDBROTransaction(MDBROTransaction&& rhs)
|
|
|
|
{
|
|
|
|
d_parent = rhs.d_parent;
|
|
|
|
d_txn = rhs.d_txn;
|
|
|
|
rhs.d_parent = 0;
|
|
|
|
rhs.d_txn = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
void reset()
|
|
|
|
{
|
|
|
|
// this does not free cursors
|
|
|
|
mdb_txn_reset(d_txn);
|
2018-12-08 14:08:26 +01:00
|
|
|
d_parent->decROTX();
|
2018-12-07 13:52:17 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
void renew()
|
|
|
|
{
|
2018-12-08 14:08:26 +01:00
|
|
|
if(d_parent->getROTX())
|
2018-12-07 13:52:17 +01:00
|
|
|
throw std::runtime_error("Duplicate transaction");
|
|
|
|
if(mdb_txn_renew(d_txn))
|
|
|
|
throw std::runtime_error("Renewing transaction");
|
2018-12-08 14:08:26 +01:00
|
|
|
d_parent->incROTX();
|
2018-12-07 13:52:17 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
int get(MDB_dbi dbi, const MDB_val& key, MDB_val& val)
|
|
|
|
{
|
2018-12-08 14:08:26 +01:00
|
|
|
if(!d_txn)
|
|
|
|
throw std::runtime_error("Attempt to use a closed RO transaction for get");
|
|
|
|
|
2018-12-08 20:58:19 +01:00
|
|
|
int rc = mdb_get(d_txn, dbi, (MDB_val*)&key, &val);
|
|
|
|
if(rc && rc != MDB_NOTFOUND)
|
|
|
|
throw std::runtime_error("getting data: " + std::string(mdb_strerror(rc)));
|
|
|
|
|
|
|
|
return rc;
|
2018-12-07 13:52:17 +01:00
|
|
|
}
|
2018-12-08 14:08:26 +01:00
|
|
|
int get(MDB_dbi dbi, string_view key, string_view& val);
|
|
|
|
|
2018-12-07 13:52:17 +01:00
|
|
|
|
|
|
|
// this is something you can do, readonly
|
|
|
|
MDBDbi openDB(const char* dbname, int flags)
|
|
|
|
{
|
|
|
|
return MDBDbi(d_parent->d_env, d_txn, dbname, flags);
|
|
|
|
}
|
|
|
|
|
|
|
|
MDBROCursor getCursor(const MDBDbi&);
|
2018-12-07 14:16:21 +01:00
|
|
|
|
2018-12-07 13:52:17 +01:00
|
|
|
~MDBROTransaction()
|
|
|
|
{
|
|
|
|
if(d_txn) {
|
2018-12-08 14:08:26 +01:00
|
|
|
d_parent->decROTX();
|
2018-12-07 14:16:21 +01:00
|
|
|
mdb_txn_commit(d_txn); // this appears to work better than abort for r/o database opening
|
2018-12-07 13:52:17 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
operator MDB_txn*&()
|
|
|
|
{
|
|
|
|
return d_txn;
|
|
|
|
}
|
|
|
|
|
|
|
|
MDBEnv* d_parent;
|
|
|
|
MDB_txn* d_txn;
|
|
|
|
};
|
|
|
|
|
|
|
|
/*
|
|
|
|
A cursor in a read-only transaction must be closed explicitly, before or after its transaction ends. It can be reused with mdb_cursor_renew() before finally closing it.
|
|
|
|
|
|
|
|
"If the parent transaction commits, the cursor must not be used again."
|
|
|
|
*/
|
|
|
|
|
|
|
|
class MDBROCursor
|
|
|
|
{
|
|
|
|
public:
|
|
|
|
MDBROCursor(MDBROTransaction* parent, const MDB_dbi& dbi) : d_parent(parent)
|
|
|
|
{
|
|
|
|
int rc= mdb_cursor_open(d_parent->d_txn, dbi, &d_cursor);
|
|
|
|
if(rc) {
|
2018-12-07 14:16:21 +01:00
|
|
|
throw std::runtime_error("Error creating RO cursor: "+std::string(mdb_strerror(rc)));
|
2018-12-07 13:52:17 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
MDBROCursor(MDBROCursor&& rhs)
|
|
|
|
{
|
|
|
|
d_cursor = rhs.d_cursor;
|
|
|
|
rhs.d_cursor=0;
|
|
|
|
}
|
|
|
|
|
|
|
|
void close()
|
|
|
|
{
|
|
|
|
mdb_cursor_close(d_cursor);
|
|
|
|
d_cursor=0;
|
|
|
|
}
|
|
|
|
|
|
|
|
~MDBROCursor()
|
|
|
|
{
|
|
|
|
if(d_cursor)
|
|
|
|
mdb_cursor_close(d_cursor);
|
|
|
|
}
|
|
|
|
|
|
|
|
int get(MDB_val& key, MDB_val& data, MDB_cursor_op op)
|
|
|
|
{
|
|
|
|
return mdb_cursor_get(d_cursor, &key, &data, op);
|
|
|
|
}
|
2018-12-08 14:08:26 +01:00
|
|
|
|
2018-12-07 13:52:17 +01:00
|
|
|
MDB_cursor* d_cursor;
|
|
|
|
MDBROTransaction* d_parent;
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class MDBRWCursor;
|
|
|
|
|
|
|
|
class MDBRWTransaction
|
|
|
|
{
|
|
|
|
public:
|
2018-12-10 10:27:07 +01:00
|
|
|
explicit MDBRWTransaction(MDBEnv* parent, int flags=0);
|
2018-12-07 13:52:17 +01:00
|
|
|
|
|
|
|
MDBRWTransaction(MDBRWTransaction&& rhs)
|
|
|
|
{
|
|
|
|
d_parent = rhs.d_parent;
|
|
|
|
d_txn = rhs.d_txn;
|
|
|
|
rhs.d_parent = 0;
|
|
|
|
rhs.d_txn = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
MDBRWTransaction& operator=(MDBRWTransaction&& rhs)
|
|
|
|
{
|
|
|
|
if(d_txn)
|
|
|
|
abort();
|
|
|
|
|
|
|
|
d_parent = rhs.d_parent;
|
|
|
|
d_txn = rhs.d_txn;
|
|
|
|
rhs.d_parent = 0;
|
|
|
|
rhs.d_txn = 0;
|
|
|
|
|
|
|
|
return *this;
|
|
|
|
}
|
|
|
|
|
|
|
|
~MDBRWTransaction()
|
|
|
|
{
|
|
|
|
if(d_txn) {
|
2018-12-08 14:08:26 +01:00
|
|
|
d_parent->decRWTX();
|
2018-12-07 13:52:17 +01:00
|
|
|
closeCursors();
|
2018-12-08 14:08:26 +01:00
|
|
|
mdb_txn_abort(d_txn); // XXX check response?
|
2018-12-07 13:52:17 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
void closeCursors();
|
|
|
|
|
|
|
|
void commit()
|
|
|
|
{
|
|
|
|
closeCursors();
|
|
|
|
if(mdb_txn_commit(d_txn)) {
|
|
|
|
throw std::runtime_error("committing");
|
|
|
|
}
|
2018-12-08 14:08:26 +01:00
|
|
|
d_parent->decRWTX();
|
2018-12-07 13:52:17 +01:00
|
|
|
|
|
|
|
d_txn=0;
|
|
|
|
}
|
|
|
|
|
|
|
|
void abort()
|
|
|
|
{
|
|
|
|
closeCursors();
|
2018-12-08 14:08:26 +01:00
|
|
|
mdb_txn_abort(d_txn); // XXX check error?
|
2018-12-07 13:52:17 +01:00
|
|
|
d_txn = 0;
|
2018-12-08 14:08:26 +01:00
|
|
|
d_parent->decRWTX();
|
2018-12-07 13:52:17 +01:00
|
|
|
}
|
|
|
|
|
2018-12-09 14:37:08 +01:00
|
|
|
void clear(MDB_dbi dbi);
|
|
|
|
|
2018-12-08 14:08:26 +01:00
|
|
|
void put(MDB_dbi dbi, const MDB_val& key, const MDB_val& val, int flags=0)
|
2018-12-07 13:52:17 +01:00
|
|
|
{
|
2018-12-08 14:08:26 +01:00
|
|
|
if(!d_txn)
|
|
|
|
throw std::runtime_error("Attempt to use a closed RW transaction for put");
|
2018-12-07 13:52:17 +01:00
|
|
|
int rc;
|
|
|
|
if((rc=mdb_put(d_txn, dbi, (MDB_val*)&key, (MDB_val*)&val, flags)))
|
|
|
|
throw std::runtime_error("putting data: " + std::string(mdb_strerror(rc)));
|
|
|
|
}
|
|
|
|
|
2018-12-08 14:08:26 +01:00
|
|
|
void put(MDB_dbi dbi, string_view key, string_view val, int flags=0);
|
|
|
|
|
2018-12-07 13:52:17 +01:00
|
|
|
|
|
|
|
int del(MDB_dbi dbi, const MDB_val& key)
|
|
|
|
{
|
|
|
|
int rc;
|
|
|
|
rc=mdb_del(d_txn, dbi, (MDB_val*)&key, 0);
|
|
|
|
if(rc && rc != MDB_NOTFOUND)
|
|
|
|
throw std::runtime_error("deleting data: " + std::string(mdb_strerror(rc)));
|
|
|
|
return rc;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
int get(MDB_dbi dbi, const MDB_val& key, MDB_val& val)
|
|
|
|
{
|
2018-12-08 14:08:26 +01:00
|
|
|
if(!d_txn)
|
2018-12-10 10:27:07 +01:00
|
|
|
throw std::runtime_error("Attempt to use a closed RW transaction for get");
|
2018-12-08 14:08:26 +01:00
|
|
|
|
2018-12-08 20:58:19 +01:00
|
|
|
int rc = mdb_get(d_txn, dbi, (MDB_val*)&key, &val);
|
|
|
|
if(rc && rc != MDB_NOTFOUND)
|
|
|
|
throw std::runtime_error("getting data: " + std::string(mdb_strerror(rc)));
|
|
|
|
return rc;
|
2018-12-07 13:52:17 +01:00
|
|
|
}
|
|
|
|
|
2018-12-08 14:08:26 +01:00
|
|
|
|
|
|
|
int get(MDB_dbi dbi, string_view key, string_view& val);
|
2018-12-07 13:52:17 +01:00
|
|
|
|
|
|
|
MDBDbi openDB(const char* dbname, int flags)
|
|
|
|
{
|
|
|
|
return MDBDbi(d_parent->d_env, d_txn, dbname, flags);
|
|
|
|
}
|
|
|
|
|
|
|
|
MDBRWCursor getCursor(const MDBDbi&);
|
|
|
|
|
|
|
|
void reportCursor(MDBRWCursor* child)
|
|
|
|
{
|
|
|
|
d_cursors.insert(child);
|
|
|
|
}
|
2018-12-09 14:37:08 +01:00
|
|
|
void unreportCursor(MDBRWCursor* child)
|
|
|
|
{
|
|
|
|
d_cursors.erase(child);
|
|
|
|
}
|
|
|
|
|
2018-12-07 13:52:17 +01:00
|
|
|
void reportCursorMove(MDBRWCursor* from, MDBRWCursor* to)
|
|
|
|
{
|
|
|
|
d_cursors.erase(from);
|
|
|
|
d_cursors.insert(to);
|
|
|
|
}
|
|
|
|
|
|
|
|
operator MDB_txn*&()
|
|
|
|
{
|
|
|
|
return d_txn;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
std::set<MDBRWCursor*> d_cursors;
|
|
|
|
MDBEnv* d_parent;
|
|
|
|
MDB_txn* d_txn;
|
|
|
|
};
|
|
|
|
|
|
|
|
/* "A cursor in a write-transaction can be closed before its transaction ends, and will otherwise be closed when its transaction ends"
|
|
|
|
This is a problem for us since it may means we are closing the cursor twice, which is bad
|
|
|
|
*/
|
|
|
|
class MDBRWCursor
|
|
|
|
{
|
|
|
|
public:
|
|
|
|
MDBRWCursor(MDBRWTransaction* parent, const MDB_dbi& dbi) : d_parent(parent)
|
|
|
|
{
|
|
|
|
int rc= mdb_cursor_open(d_parent->d_txn, dbi, &d_cursor);
|
|
|
|
if(rc) {
|
2018-12-07 14:16:21 +01:00
|
|
|
throw std::runtime_error("Error creating RW cursor: "+std::string(mdb_strerror(rc)));
|
2018-12-07 13:52:17 +01:00
|
|
|
}
|
|
|
|
d_parent->reportCursor(this);
|
|
|
|
}
|
|
|
|
MDBRWCursor(MDBRWCursor&& rhs)
|
|
|
|
{
|
|
|
|
cout<<"Got move constructed, this was: "<<(void*)&rhs<<", now: "<<(void*)this<<endl;
|
|
|
|
d_parent = rhs.d_parent;
|
|
|
|
d_cursor = rhs.d_cursor;
|
|
|
|
rhs.d_cursor=0;
|
|
|
|
d_parent->reportCursorMove(&rhs, this);
|
|
|
|
}
|
|
|
|
|
|
|
|
void close()
|
|
|
|
{
|
|
|
|
if(d_cursor)
|
|
|
|
mdb_cursor_close(d_cursor);
|
|
|
|
d_cursor=0;
|
|
|
|
}
|
|
|
|
|
|
|
|
~MDBRWCursor()
|
|
|
|
{
|
|
|
|
if(d_cursor)
|
|
|
|
mdb_cursor_close(d_cursor);
|
2018-12-09 14:37:08 +01:00
|
|
|
d_parent->unreportCursor(this);
|
2018-12-07 13:52:17 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
int get(MDB_val& key, MDB_val& data, MDB_cursor_op op)
|
|
|
|
{
|
2018-12-09 14:37:08 +01:00
|
|
|
int rc = mdb_cursor_get(d_cursor, &key, &data, op);
|
|
|
|
if(rc && rc != MDB_NOTFOUND)
|
|
|
|
throw std::runtime_error("mdb_cursor_get: " + string(mdb_strerror(rc)));
|
|
|
|
return rc;
|
2018-12-07 13:52:17 +01:00
|
|
|
}
|
|
|
|
|
2018-12-08 14:08:26 +01:00
|
|
|
int put(MDB_val& key, MDB_val& data, int flags=0)
|
2018-12-07 13:52:17 +01:00
|
|
|
{
|
|
|
|
return mdb_cursor_put(d_cursor, &key, &data, flags);
|
|
|
|
}
|
|
|
|
|
|
|
|
int del(MDB_val& key, int flags)
|
|
|
|
{
|
|
|
|
return mdb_cursor_del(d_cursor, flags);
|
|
|
|
}
|
|
|
|
|
|
|
|
MDB_cursor* d_cursor;
|
|
|
|
MDBRWTransaction* d_parent;
|
|
|
|
};
|
|
|
|
|