2018-12-07 13:52:17 +01:00
# include "lmdb-safe.hh"
2018-12-07 18:17:03 +01:00
# include <fcntl.h>
# include <mutex>
# include <memory>
# include <sys/stat.h>
# include <string.h>
# include <map>
2018-12-10 22:08:49 +01:00
using namespace std ;
2018-12-08 14:08:26 +01:00
static string MDBError ( int rc )
{
return mdb_strerror ( rc ) ;
}
2021-12-05 19:28:08 +01:00
MDBDbi : : MDBDbi ( MDB_env * env , MDB_txn * txn , const string_view dbname , unsigned int flags )
2018-12-08 14:08:26 +01:00
{
2021-12-05 19:28:08 +01:00
( void ) env ;
2018-12-08 14:08:26 +01:00
// A transaction that uses this function must finish (either commit or abort) before any other transaction in the process may use this function.
2018-12-27 17:49:41 +01:00
int rc = mdb_dbi_open ( txn , dbname . empty ( ) ? 0 : & dbname [ 0 ] , flags , & d_dbi ) ;
2018-12-08 14:08:26 +01:00
if ( rc )
2018-12-08 20:58:19 +01:00
throw std : : runtime_error ( " Unable to open named database: " + MDBError ( rc ) ) ;
2018-12-08 14:08:26 +01:00
// Database names are keys in the unnamed database, and may be read but not written.
}
2021-12-05 19:28:08 +01:00
MDBEnv : : MDBEnv ( const char * fname , unsigned int flags , mdb_mode_t mode )
2018-12-08 14:08:26 +01:00
{
mdb_env_create ( & d_env ) ;
2018-12-14 23:00:44 +01:00
if ( mdb_env_set_mapsize ( d_env , 16ULL * 4096 * 244140ULL ) ) // 4GB
2018-12-08 14:08:26 +01:00
throw std : : runtime_error ( " setting map size " ) ;
/*
Various other options may also need to be set before opening the handle , e . g . mdb_env_set_mapsize ( ) , mdb_env_set_maxreaders ( ) , mdb_env_set_maxdbs ( ) ,
*/
mdb_env_set_maxdbs ( d_env , 128 ) ;
// we need MDB_NOTLS since we rely on its semantics
2018-12-10 10:27:07 +01:00
if ( int rc = mdb_env_open ( d_env , fname , flags | MDB_NOTLS , mode ) ) {
2018-12-08 14:08:26 +01:00
// If this function fails, mdb_env_close() must be called to discard the MDB_env handle.
mdb_env_close ( d_env ) ;
2018-12-08 20:58:19 +01:00
throw std : : runtime_error ( " Unable to open database file " + std : : string ( fname ) + " : " + MDBError ( rc ) ) ;
2018-12-08 14:08:26 +01:00
}
}
void MDBEnv : : incROTX ( )
{
2018-12-08 20:58:19 +01:00
std : : lock_guard < std : : mutex > l ( d_countmutex ) ;
2018-12-08 14:08:26 +01:00
+ + d_ROtransactionsOut [ std : : this_thread : : get_id ( ) ] ;
}
void MDBEnv : : decROTX ( )
{
2018-12-08 20:58:19 +01:00
std : : lock_guard < std : : mutex > l ( d_countmutex ) ;
2018-12-08 14:08:26 +01:00
- - d_ROtransactionsOut [ std : : this_thread : : get_id ( ) ] ;
}
void MDBEnv : : incRWTX ( )
{
2018-12-08 20:58:19 +01:00
std : : lock_guard < std : : mutex > l ( d_countmutex ) ;
2018-12-08 14:08:26 +01:00
+ + d_RWtransactionsOut [ std : : this_thread : : get_id ( ) ] ;
}
void MDBEnv : : decRWTX ( )
{
2018-12-08 20:58:19 +01:00
std : : lock_guard < std : : mutex > l ( d_countmutex ) ;
2018-12-08 14:08:26 +01:00
- - d_RWtransactionsOut [ std : : this_thread : : get_id ( ) ] ;
}
int MDBEnv : : getRWTX ( )
{
2018-12-08 20:58:19 +01:00
std : : lock_guard < std : : mutex > l ( d_countmutex ) ;
2018-12-08 14:08:26 +01:00
return d_RWtransactionsOut [ std : : this_thread : : get_id ( ) ] ;
}
int MDBEnv : : getROTX ( )
{
2018-12-08 20:58:19 +01:00
std : : lock_guard < std : : mutex > l ( d_countmutex ) ;
2018-12-08 14:08:26 +01:00
return d_ROtransactionsOut [ std : : this_thread : : get_id ( ) ] ;
}
2021-12-05 19:28:08 +01:00
std : : shared_ptr < MDBEnv > getMDBEnv ( const char * fname , unsigned int flags , mdb_mode_t mode )
2018-12-07 18:17:03 +01:00
{
struct Value
{
weak_ptr < MDBEnv > wp ;
2021-12-05 19:28:08 +01:00
unsigned int flags ;
2018-12-07 18:17:03 +01:00
} ;
static std : : map < tuple < dev_t , ino_t > , Value > s_envs ;
static std : : mutex mut ;
struct stat statbuf ;
if ( stat ( fname , & statbuf ) ) {
if ( errno ! = ENOENT )
throw std : : runtime_error ( " Unable to stat prospective mdb database: " + string ( strerror ( errno ) ) ) ;
else {
std : : lock_guard < std : : mutex > l ( mut ) ;
2018-12-10 10:27:07 +01:00
auto fresh = std : : make_shared < MDBEnv > ( fname , flags , mode ) ;
2018-12-07 18:17:03 +01:00
if ( stat ( fname , & statbuf ) )
throw std : : runtime_error ( " Unable to stat prospective mdb database: " + string ( strerror ( errno ) ) ) ;
auto key = std : : tie ( statbuf . st_dev , statbuf . st_ino ) ;
s_envs [ key ] = { fresh , flags } ;
return fresh ;
}
}
std : : lock_guard < std : : mutex > l ( mut ) ;
auto key = std : : tie ( statbuf . st_dev , statbuf . st_ino ) ;
auto iter = s_envs . find ( key ) ;
if ( iter ! = s_envs . end ( ) ) {
auto sp = iter - > second . wp . lock ( ) ;
if ( sp ) {
if ( iter - > second . flags ! = flags )
throw std : : runtime_error ( " Can't open mdb with differing flags " ) ;
return sp ;
}
else {
s_envs . erase ( iter ) ; // useful if make_shared fails
}
}
2018-12-08 14:08:26 +01:00
2018-12-10 10:27:07 +01:00
auto fresh = std : : make_shared < MDBEnv > ( fname , flags , mode ) ;
2018-12-07 18:17:03 +01:00
s_envs [ key ] = { fresh , flags } ;
return fresh ;
}
2018-12-07 13:52:17 +01:00
2021-12-05 19:28:08 +01:00
MDBDbi MDBEnv : : openDB ( const string_view dbname , unsigned int flags )
2018-12-07 13:52:17 +01:00
{
unsigned int envflags ;
mdb_env_get_flags ( d_env , & envflags ) ;
2018-12-08 20:58:19 +01:00
/*
This function must not be called from multiple concurrent transactions in the same process . A transaction that uses this function must finish ( either commit or abort ) before any other transaction in the process may use this function .
*/
std : : lock_guard < std : : mutex > l ( d_openmut ) ;
2018-12-07 13:52:17 +01:00
if ( ! ( envflags & MDB_RDONLY ) ) {
auto rwt = getRWTransaction ( ) ;
2019-10-26 11:42:38 +02:00
MDBDbi ret = rwt - > openDB ( dbname , flags ) ;
rwt - > commit ( ) ;
2018-12-07 13:52:17 +01:00
return ret ;
}
2018-12-27 17:49:41 +01:00
MDBDbi ret ;
{
auto rwt = getROTransaction ( ) ;
2019-10-26 11:42:38 +02:00
ret = rwt - > openDB ( dbname , flags ) ;
2018-12-27 17:49:41 +01:00
}
return ret ;
2018-12-07 13:52:17 +01:00
}
2019-10-26 12:07:03 +02:00
MDBRWTransactionImpl : : MDBRWTransactionImpl ( MDBEnv * parent , MDB_txn * txn ) :
MDBROTransactionImpl ( parent , txn )
{
}
2021-12-05 19:28:08 +01:00
MDB_txn * MDBRWTransactionImpl : : openRWTransaction ( MDBEnv * env , MDB_txn * parent , unsigned int flags )
2018-12-10 10:27:07 +01:00
{
2019-10-24 17:56:45 +02:00
MDB_txn * result ;
if ( env - > getROTX ( ) | | env - > getRWTX ( ) )
2018-12-13 23:16:46 +01:00
throw std : : runtime_error ( " Duplicate RW transaction " ) ;
2018-12-10 10:27:07 +01:00
for ( int tries = 0 ; tries < 3 ; + + tries ) { // it might happen twice, who knows
2019-10-24 17:56:45 +02:00
if ( int rc = mdb_txn_begin ( env - > d_env , parent , flags , & result ) ) {
2018-12-10 10:27:07 +01:00
if ( rc = = MDB_MAP_RESIZED & & tries < 2 ) {
// "If the mapsize is increased by another process (..) mdb_txn_begin() will return MDB_MAP_RESIZED.
// call mdb_env_set_mapsize with a size of zero to adopt the new size."
2019-10-24 17:56:45 +02:00
mdb_env_set_mapsize ( env - > d_env , 0 ) ;
2018-12-10 10:27:07 +01:00
continue ;
}
throw std : : runtime_error ( " Unable to start RW transaction: " + std : : string ( mdb_strerror ( rc ) ) ) ;
}
break ;
}
2019-10-24 17:56:45 +02:00
env - > incRWTX ( ) ;
return result ;
2018-12-10 10:27:07 +01:00
}
2021-12-05 19:28:08 +01:00
MDBRWTransactionImpl : : MDBRWTransactionImpl ( MDBEnv * parent , unsigned int flags ) :
2019-10-26 12:07:03 +02:00
MDBRWTransactionImpl ( parent , openRWTransaction ( parent , nullptr , flags ) )
2018-12-10 10:27:07 +01:00
{
2019-10-24 17:56:45 +02:00
}
2019-10-26 11:42:38 +02:00
MDBRWTransactionImpl : : ~ MDBRWTransactionImpl ( )
2019-10-24 17:56:45 +02:00
{
2021-12-23 22:03:28 +01:00
MDBRWTransactionImpl : : abort ( ) ;
2019-10-24 17:56:45 +02:00
}
2019-10-26 11:42:38 +02:00
void MDBRWTransactionImpl : : commit ( )
2019-10-24 17:56:45 +02:00
{
closeRORWCursors ( ) ;
if ( ! d_txn ) {
return ;
}
if ( int rc = mdb_txn_commit ( d_txn ) ) {
throw std : : runtime_error ( " committing: " + std : : string ( mdb_strerror ( rc ) ) ) ;
}
environment ( ) . decRWTX ( ) ;
d_txn = nullptr ;
}
2019-10-26 11:42:38 +02:00
void MDBRWTransactionImpl : : abort ( )
2019-10-24 17:56:45 +02:00
{
closeRORWCursors ( ) ;
if ( ! d_txn ) {
return ;
}
mdb_txn_abort ( d_txn ) ;
// prevent the RO destructor from cleaning up the transaction itself
environment ( ) . decRWTX ( ) ;
d_txn = nullptr ;
}
2019-10-26 11:42:38 +02:00
MDBROTransactionImpl : : MDBROTransactionImpl ( MDBEnv * parent , MDB_txn * txn ) :
2019-10-24 17:56:45 +02:00
d_parent ( parent ) ,
2019-10-26 11:42:38 +02:00
d_cursors ( ) ,
2019-10-24 17:56:45 +02:00
d_txn ( txn )
{
}
2021-12-05 19:28:08 +01:00
MDB_txn * MDBROTransactionImpl : : openROTransaction ( MDBEnv * env , MDB_txn * parent , unsigned int flags )
2019-10-24 17:56:45 +02:00
{
if ( env - > getRWTX ( ) )
2018-12-13 23:16:46 +01:00
throw std : : runtime_error ( " Duplicate RO transaction " ) ;
2018-12-10 10:27:07 +01:00
/*
A transaction and its cursors must only be used by a single thread , and a thread may only have a single transaction at a time . If MDB_NOTLS is in use , this does not apply to read - only transactions . */
2019-10-24 17:56:45 +02:00
MDB_txn * result = nullptr ;
2018-12-10 10:27:07 +01:00
for ( int tries = 0 ; tries < 3 ; + + tries ) { // it might happen twice, who knows
2019-10-24 17:56:45 +02:00
if ( int rc = mdb_txn_begin ( env - > d_env , parent , MDB_RDONLY | flags , & result ) ) {
2018-12-10 10:27:07 +01:00
if ( rc = = MDB_MAP_RESIZED & & tries < 2 ) {
// "If the mapsize is increased by another process (..) mdb_txn_begin() will return MDB_MAP_RESIZED.
// call mdb_env_set_mapsize with a size of zero to adopt the new size."
2019-10-24 17:56:45 +02:00
mdb_env_set_mapsize ( env - > d_env , 0 ) ;
2018-12-10 10:27:07 +01:00
continue ;
}
throw std : : runtime_error ( " Unable to start RO transaction: " + string ( mdb_strerror ( rc ) ) ) ;
}
break ;
}
2019-10-24 17:56:45 +02:00
env - > incROTX ( ) ;
return result ;
}
2019-10-26 11:42:38 +02:00
void MDBROTransactionImpl : : closeROCursors ( )
2019-10-24 17:56:45 +02:00
{
// we need to move the vector away to ensure that the cursors don’ t mess with our iteration.
std : : vector < MDBROCursor * > buf ;
2019-10-26 11:42:38 +02:00
std : : swap ( d_cursors , buf ) ;
2019-10-24 17:56:45 +02:00
for ( auto & cursor : buf ) {
cursor - > close ( ) ;
}
}
2021-12-05 19:28:08 +01:00
MDBROTransactionImpl : : MDBROTransactionImpl ( MDBEnv * parent , unsigned int flags ) :
2019-10-26 11:42:38 +02:00
MDBROTransactionImpl ( parent , openROTransaction ( parent , nullptr , flags ) )
2019-10-24 17:56:45 +02:00
{
}
2019-10-26 11:42:38 +02:00
MDBROTransactionImpl : : ~ MDBROTransactionImpl ( )
2019-10-24 17:56:45 +02:00
{
// this is safe because C++ will not call overrides of virtual methods in destructors.
2021-12-23 22:03:28 +01:00
MDBROTransactionImpl : : commit ( ) ;
2019-10-24 17:56:45 +02:00
}
2019-10-26 11:42:38 +02:00
void MDBROTransactionImpl : : abort ( )
2019-10-24 17:56:45 +02:00
{
closeROCursors ( ) ;
// if d_txn is non-nullptr here, either the transaction object was invalidated earlier (e.g. by moving from it), or it is an RW transaction which has already cleaned up the d_txn pointer (with an abort).
if ( d_txn ) {
d_parent - > decROTX ( ) ;
mdb_txn_abort ( d_txn ) ; // this appears to work better than abort for r/o database opening
d_txn = nullptr ;
}
}
2019-10-26 11:42:38 +02:00
void MDBROTransactionImpl : : commit ( )
2019-10-24 17:56:45 +02:00
{
closeROCursors ( ) ;
// if d_txn is non-nullptr here, either the transaction object was invalidated earlier (e.g. by moving from it), or it is an RW transaction which has already cleaned up the d_txn pointer (with an abort).
if ( d_txn ) {
d_parent - > decROTX ( ) ;
2021-12-25 22:54:46 +01:00
if ( const auto rc = mdb_txn_commit ( d_txn ) ) { // this appears to work better than abort for r/o database opening
throw std : : runtime_error ( " Error comitting transaction: " + MDBError ( rc ) ) ;
}
2019-10-24 17:56:45 +02:00
d_txn = nullptr ;
}
2018-12-10 10:27:07 +01:00
}
2019-10-26 11:42:38 +02:00
void MDBRWTransactionImpl : : clear ( MDB_dbi dbi )
2018-12-09 14:37:08 +01:00
{
if ( int rc = mdb_drop ( d_txn , dbi , 0 ) ) {
throw runtime_error ( " Error clearing database: " + MDBError ( rc ) ) ;
}
}
2019-10-26 11:42:38 +02:00
MDBRWCursor MDBRWTransactionImpl : : getRWCursor ( const MDBDbi & dbi )
2019-10-24 17:56:45 +02:00
{
MDB_cursor * cursor ;
int rc = mdb_cursor_open ( d_txn , dbi , & cursor ) ;
if ( rc ) {
throw std : : runtime_error ( " Error creating RO cursor: " + std : : string ( mdb_strerror ( rc ) ) ) ;
}
2019-10-26 11:42:38 +02:00
return MDBRWCursor ( d_rw_cursors , cursor ) ;
2019-10-24 17:56:45 +02:00
}
2019-10-26 11:42:38 +02:00
MDBRWCursor MDBRWTransactionImpl : : getCursor ( const MDBDbi & dbi )
2018-12-07 13:52:17 +01:00
{
2019-10-24 17:56:45 +02:00
return getRWCursor ( dbi ) ;
2018-12-07 13:52:17 +01:00
}
2019-10-26 12:07:03 +02:00
MDBRWTransaction MDBRWTransactionImpl : : getRWTransaction ( )
{
MDB_txn * txn ;
if ( int rc = mdb_txn_begin ( environment ( ) , * this , 0 , & txn ) ) {
throw std : : runtime_error ( std : : string ( " failed to start child transaction: " ) + mdb_strerror ( rc ) ) ;
}
// we need to increase the counter here because commit/abort on the child transaction will decrease it
environment ( ) . incRWTX ( ) ;
return MDBRWTransaction ( new MDBRWTransactionImpl ( & environment ( ) , txn ) ) ;
}
MDBROTransaction MDBRWTransactionImpl : : getROTransaction ( )
{
2021-12-05 19:28:08 +01:00
return getRWTransaction ( ) ;
2019-10-26 12:07:03 +02:00
}
2018-12-07 13:52:17 +01:00
MDBROTransaction MDBEnv : : getROTransaction ( )
{
2019-10-26 11:42:38 +02:00
return MDBROTransaction ( new MDBROTransactionImpl ( this ) ) ;
2018-12-07 13:52:17 +01:00
}
MDBRWTransaction MDBEnv : : getRWTransaction ( )
{
2019-10-26 11:42:38 +02:00
return MDBRWTransaction ( new MDBRWTransactionImpl ( this ) ) ;
2018-12-07 13:52:17 +01:00
}
2019-10-26 11:42:38 +02:00
void MDBRWTransactionImpl : : closeRWCursors ( )
2018-12-07 13:52:17 +01:00
{
2019-10-26 11:42:38 +02:00
decltype ( d_rw_cursors ) buf ;
std : : swap ( d_rw_cursors , buf ) ;
2019-10-24 17:56:45 +02:00
for ( auto & cursor : buf ) {
cursor - > close ( ) ;
}
2018-12-07 13:52:17 +01:00
}
2019-10-26 11:42:38 +02:00
MDBROCursor MDBROTransactionImpl : : getCursor ( const MDBDbi & dbi )
2018-12-07 13:52:17 +01:00
{
2019-10-24 17:56:45 +02:00
return getROCursor ( dbi ) ;
}
2019-10-26 11:42:38 +02:00
MDBROCursor MDBROTransactionImpl : : getROCursor ( const MDBDbi & dbi )
2019-10-24 17:56:45 +02:00
{
MDB_cursor * cursor ;
int rc = mdb_cursor_open ( d_txn , dbi , & cursor ) ;
if ( rc ) {
throw std : : runtime_error ( " Error creating RO cursor: " + std : : string ( mdb_strerror ( rc ) ) ) ;
}
2019-10-26 11:42:38 +02:00
return MDBROCursor ( d_cursors , cursor ) ;
2018-12-07 13:52:17 +01:00
}