2018-12-07 13:52:17 +01:00
# include "lmdb-safe.hh"
2022-01-18 21:58:46 +01:00
2018-12-07 18:17:03 +01:00
# include <fcntl.h>
2022-01-18 21:58:46 +01:00
# include <sys/stat.h>
# include <cstring>
2018-12-07 18:17:03 +01:00
# include <map>
2022-01-30 21:14:43 +01:00
# include <memory>
# include <mutex>
2018-12-07 18:17:03 +01:00
2018-12-10 22:08:49 +01:00
using namespace std ;
2022-01-18 22:08:36 +01:00
namespace LMDBSafe {
2022-01-30 21:14:43 +01:00
MDBDbi : : MDBDbi ( MDB_env * env , MDB_txn * txn , const string_view dbname , unsigned int flags )
2018-12-08 14:08:26 +01:00
{
2022-01-30 21:14:43 +01:00
( void ) env ;
// A transaction that uses this function must finish (either commit or abort) before any other transaction in the process may use this function.
if ( const auto rc = mdb_dbi_open ( txn , dbname . empty ( ) ? 0 : & dbname [ 0 ] , flags , & d_dbi ) )
throw LMDBError ( " Unable to open named database: " , rc ) ;
// Database names are keys in the unnamed database, and may be read but not written.
2018-12-08 14:08:26 +01:00
}
2022-01-30 21:14:43 +01:00
MDBEnv : : MDBEnv ( const char * fname , unsigned int flags , mdb_mode_t mode , MDB_dbi maxDBs )
2018-12-08 14:08:26 +01:00
{
2022-01-30 21:14:43 +01:00
mdb_env_create ( & d_env ) ;
if ( const auto rc = mdb_env_set_mapsize ( d_env , 16ULL * 4096 * 244140ULL ) ) { // 4GB
throw LMDBError ( " Setting map size: " , rc ) ;
}
// Various other options may also need to be set before opening the handle, e.g. mdb_env_set_mapsize(), mdb_env_set_maxreaders(), mdb_env_set_maxdbs(),
if ( const auto rc = mdb_env_set_maxdbs ( d_env , maxDBs ) ) {
throw LMDBError ( " Setting maxdbs: " , rc ) ;
}
2018-12-08 14:08:26 +01:00
2022-01-30 21:14:43 +01:00
// we need MDB_NOTLS since we rely on its semantics
if ( const auto rc = mdb_env_open ( d_env , fname , flags | MDB_NOTLS , mode ) ) {
// If this function fails, mdb_env_close() must be called to discard the MDB_env handle.
mdb_env_close ( d_env ) ;
throw LMDBError ( " Unable to open database file " + std : : string ( fname ) + " : " , rc ) ;
}
2018-12-08 14:08:26 +01:00
}
void MDBEnv : : incROTX ( )
{
2022-01-30 21:14:43 +01:00
std : : lock_guard < std : : mutex > l ( d_countmutex ) ;
+ + d_ROtransactionsOut [ std : : this_thread : : get_id ( ) ] ;
2018-12-08 14:08:26 +01:00
}
void MDBEnv : : decROTX ( )
{
2022-01-30 21:14:43 +01:00
std : : lock_guard < std : : mutex > l ( d_countmutex ) ;
- - d_ROtransactionsOut [ std : : this_thread : : get_id ( ) ] ;
2018-12-08 14:08:26 +01:00
}
void MDBEnv : : incRWTX ( )
{
2022-01-30 21:14:43 +01:00
std : : lock_guard < std : : mutex > l ( d_countmutex ) ;
+ + d_RWtransactionsOut [ std : : this_thread : : get_id ( ) ] ;
2018-12-08 14:08:26 +01:00
}
void MDBEnv : : decRWTX ( )
{
2022-01-30 21:14:43 +01:00
std : : lock_guard < std : : mutex > l ( d_countmutex ) ;
- - d_RWtransactionsOut [ std : : this_thread : : get_id ( ) ] ;
2018-12-08 14:08:26 +01:00
}
int MDBEnv : : getRWTX ( )
{
2022-01-30 21:14:43 +01:00
std : : lock_guard < std : : mutex > l ( d_countmutex ) ;
return d_RWtransactionsOut [ std : : this_thread : : get_id ( ) ] ;
2018-12-08 14:08:26 +01:00
}
int MDBEnv : : getROTX ( )
{
2022-01-30 21:14:43 +01:00
std : : lock_guard < std : : mutex > l ( d_countmutex ) ;
return d_ROtransactionsOut [ std : : this_thread : : get_id ( ) ] ;
}
std : : shared_ptr < MDBEnv > getMDBEnv ( const char * fname , unsigned int flags , mdb_mode_t mode , MDB_dbi maxDBs )
{
struct Value {
weak_ptr < MDBEnv > wp ;
unsigned int flags ;
} ;
static std : : map < tuple < dev_t , ino_t > , Value > s_envs ;
static std : : mutex mut ;
struct stat statbuf ;
if ( stat ( fname , & statbuf ) ) {
if ( errno ! = ENOENT )
throw LMDBError ( " Unable to stat prospective mdb database: " + string ( strerror ( errno ) ) ) ;
else {
std : : lock_guard < std : : mutex > l ( mut ) ;
auto fresh = std : : make_shared < MDBEnv > ( fname , flags , mode , maxDBs ) ;
if ( stat ( fname , & statbuf ) )
throw LMDBError ( " Unable to stat prospective mdb database: " + string ( strerror ( errno ) ) ) ;
auto key = std : : tie ( statbuf . st_dev , statbuf . st_ino ) ;
s_envs [ key ] = { fresh , flags } ;
return fresh ;
}
2018-12-07 18:17:03 +01:00
}
2022-01-30 21:14:43 +01:00
std : : lock_guard < std : : mutex > l ( mut ) ;
auto key = std : : tie ( statbuf . st_dev , statbuf . st_ino ) ;
auto iter = s_envs . find ( key ) ;
if ( iter ! = s_envs . end ( ) ) {
auto sp = iter - > second . wp . lock ( ) ;
if ( sp ) {
if ( iter - > second . flags ! = flags )
throw LMDBError ( " Can't open mdb with differing flags " ) ;
return sp ;
} else {
s_envs . erase ( iter ) ; // useful if make_shared fails
}
2018-12-07 18:17:03 +01:00
}
2018-12-08 14:08:26 +01:00
2022-01-30 21:14:43 +01:00
auto fresh = std : : make_shared < MDBEnv > ( fname , flags , mode , maxDBs ) ;
s_envs [ key ] = { fresh , flags } ;
2018-12-07 18:17:03 +01:00
2022-01-30 21:14:43 +01:00
return fresh ;
}
2018-12-07 13:52:17 +01:00
2021-12-05 19:28:08 +01:00
MDBDbi MDBEnv : : openDB ( const string_view dbname , unsigned int flags )
2018-12-07 13:52:17 +01:00
{
2022-01-30 21:14:43 +01:00
unsigned int envflags ;
mdb_env_get_flags ( d_env , & envflags ) ;
/*
2018-12-08 20:58:19 +01:00
This function must not be called from multiple concurrent transactions in the same process . A transaction that uses this function must finish ( either commit or abort ) before any other transaction in the process may use this function .
*/
2022-01-30 21:14:43 +01:00
std : : lock_guard < std : : mutex > l ( d_openmut ) ;
if ( ! ( envflags & MDB_RDONLY ) ) {
auto rwt = getRWTransaction ( ) ;
MDBDbi ret = rwt - > openDB ( dbname , flags ) ;
rwt - > commit ( ) ;
return ret ;
}
2018-12-27 17:49:41 +01:00
2022-01-30 21:14:43 +01:00
MDBDbi ret ;
{
auto rwt = getROTransaction ( ) ;
ret = rwt - > openDB ( dbname , flags ) ;
}
return ret ;
2018-12-07 13:52:17 +01:00
}
2022-01-30 21:14:43 +01:00
MDBRWTransactionImpl : : MDBRWTransactionImpl ( MDBEnv * parent , MDB_txn * txn )
: MDBROTransactionImpl ( parent , txn )
2019-10-26 12:07:03 +02:00
{
}
2021-12-05 19:28:08 +01:00
MDB_txn * MDBRWTransactionImpl : : openRWTransaction ( MDBEnv * env , MDB_txn * parent , unsigned int flags )
2018-12-10 10:27:07 +01:00
{
2022-01-30 21:14:43 +01:00
MDB_txn * result ;
if ( env - > getRWTX ( ) )
throw LMDBError ( " Duplicate RW transaction " ) ;
for ( int tries = 0 ; tries < 3 ; + + tries ) { // it might happen twice, who knows
if ( int rc = mdb_txn_begin ( env - > d_env , parent , flags , & result ) ) {
if ( rc = = MDB_MAP_RESIZED & & tries < 2 ) {
// "If the mapsize is increased by another process (..) mdb_txn_begin() will return MDB_MAP_RESIZED.
// call mdb_env_set_mapsize with a size of zero to adopt the new size."
mdb_env_set_mapsize ( env - > d_env , 0 ) ;
continue ;
}
throw LMDBError ( " Unable to start RW transaction: " , rc ) ;
}
break ;
2018-12-10 10:27:07 +01:00
}
2022-01-30 21:14:43 +01:00
env - > incRWTX ( ) ;
return result ;
2018-12-10 10:27:07 +01:00
}
2022-01-30 21:14:43 +01:00
MDBRWTransactionImpl : : MDBRWTransactionImpl ( MDBEnv * parent , unsigned int flags )
: MDBRWTransactionImpl ( parent , openRWTransaction ( parent , nullptr , flags ) )
2018-12-10 10:27:07 +01:00
{
2019-10-24 17:56:45 +02:00
}
2019-10-26 11:42:38 +02:00
MDBRWTransactionImpl : : ~ MDBRWTransactionImpl ( )
2019-10-24 17:56:45 +02:00
{
2022-01-30 21:14:43 +01:00
MDBRWTransactionImpl : : abort ( ) ;
2019-10-24 17:56:45 +02:00
}
2019-10-26 11:42:38 +02:00
void MDBRWTransactionImpl : : commit ( )
2019-10-24 17:56:45 +02:00
{
2022-01-30 21:14:43 +01:00
closeRORWCursors ( ) ;
if ( ! d_txn ) {
return ;
}
2019-10-24 17:56:45 +02:00
2022-01-30 21:14:43 +01:00
if ( const auto rc = mdb_txn_commit ( d_txn ) ) {
throw LMDBError ( " Committing transaction: " , rc ) ;
}
environment ( ) . decRWTX ( ) ;
d_txn = nullptr ;
2019-10-24 17:56:45 +02:00
}
2019-10-26 11:42:38 +02:00
void MDBRWTransactionImpl : : abort ( )
2019-10-24 17:56:45 +02:00
{
2022-01-30 21:14:43 +01:00
closeRORWCursors ( ) ;
if ( ! d_txn ) {
return ;
}
2019-10-24 17:56:45 +02:00
2022-01-30 21:14:43 +01:00
mdb_txn_abort ( d_txn ) ;
// prevent the RO destructor from cleaning up the transaction itself
environment ( ) . decRWTX ( ) ;
d_txn = nullptr ;
2019-10-24 17:56:45 +02:00
}
2022-01-30 21:14:43 +01:00
MDBROTransactionImpl : : MDBROTransactionImpl ( MDBEnv * parent , MDB_txn * txn )
: d_parent ( parent )
, d_cursors ( )
, d_txn ( txn )
2019-10-24 17:56:45 +02:00
{
}
2021-12-05 19:28:08 +01:00
MDB_txn * MDBROTransactionImpl : : openROTransaction ( MDBEnv * env , MDB_txn * parent , unsigned int flags )
2019-10-24 17:56:45 +02:00
{
2022-01-30 21:14:43 +01:00
if ( env - > getRWTX ( ) )
throw LMDBError ( " Duplicate RO transaction " ) ;
/*
2018-12-10 10:27:07 +01:00
A transaction and its cursors must only be used by a single thread , and a thread may only have a single transaction at a time . If MDB_NOTLS is in use , this does not apply to read - only transactions . */
2022-01-30 21:14:43 +01:00
MDB_txn * result = nullptr ;
for ( int tries = 0 ; tries < 3 ; + + tries ) { // it might happen twice, who knows
if ( const auto rc = mdb_txn_begin ( env - > d_env , parent , MDB_RDONLY | flags , & result ) ) {
if ( rc = = MDB_MAP_RESIZED & & tries < 2 ) {
// "If the mapsize is increased by another process (..) mdb_txn_begin() will return MDB_MAP_RESIZED.
// call mdb_env_set_mapsize with a size of zero to adopt the new size."
mdb_env_set_mapsize ( env - > d_env , 0 ) ;
continue ;
}
throw LMDBError ( " Unable to start RO transaction: " , rc ) ;
}
break ;
2018-12-10 10:27:07 +01:00
}
2022-01-30 21:14:43 +01:00
env - > incROTX ( ) ;
2019-10-24 17:56:45 +02:00
2022-01-30 21:14:43 +01:00
return result ;
2019-10-24 17:56:45 +02:00
}
2019-10-26 11:42:38 +02:00
void MDBROTransactionImpl : : closeROCursors ( )
2019-10-24 17:56:45 +02:00
{
2022-01-30 21:14:43 +01:00
// we need to move the vector away to ensure that the cursors don’ t mess with our iteration.
std : : vector < MDBROCursor * > buf ;
std : : swap ( d_cursors , buf ) ;
for ( auto & cursor : buf ) {
cursor - > close ( ) ;
}
2019-10-24 17:56:45 +02:00
}
2022-01-30 21:14:43 +01:00
MDBROTransactionImpl : : MDBROTransactionImpl ( MDBEnv * parent , unsigned int flags )
: MDBROTransactionImpl ( parent , openROTransaction ( parent , nullptr , flags ) )
2019-10-24 17:56:45 +02:00
{
}
2019-10-26 11:42:38 +02:00
MDBROTransactionImpl : : ~ MDBROTransactionImpl ( )
2019-10-24 17:56:45 +02:00
{
2022-01-30 21:14:43 +01:00
// this is safe because C++ will not call overrides of virtual methods in destructors.
MDBROTransactionImpl : : commit ( ) ;
2019-10-24 17:56:45 +02:00
}
2019-10-26 11:42:38 +02:00
void MDBROTransactionImpl : : abort ( )
2019-10-24 17:56:45 +02:00
{
2022-01-30 21:14:43 +01:00
closeROCursors ( ) ;
// if d_txn is non-nullptr here, either the transaction object was invalidated earlier (e.g. by moving from it), or it is an RW transaction which has already cleaned up the d_txn pointer (with an abort).
if ( d_txn ) {
d_parent - > decROTX ( ) ;
mdb_txn_abort ( d_txn ) ; // this appears to work better than abort for r/o database opening
d_txn = nullptr ;
}
2019-10-24 17:56:45 +02:00
}
2019-10-26 11:42:38 +02:00
void MDBROTransactionImpl : : commit ( )
2019-10-24 17:56:45 +02:00
{
2022-01-30 21:14:43 +01:00
closeROCursors ( ) ;
// if d_txn is non-nullptr here, either the transaction object was invalidated earlier (e.g. by moving from it), or it is an RW transaction which has already cleaned up the d_txn pointer (with an abort).
if ( d_txn ) {
d_parent - > decROTX ( ) ;
if ( const auto rc = mdb_txn_commit ( d_txn ) ) { // this appears to work better than abort for r/o database opening
2022-03-01 00:42:50 +01:00
throw LMDBError ( " Error committing transaction: " , rc ) ;
2022-01-30 21:14:43 +01:00
}
d_txn = nullptr ;
2021-12-25 22:54:46 +01:00
}
2018-12-10 10:27:07 +01:00
}
2019-10-26 11:42:38 +02:00
void MDBRWTransactionImpl : : clear ( MDB_dbi dbi )
2018-12-09 14:37:08 +01:00
{
2022-01-30 21:14:43 +01:00
if ( const auto rc = mdb_drop ( d_txn , dbi , 0 ) ) {
throw LMDBError ( " Error clearing database: " , rc ) ;
}
2018-12-09 14:37:08 +01:00
}
2022-01-30 21:14:43 +01:00
MDBRWCursor MDBRWTransactionImpl : : getRWCursor ( const MDBDbi & dbi )
2019-10-24 17:56:45 +02:00
{
2022-01-30 21:14:43 +01:00
MDB_cursor * cursor ;
;
if ( const auto rc = mdb_cursor_open ( d_txn , dbi , & cursor ) ) {
throw LMDBError ( " Error creating RO cursor: " , rc ) ;
}
return MDBRWCursor ( d_rw_cursors , cursor ) ;
2019-10-24 17:56:45 +02:00
}
2019-10-26 11:42:38 +02:00
MDBRWCursor MDBRWTransactionImpl : : getCursor ( const MDBDbi & dbi )
2018-12-07 13:52:17 +01:00
{
2022-01-30 21:14:43 +01:00
return getRWCursor ( dbi ) ;
2018-12-07 13:52:17 +01:00
}
2019-10-26 12:07:03 +02:00
MDBRWTransaction MDBRWTransactionImpl : : getRWTransaction ( )
{
2022-01-30 21:14:43 +01:00
MDB_txn * txn ;
if ( const auto rc = mdb_txn_begin ( environment ( ) , * this , 0 , & txn ) ) {
throw LMDBError ( " Failed to start child transaction: " , rc ) ;
}
// we need to increase the counter here because commit/abort on the child transaction will decrease it
environment ( ) . incRWTX ( ) ;
return MDBRWTransaction ( new MDBRWTransactionImpl ( & environment ( ) , txn ) ) ;
2019-10-26 12:07:03 +02:00
}
MDBROTransaction MDBRWTransactionImpl : : getROTransaction ( )
{
2022-01-30 21:14:43 +01:00
return getRWTransaction ( ) ;
2019-10-26 12:07:03 +02:00
}
2018-12-07 13:52:17 +01:00
MDBROTransaction MDBEnv : : getROTransaction ( )
{
2022-01-30 21:14:43 +01:00
return MDBROTransaction ( new MDBROTransactionImpl ( this ) ) ;
2018-12-07 13:52:17 +01:00
}
MDBRWTransaction MDBEnv : : getRWTransaction ( )
{
2022-01-30 21:14:43 +01:00
return MDBRWTransaction ( new MDBRWTransactionImpl ( this ) ) ;
2018-12-07 13:52:17 +01:00
}
2019-10-26 11:42:38 +02:00
void MDBRWTransactionImpl : : closeRWCursors ( )
2018-12-07 13:52:17 +01:00
{
2022-01-30 21:14:43 +01:00
decltype ( d_rw_cursors ) buf ;
std : : swap ( d_rw_cursors , buf ) ;
for ( auto & cursor : buf ) {
cursor - > close ( ) ;
}
2018-12-07 13:52:17 +01:00
}
2022-01-30 21:14:43 +01:00
MDBROCursor MDBROTransactionImpl : : getCursor ( const MDBDbi & dbi )
2018-12-07 13:52:17 +01:00
{
2022-01-30 21:14:43 +01:00
return getROCursor ( dbi ) ;
2019-10-24 17:56:45 +02:00
}
2019-10-26 11:42:38 +02:00
MDBROCursor MDBROTransactionImpl : : getROCursor ( const MDBDbi & dbi )
2019-10-24 17:56:45 +02:00
{
2022-01-30 21:14:43 +01:00
MDB_cursor * cursor ;
if ( const auto rc = mdb_cursor_open ( d_txn , dbi , & cursor ) ) {
throw LMDBError ( " Error creating RO cursor: " , rc ) ;
}
return MDBROCursor ( d_cursors , cursor ) ;
2018-12-07 13:52:17 +01:00
}
2022-01-18 22:08:36 +01:00
2022-01-30 21:14:43 +01:00
} // namespace LMDBSafe