2022-02-22 22:46:53 +01:00
// ignore warning about "return result_type{ storageEntry.id, storageEntry.ref.relatedStorage };"
# pragma GCC diagnostic ignored "-Wnull-dereference"
2021-12-05 23:40:51 +01:00
# include "./storageprivate.h"
# include <c++utilities/conversion/stringbuilder.h>
2022-06-19 22:47:31 +02:00
# include <c++utilities/io/ansiescapecodes.h>
# include <iostream>
2021-12-05 23:40:51 +01:00
using namespace CppUtilities ;
namespace LibPkg {
2022-01-20 23:33:02 +01:00
template < typename StorageEntryType >
template < typename IndexType >
auto StorageCacheEntries < StorageEntryType > : : find ( const IndexType & ref ) - > StorageEntry *
2021-12-05 23:40:51 +01:00
{
2022-01-20 23:33:02 +01:00
const auto & index = m_entries . template get < IndexType > ( ) ;
2022-01-19 23:27:14 +01:00
if ( auto i = index . find ( ref ) ; i ! = index . end ( ) ) {
m_entries . relocate ( m_entries . begin ( ) , m_entries . template project < 0 > ( i ) ) ;
2022-01-20 23:33:02 +01:00
return & i . get_node ( ) - > value ( ) ;
2022-01-19 23:27:14 +01:00
}
2022-01-20 23:33:02 +01:00
return nullptr ;
}
template < typename StorageEntryType > auto StorageCacheEntries < StorageEntryType > : : insert ( StorageEntry & & entry ) - > StorageEntry &
{
const auto [ i , newItem ] = m_entries . emplace_front ( entry ) ;
2022-01-19 23:27:14 +01:00
if ( ! newItem ) {
m_entries . relocate ( m_entries . begin ( ) , i ) ;
} else if ( m_entries . size ( ) > m_limit ) {
m_entries . pop_back ( ) ;
}
return i . get_node ( ) - > value ( ) ;
}
template < typename StorageEntryType > std : : size_t StorageCacheEntries < StorageEntryType > : : clear ( const Storage & storage )
{
auto count = std : : size_t ( ) ;
for ( auto i = m_entries . begin ( ) ; i ! = m_entries . end ( ) ; ) {
if ( i - > ref . relatedStorage = = & storage ) {
i = m_entries . erase ( i ) ;
+ + count ;
} else {
+ + i ;
}
}
return count ;
2021-12-05 23:40:51 +01:00
}
2022-01-26 00:41:53 +01:00
template < typename StorageEntryType > void StorageCacheEntries < StorageEntryType > : : setLimit ( std : : size_t limit )
{
m_limit = limit ;
while ( m_entries . size ( ) > limit ) {
m_entries . pop_back ( ) ;
}
}
2022-01-21 20:35:43 +01:00
template < typename StorageEntriesType , typename StorageType , typename SpecType >
auto StorageCache < StorageEntriesType , StorageType , SpecType > : : retrieve ( Storage & storage , ROTxn * txn , StorageID storageID ) - > SpecType
2022-01-20 23:33:02 +01:00
{
2022-03-05 18:55:52 +01:00
// check for package in cache, should be ok even if the db is being updated
2022-01-20 23:33:02 +01:00
const auto ref = typename StorageEntryByID < typename Entries : : StorageEntry > : : result_type { storageID , & storage } ;
auto lock = std : : unique_lock ( m_mutex ) ;
if ( auto * const existingCacheEntry = m_entries . find ( ref ) ) {
return SpecType ( existingCacheEntry - > id , existingCacheEntry - > entry ) ;
}
// check for package in storage, populate cache entry
lock . unlock ( ) ;
auto entry = std : : make_shared < Entry > ( ) ;
2022-01-21 20:35:43 +01:00
if ( auto id = txn ? txn - > get ( storageID , * entry ) : storage . packages . getROTransaction ( ) . get ( storageID , * entry ) ) {
2022-03-05 18:55:52 +01:00
// try to acquire update lock to avoid update existing cache entries while db is being updated
if ( const auto updateLock = std : : unique_lock ( storage . updateMutex , std : : try_to_lock ) ) {
using CacheEntry = typename Entries : : StorageEntry ;
using CacheRef = typename Entries : : Ref ;
auto newCacheEntry = CacheEntry ( CacheRef ( storage , entry ) , id ) ;
newCacheEntry . entry = entry ;
lock = std : : unique_lock ( m_mutex ) ;
m_entries . insert ( std : : move ( newCacheEntry ) ) ;
lock . unlock ( ) ;
}
2022-01-20 23:33:02 +01:00
return SpecType ( id , entry ) ;
}
return SpecType ( 0 , std : : shared_ptr < Entry > ( ) ) ;
}
2022-01-21 20:35:43 +01:00
template < typename StorageEntriesType , typename StorageType , typename SpecType >
auto StorageCache < StorageEntriesType , StorageType , SpecType > : : retrieve ( Storage & storage , StorageID storageID ) - > SpecType
{
return retrieve ( storage , nullptr , storageID ) ;
}
template < typename StorageEntriesType , typename StorageType , typename SpecType >
2022-01-31 20:51:45 +01:00
auto StorageCache < StorageEntriesType , StorageType , SpecType > : : retrieve ( Storage & storage , RWTxn * txn , const std : : string & entryName ) - > SpecType
2021-12-05 23:40:51 +01:00
{
2022-03-01 00:43:06 +01:00
// do not attempt to fetch empty entries (apparently can lead to "Getting data: MDB_BAD_VALSIZE: Unsupported size of key/DB name/data, or wrong DUPFIXED size")
2022-02-20 19:28:44 +01:00
if ( entryName . empty ( ) ) {
return SpecType ( 0 , std : : shared_ptr < Entry > ( ) ) ;
}
2022-03-05 18:55:52 +01:00
// check for package in cache, should be ok even if the db is being updated
2022-01-20 23:33:02 +01:00
using CacheRef = typename Entries : : Ref ;
const auto ref = CacheRef ( storage , entryName ) ;
auto lock = std : : unique_lock ( m_mutex ) ;
if ( auto * const existingCacheEntry = m_entries . find ( ref ) ) {
return SpecType ( existingCacheEntry - > id , existingCacheEntry - > entry ) ;
2021-12-05 23:40:51 +01:00
}
2022-01-20 23:33:02 +01:00
lock . unlock ( ) ;
2021-12-05 23:40:51 +01:00
// check for package in storage, populate cache entry
2022-01-20 23:33:02 +01:00
auto entry = std : : make_shared < Entry > ( ) ;
2022-01-31 20:51:45 +01:00
if ( auto id = txn ? txn - > template get < 0 > ( entryName , * entry ) : storage . packages . getROTransaction ( ) . template get < 0 > ( entryName , * entry ) ) {
2022-03-05 18:55:52 +01:00
// try to acquire update lock to avoid update existing cache entries while db is being updated
if ( const auto updateLock = std : : unique_lock ( storage . updateMutex , std : : try_to_lock ) ) {
using CacheEntry = typename Entries : : StorageEntry ;
auto newCacheEntry = CacheEntry ( CacheRef ( storage , entry ) , id ) ;
newCacheEntry . entry = entry ;
lock = std : : unique_lock ( m_mutex ) ;
m_entries . insert ( std : : move ( newCacheEntry ) ) ;
lock . unlock ( ) ;
}
2022-01-20 23:33:02 +01:00
return SpecType ( id , entry ) ;
2021-12-05 23:40:51 +01:00
}
2022-01-20 23:33:02 +01:00
return SpecType ( 0 , std : : shared_ptr < Entry > ( ) ) ;
2021-12-05 23:40:51 +01:00
}
2022-01-31 20:51:45 +01:00
template < typename StorageEntriesType , typename StorageType , typename SpecType >
auto StorageCache < StorageEntriesType , StorageType , SpecType > : : retrieve ( Storage & storage , const std : : string & entryName ) - > SpecType
{
return retrieve ( storage , nullptr , entryName ) ;
}
2023-12-21 20:51:38 +01:00
/*!
* \ brief Stores the specified \ a entry .
* \ remarks The entry may exist or may not exist . A lookup for an existing entry is done to take over
* deps / provides from the existing entry if it makes sense .
*/
2022-01-21 20:35:43 +01:00
template < typename StorageEntriesType , typename StorageType , typename SpecType >
2023-12-19 21:34:18 +01:00
auto StorageCache < StorageEntriesType , StorageType , SpecType > : : store ( Storage & storage , RWTxn & txn , const std : : shared_ptr < Entry > & entry ) - > StoreResult
2021-12-05 23:40:51 +01:00
{
// check for package in cache
2022-01-20 23:33:02 +01:00
using CacheEntry = typename Entries : : StorageEntry ;
using CacheRef = typename Entries : : Ref ;
2022-01-19 23:27:14 +01:00
auto res = StorageCache : : StoreResult ( ) ;
2022-03-10 22:45:11 +01:00
if ( entry - > name . empty ( ) ) {
return res ;
}
const auto ref = CacheRef ( storage , entry ) ;
2021-12-05 23:40:51 +01:00
auto lock = std : : unique_lock ( m_mutex ) ;
2022-01-20 23:33:02 +01:00
auto * cacheEntry = m_entries . find ( ref ) ;
if ( cacheEntry ) {
2021-12-05 23:40:51 +01:00
// retain certain information obtained from package contents if this is actually the same package as before
2022-01-20 23:33:02 +01:00
res . id = cacheEntry - > id ;
2023-12-19 21:34:18 +01:00
res . oldEntry = cacheEntry - > entry ;
entry - > addDepsAndProvidesFromOtherPackage ( * res . oldEntry ) ;
2022-01-20 23:33:02 +01:00
}
lock . unlock ( ) ;
2023-12-21 20:51:38 +01:00
2022-01-20 23:33:02 +01:00
// check for package in storage
if ( ! res . oldEntry ) {
res . oldEntry = std : : make_shared < Entry > ( ) ;
2022-01-25 00:04:25 +01:00
if ( ( res . id = txn . template get < 0 > ( entry - > name , * res . oldEntry ) ) ) {
2022-01-20 23:33:02 +01:00
entry - > addDepsAndProvidesFromOtherPackage ( * res . oldEntry ) ;
} else {
res . oldEntry . reset ( ) ;
2021-12-05 23:40:51 +01:00
}
}
2023-12-21 20:51:38 +01:00
2021-12-05 23:40:51 +01:00
// update package in storage
2022-01-20 23:33:02 +01:00
res . id = txn . put ( * entry , res . id ) ;
2023-12-21 20:51:38 +01:00
2022-01-20 23:33:02 +01:00
// update cache entry
lock = std : : unique_lock ( m_mutex ) ;
if ( cacheEntry ) {
cacheEntry - > ref . entryName = & entry - > name ;
} else {
cacheEntry = & m_entries . insert ( CacheEntry ( ref , res . id ) ) ;
}
cacheEntry - > entry = entry ;
lock . unlock ( ) ;
2023-12-21 20:51:38 +01:00
2021-12-05 23:40:51 +01:00
res . updated = true ;
return res ;
}
2023-12-21 20:51:38 +01:00
/*!
* \ brief Stores the specified \ a entry with the specified \ a storageID .
* \ remarks This is used to update an existing entry with a known ID .
*/
template < typename StorageEntriesType , typename StorageType , typename SpecType >
auto StorageCache < StorageEntriesType , StorageType , SpecType > : : store (
Storage & storage , RWTxn & txn , StorageID storageID , const std : : shared_ptr < Entry > & entry ) - > void
{
// update package in storage
const auto id = txn . put ( * entry , storageID ) ;
// update cache entry
using CacheEntry = typename Entries : : StorageEntry ;
using CacheRef = typename Entries : : Ref ;
const auto ref = CacheRef ( storage , entry ) ;
const auto lock = std : : unique_lock ( m_mutex ) ;
const auto cacheEntry = & m_entries . insert ( CacheEntry ( ref , id ) ) ;
cacheEntry - > entry = entry ;
}
2022-01-21 20:35:43 +01:00
template < typename StorageEntriesType , typename StorageType , typename SpecType >
bool StorageCache < StorageEntriesType , StorageType , SpecType > : : invalidate ( Storage & storage , const std : : string & entryName )
2021-12-05 23:40:51 +01:00
{
// remove package from cache
2023-12-16 23:07:22 +01:00
invalidateCacheOnly ( storage , entryName ) ;
2021-12-05 23:40:51 +01:00
// remove package from storage
2022-01-19 23:27:14 +01:00
auto txn = storage . packages . getRWTransaction ( ) ;
if ( auto i = txn . template find < 0 > ( entryName ) ; i ! = txn . end ( ) ) {
2021-12-05 23:40:51 +01:00
i . del ( ) ;
txn . commit ( ) ;
return true ;
}
return false ;
}
2023-12-16 23:07:22 +01:00
template < typename StorageEntriesType , typename StorageType , typename SpecType >
bool StorageCache < StorageEntriesType , StorageType , SpecType > : : invalidateCacheOnly ( Storage & storage , const std : : string & entryName )
{
const auto ref = typename Entries : : Ref ( storage , entryName ) ;
const auto lock = std : : unique_lock ( m_mutex ) ;
m_entries . erase ( ref ) ;
return true ;
}
2022-01-21 20:35:43 +01:00
template < typename StorageEntriesType , typename StorageType , typename SpecType >
void StorageCache < StorageEntriesType , StorageType , SpecType > : : clear ( Storage & storage )
2021-12-05 23:40:51 +01:00
{
2022-01-19 23:27:14 +01:00
clearCacheOnly ( storage ) ;
auto packagesTxn = storage . packages . getRWTransaction ( ) ;
2021-12-05 23:40:51 +01:00
packagesTxn . clear ( ) ;
packagesTxn . commit ( ) ;
2022-01-19 23:27:14 +01:00
auto providedDepsTxn = storage . providedDeps . getRWTransaction ( ) ;
2021-12-05 23:40:51 +01:00
providedDepsTxn . clear ( ) ;
providedDepsTxn . commit ( ) ;
2022-01-19 23:27:14 +01:00
auto requiredDepsTxn = storage . requiredDeps . getRWTransaction ( ) ;
2021-12-05 23:40:51 +01:00
requiredDepsTxn . clear ( ) ;
requiredDepsTxn . commit ( ) ;
2022-01-19 23:27:14 +01:00
auto providedLibsTxn = storage . providedLibs . getRWTransaction ( ) ;
2021-12-05 23:40:51 +01:00
providedLibsTxn . clear ( ) ;
providedLibsTxn . commit ( ) ;
2022-01-19 23:27:14 +01:00
auto requiredLibsTxn = storage . requiredLibs . getRWTransaction ( ) ;
2021-12-05 23:40:51 +01:00
requiredLibsTxn . clear ( ) ;
requiredLibsTxn . commit ( ) ;
}
2022-01-21 20:35:43 +01:00
template < typename StorageEntriesType , typename StorageType , typename SpecType >
void StorageCache < StorageEntriesType , StorageType , SpecType > : : clearCacheOnly ( Storage & storage )
2021-12-05 23:40:51 +01:00
{
const auto lock = std : : unique_lock ( m_mutex ) ;
2022-01-19 23:27:14 +01:00
m_entries . clear ( storage ) ;
}
2022-01-26 00:41:53 +01:00
template < typename StorageEntriesType , typename StorageType , typename SpecType >
void StorageCache < StorageEntriesType , StorageType , SpecType > : : setLimit ( std : : size_t limit )
{
const auto lock = std : : unique_lock ( m_mutex ) ;
m_entries . setLimit ( limit ) ;
}
2022-01-19 23:27:14 +01:00
template struct StorageCacheRef < DatabaseStorage , Package > ;
template struct StorageCacheEntry < PackageCacheRef , Package > ;
template class StorageCacheEntries < PackageCacheEntry > ;
2022-01-21 20:35:43 +01:00
template struct StorageCache < PackageCacheEntries , PackageStorage , PackageSpec > ;
2022-01-19 23:27:14 +01:00
StorageDistribution : : StorageDistribution ( const char * path , std : : uint32_t maxDbs )
{
m_env = LMDBSafe : : getMDBEnv ( path , MDB_NOSUBDIR , 0600 , maxDbs ) ;
2021-12-05 23:40:51 +01:00
}
2022-01-18 22:25:28 +01:00
DatabaseStorage : : DatabaseStorage ( const std : : shared_ptr < LMDBSafe : : MDBEnv > & env , PackageCache & packageCache , std : : string_view uniqueDatabaseName )
2021-12-05 23:40:51 +01:00
: packageCache ( packageCache )
, packages ( env , argsToString ( uniqueDatabaseName , " _packages " ) )
, providedDeps ( env , argsToString ( uniqueDatabaseName , " _provides " ) )
, requiredDeps ( env , argsToString ( uniqueDatabaseName , " _requires " ) )
, providedLibs ( env , argsToString ( uniqueDatabaseName , " _libprovides " ) )
, requiredLibs ( env , argsToString ( uniqueDatabaseName , " _librequires " ) )
, m_env ( env )
{
2022-06-19 22:47:31 +02:00
std : : cout < < EscapeCodes : : Phrases : : InfoMessage < < " Initialized database storage for \" " < < uniqueDatabaseName < < " \" \n " ;
2021-12-05 23:40:51 +01:00
}
std : : size_t hash_value ( const PackageCacheRef & ref )
{
const auto hasher1 = boost : : hash < const LibPkg : : DatabaseStorage * > ( ) ;
const auto hasher2 = boost : : hash < std : : string > ( ) ;
2022-01-19 23:27:14 +01:00
return ( ( hasher1 ( ref . relatedStorage ) ^ ( hasher2 ( * ref . entryName ) < < 1 ) ) > > 1 ) ;
2021-12-05 23:40:51 +01:00
}
2022-01-20 23:33:02 +01:00
std : : size_t hash_value ( const PackageCacheEntryByID & entryByID )
{
const auto hasher1 = boost : : hash < StorageID > ( ) ;
const auto hasher2 = boost : : hash < const LibPkg : : DatabaseStorage * > ( ) ;
return ( ( hasher1 ( entryByID . id ) ^ ( hasher2 ( entryByID . storage ) < < 1 ) ) > > 1 ) ;
}
2021-12-05 23:40:51 +01:00
} // namespace LibPkg