bdb.cpp raw
1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2022 The Limenka developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
5
6 #include <compat/compat.h>
7 #include <logging.h>
8 #include <util/fs.h>
9 #include <util/time.h>
10 #include <wallet/bdb.h>
11 #include <wallet/db.h>
12
13 #include <sync.h>
14 #include <util/check.h>
15 #include <util/fs_helpers.h>
16 #include <util/strencodings.h>
17 #include <util/translation.h>
18
19 #include <set>
20 #include <stdint.h>
21
22 #include <db_cxx.h>
23 #include <sys/stat.h>
24
25 // Windows may not define S_IRUSR or S_IWUSR. We define both
26 // here, with the same values as glibc (see stat.h).
27 #ifdef WIN32
28 #ifndef S_IRUSR
29 #define S_IRUSR 0400
30 #define S_IWUSR 0200
31 #endif
32 #endif
33
34 static_assert(BDB_DB_FILE_ID_LEN == DB_FILE_ID_LEN, "DB_FILE_ID_LEN should be 20.");
35
36 namespace wallet {
37 namespace {
38
39 //! Make sure database has a unique fileid within the environment. If it
40 //! doesn't, throw an error. BDB caches do not work properly when more than one
41 //! open database has the same fileid (values written to one database may show
42 //! up in reads to other databases).
43 //!
44 //! BerkeleyDB generates unique fileids by default
45 //! (https://docs.oracle.com/cd/E17275_01/html/programmer_reference/program_copy.html),
46 //! so limenka should never create different databases with the same fileid, but
47 //! this error can be triggered if users manually copy database files.
48 void CheckUniqueFileid(const BerkeleyEnvironment& env, const std::string& filename, Db& db, WalletDatabaseFileId& fileid)
49 {
50 if (env.IsMock()) return;
51
52 int ret = db.get_mpf()->get_fileid(fileid.value);
53 if (ret != 0) {
54 throw std::runtime_error(strprintf("BerkeleyDatabase: Can't open database %s (get_fileid failed with %d)", filename, ret));
55 }
56
57 for (const auto& item : env.m_fileids) {
58 if (fileid == item.second && &fileid != &item.second) {
59 throw std::runtime_error(strprintf("BerkeleyDatabase: Can't open database %s (duplicates fileid %s from %s)", filename,
60 HexStr(item.second.value), item.first));
61 }
62 }
63 }
64
65 RecursiveMutex cs_db;
66 std::map<std::string, std::weak_ptr<BerkeleyEnvironment>> g_dbenvs GUARDED_BY(cs_db); //!< Map from directory name to db environment.
67 } // namespace
68
69 static constexpr auto REVERSE_BYTE_ORDER{std::endian::native == std::endian::little ? 4321 : 1234};
70
71 bool WalletDatabaseFileId::operator==(const WalletDatabaseFileId& rhs) const
72 {
73 return memcmp(value, &rhs.value, sizeof(value)) == 0;
74 }
75
76 /**
77 * @param[in] env_directory Path to environment directory
78 * @return A shared pointer to the BerkeleyEnvironment object for the wallet directory, never empty because ~BerkeleyEnvironment
79 * erases the weak pointer from the g_dbenvs map.
80 * @post A new BerkeleyEnvironment weak pointer is inserted into g_dbenvs if the directory path key was not already in the map.
81 */
82 std::shared_ptr<BerkeleyEnvironment> GetBerkeleyEnv(const fs::path& env_directory, bool use_shared_memory)
83 {
84 LOCK(cs_db);
85 auto inserted = g_dbenvs.emplace(fs::PathToString(env_directory), std::weak_ptr<BerkeleyEnvironment>());
86 if (inserted.second) {
87 auto env = std::make_shared<BerkeleyEnvironment>(env_directory, use_shared_memory);
88 inserted.first->second = env;
89 return env;
90 }
91 return inserted.first->second.lock();
92 }
93
94 //
95 // BerkeleyBatch
96 //
97
98 void BerkeleyEnvironment::Close()
99 {
100 if (!fDbEnvInit)
101 return;
102
103 fDbEnvInit = false;
104
105 for (auto& db : m_databases) {
106 BerkeleyDatabase& database = db.second.get();
107 assert(database.m_refcount <= 0);
108 if (database.m_db) {
109 database.m_db->close(0);
110 database.m_db.reset();
111 }
112 }
113
114 FILE* error_file = nullptr;
115 dbenv->get_errfile(&error_file);
116
117 int ret = dbenv->close(0);
118 if (ret != 0)
119 LogWarning("BerkeleyEnvironment::Close: Error %d closing database environment: %s", ret, DbEnv::strerror(ret));
120 if (!fMockDb)
121 DbEnv(uint32_t{0}).remove(strPath.c_str(), 0);
122
123 if (error_file) fclose(error_file);
124
125 UnlockDirectory(fs::PathFromString(strPath), ".walletlock");
126 }
127
128 void BerkeleyEnvironment::Reset()
129 {
130 dbenv.reset(new DbEnv(DB_CXX_NO_EXCEPTIONS));
131 fDbEnvInit = false;
132 fMockDb = false;
133 }
134
135 BerkeleyEnvironment::BerkeleyEnvironment(const fs::path& dir_path, bool use_shared_memory) : strPath(fs::PathToString(dir_path)), m_use_shared_memory(use_shared_memory)
136 {
137 Reset();
138 }
139
140 BerkeleyEnvironment::~BerkeleyEnvironment()
141 {
142 LOCK(cs_db);
143 g_dbenvs.erase(strPath);
144 Close();
145 }
146
147 bool BerkeleyEnvironment::Open(bilingual_str& err)
148 {
149 if (fDbEnvInit) {
150 return true;
151 }
152
153 fs::path pathIn = fs::PathFromString(strPath);
154 TryCreateDirectories(pathIn);
155 if (!IsDirWritable(pathIn)) {
156 throw std::runtime_error(strprintf("BerkeleyEnvironment: Failed to open database in directory '%s': directory is not writable", fs::PathToString(pathIn)));
157 }
158 if (util::LockDirectory(pathIn, ".walletlock") != util::LockResult::Success) {
159 LogWarning("Cannot obtain a lock on wallet directory %s. Another instance may be using it.", strPath);
160 err = strprintf(_("Error initializing wallet database environment %s!"), fs::quoted(fs::PathToString(Directory())));
161 return false;
162 }
163
164 fs::path pathLogDir = pathIn / "database";
165 TryCreateDirectories(pathLogDir);
166 fs::path pathErrorFile = pathIn / "db.log";
167 LogPrintf("BerkeleyEnvironment::Open: LogDir=%s ErrorFile=%s\n", fs::PathToString(pathLogDir), fs::PathToString(pathErrorFile));
168
169 unsigned int nEnvFlags = 0;
170 if (!m_use_shared_memory) {
171 nEnvFlags |= DB_PRIVATE;
172 }
173
174 dbenv->set_lg_dir(fs::PathToString(pathLogDir).c_str());
175 dbenv->set_cachesize(0, 0x100000, 1); // 1 MiB should be enough for just the wallet
176 dbenv->set_lg_bsize(0x10000);
177 dbenv->set_lg_max(1048576);
178 dbenv->set_lk_max_locks(40000);
179 dbenv->set_lk_max_objects(40000);
180 dbenv->set_errfile(fsbridge::fopen(pathErrorFile, "a")); /// debug
181 dbenv->set_flags(DB_AUTO_COMMIT, 1);
182 dbenv->set_flags(DB_TXN_WRITE_NOSYNC, 1);
183 dbenv->log_set_config(DB_LOG_AUTO_REMOVE, 1);
184 int ret = dbenv->open(strPath.c_str(),
185 DB_CREATE |
186 DB_INIT_LOCK |
187 DB_INIT_LOG |
188 DB_INIT_MPOOL |
189 DB_INIT_TXN |
190 DB_THREAD |
191 DB_RECOVER |
192 nEnvFlags,
193 S_IRUSR | S_IWUSR);
194 if (ret != 0) {
195 LogWarning("BerkeleyEnvironment::Open: Error %d opening database environment: %s", ret, DbEnv::strerror(ret));
196 int ret2 = dbenv->close(0);
197 if (ret2 != 0) {
198 LogWarning("BerkeleyEnvironment::Open: Error %d closing failed database environment: %s", ret2, DbEnv::strerror(ret2));
199 }
200 Reset();
201 err = strprintf(_("Error initializing wallet database environment %s!"), fs::quoted(fs::PathToString(Directory())));
202 if (ret == DB_RUNRECOVERY) {
203 err += Untranslated(" ") + _("This error could occur if this wallet was last loaded using a build with a newer version of Berkeley DB.");
204 }
205 return false;
206 }
207
208 fDbEnvInit = true;
209 fMockDb = false;
210 return true;
211 }
212
213 //! Construct an in-memory mock Berkeley environment for testing
214 BerkeleyEnvironment::BerkeleyEnvironment() : m_use_shared_memory(false)
215 {
216 Reset();
217
218 LogDebug(BCLog::WALLETDB, "BerkeleyEnvironment::MakeMock\n");
219
220 dbenv->set_cachesize(1, 0, 1);
221 dbenv->set_lg_bsize(10485760 * 4);
222 dbenv->set_lg_max(10485760);
223 dbenv->set_lk_max_locks(10000);
224 dbenv->set_lk_max_objects(10000);
225 dbenv->set_flags(DB_AUTO_COMMIT, 1);
226 dbenv->log_set_config(DB_LOG_IN_MEMORY, 1);
227 int ret = dbenv->open(nullptr,
228 DB_CREATE |
229 DB_INIT_LOCK |
230 DB_INIT_LOG |
231 DB_INIT_MPOOL |
232 DB_INIT_TXN |
233 DB_THREAD |
234 DB_PRIVATE,
235 S_IRUSR | S_IWUSR);
236 if (ret > 0) {
237 throw std::runtime_error(strprintf("BerkeleyEnvironment::MakeMock: Error %d opening database environment.", ret));
238 }
239
240 fDbEnvInit = true;
241 fMockDb = true;
242 }
243
244 /** RAII class that automatically cleanses its data on destruction */
245 class SafeDbt final
246 {
247 Dbt m_dbt;
248
249 public:
250 // construct Dbt with internally-managed data
251 SafeDbt();
252 // construct Dbt with provided data
253 SafeDbt(void* data, size_t size);
254 ~SafeDbt();
255
256 // delegate to Dbt
257 const void* get_data() const;
258 uint32_t get_size() const;
259
260 // conversion operator to access the underlying Dbt
261 operator Dbt*();
262 };
263
264 SafeDbt::SafeDbt()
265 {
266 m_dbt.set_flags(DB_DBT_MALLOC);
267 }
268
269 SafeDbt::SafeDbt(void* data, size_t size)
270 : m_dbt(data, size)
271 {
272 }
273
274 SafeDbt::~SafeDbt()
275 {
276 if (m_dbt.get_data() != nullptr) {
277 // Clear memory, e.g. in case it was a private key
278 memory_cleanse(m_dbt.get_data(), m_dbt.get_size());
279 // under DB_DBT_MALLOC, data is malloced by the Dbt, but must be
280 // freed by the caller.
281 // https://docs.oracle.com/cd/E17275_01/html/api_reference/C/dbt.html
282 if (m_dbt.get_flags() & DB_DBT_MALLOC) {
283 free(m_dbt.get_data());
284 }
285 }
286 }
287
288 const void* SafeDbt::get_data() const
289 {
290 return m_dbt.get_data();
291 }
292
293 uint32_t SafeDbt::get_size() const
294 {
295 return m_dbt.get_size();
296 }
297
298 SafeDbt::operator Dbt*()
299 {
300 return &m_dbt;
301 }
302
303 static Span<const std::byte> SpanFromDbt(const SafeDbt& dbt)
304 {
305 return {reinterpret_cast<const std::byte*>(dbt.get_data()), dbt.get_size()};
306 }
307
308 BerkeleyDatabase::BerkeleyDatabase(std::shared_ptr<BerkeleyEnvironment> env, fs::path filename, const DatabaseOptions& options) :
309 WalletDatabase(),
310 env(std::move(env)),
311 m_byteswap(options.require_format == DatabaseFormat::BERKELEY_SWAP),
312 m_filename(std::move(filename)),
313 m_max_log_mb(options.max_log_mb)
314 {
315 auto inserted = this->env->m_databases.emplace(m_filename, std::ref(*this));
316 assert(inserted.second);
317 }
318
319 bool BerkeleyDatabase::Verify(bilingual_str& errorStr)
320 {
321 fs::path walletDir = env->Directory();
322 fs::path file_path = walletDir / m_filename;
323
324 LogPrintf("Using BerkeleyDB version %s\n", BerkeleyDatabaseVersion());
325 LogPrintf("Using wallet %s\n", fs::PathToString(file_path));
326
327 if (!env->Open(errorStr)) {
328 return false;
329 }
330
331 if (fs::exists(file_path))
332 {
333 assert(m_refcount == 0);
334
335 Db db(env->dbenv.get(), 0);
336 const std::string strFile = fs::PathToString(m_filename);
337 int result = db.verify(strFile.c_str(), nullptr, nullptr, 0);
338 if (result != 0) {
339 errorStr = strprintf(_("%s corrupt. Try using the wallet tool limenka-wallet to salvage or restoring a backup."), fs::quoted(fs::PathToString(file_path)));
340 return false;
341 }
342 }
343 // also return true if files does not exists
344 return true;
345 }
346
347 std::vector<fs::path> BerkeleyDatabase::Files()
348 {
349 std::vector<fs::path> files;
350 // If the wallet is the *only* file, clean up the entire BDB environment
351 constexpr auto build_files_list = [](std::vector<fs::path>& files, const std::shared_ptr<BerkeleyEnvironment>& env, const fs::path& filename) {
352 if (env->m_databases.size() != 1) return false;
353
354 const auto env_dir = env->Directory();
355 const auto db_subdir = env_dir / "database";
356 if (fs::exists(db_subdir)) {
357 if (!fs::is_directory(db_subdir)) return false;
358 for (const auto& entry : fs::directory_iterator(db_subdir)) {
359 const auto& path = entry.path().filename();
360 if (!fs::PathToString(path).starts_with("log.")) {
361 return false;
362 }
363 files.emplace_back(entry.path());
364 }
365 }
366 const std::set<fs::path> allowed_paths = {
367 filename,
368 "db.log",
369 ".walletlock",
370 "database"
371 };
372 for (const auto& entry : fs::directory_iterator(env_dir)) {
373 const auto& path = entry.path().filename();
374 if (allowed_paths.contains(path)) {
375 files.emplace_back(entry.path());
376 } else if (fs::is_directory(entry.path())) {
377 // Subdirectories can't possibly be using this db env, and is expected if this is a non-directory wallet
378 // Do not include them in Files, but still allow the env cleanup
379 } else {
380 return false;
381 }
382 }
383 return true;
384 };
385 try {
386 if (build_files_list(files, env, m_filename)) return files;
387 } catch (...) {
388 // Give up building the comprehensive file list if any error occurs
389 }
390 // Otherwise, it's only really safe to delete the one wallet file
391 return {env->Directory() / m_filename};
392 }
393
394 bool BerkeleyEnvironment::CheckpointLSN(const std::string& strFile)
395 {
396 if (dbenv->txn_checkpoint(0, 0, 0) != 0) {
397 return false;
398 }
399 if (!fMockDb) {
400 if (dbenv->lsn_reset(strFile.c_str(), 0) != 0) {
401 return false;
402 }
403 }
404 return true;
405 }
406
407 BerkeleyDatabase::~BerkeleyDatabase()
408 {
409 if (env) {
410 LOCK(cs_db);
411 env->CloseDb(m_filename);
412 assert(!m_db);
413 size_t erased = env->m_databases.erase(m_filename);
414 assert(erased == 1);
415 env->m_fileids.erase(fs::PathToString(m_filename));
416 }
417 }
418
419 BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase& database, const bool read_only, bool fFlushOnCloseIn) : m_database(database)
420 {
421 database.AddRef();
422 database.Open();
423 fReadOnly = read_only;
424 fFlushOnClose = fFlushOnCloseIn;
425 env = database.env.get();
426 pdb = database.m_db.get();
427 strFile = fs::PathToString(database.m_filename);
428 }
429
430 void BerkeleyDatabase::Open()
431 {
432 unsigned int nFlags = DB_THREAD | DB_CREATE;
433
434 {
435 LOCK(cs_db);
436 bilingual_str open_err;
437 if (!env->Open(open_err))
438 throw std::runtime_error("BerkeleyDatabase: Failed to open database environment.");
439
440 if (m_db == nullptr) {
441 int ret;
442 std::unique_ptr<Db> pdb_temp = std::make_unique<Db>(env->dbenv.get(), 0);
443 const std::string strFile = fs::PathToString(m_filename);
444
445 bool fMockDb = env->IsMock();
446 if (fMockDb) {
447 DbMpoolFile* mpf = pdb_temp->get_mpf();
448 ret = mpf->set_flags(DB_MPOOL_NOFILE, 1);
449 if (ret != 0) {
450 throw std::runtime_error(strprintf("BerkeleyDatabase: Failed to configure for no temp file backing for database %s", strFile));
451 }
452 }
453
454 if (m_byteswap) {
455 pdb_temp->set_lorder(REVERSE_BYTE_ORDER);
456 }
457
458 ret = pdb_temp->open(nullptr, // Txn pointer
459 fMockDb ? nullptr : strFile.c_str(), // Filename
460 fMockDb ? strFile.c_str() : "main", // Logical db name
461 DB_BTREE, // Database type
462 nFlags, // Flags
463 0);
464
465 if (ret != 0) {
466 throw std::runtime_error(strprintf("BerkeleyDatabase: Error %d, can't open database %s", ret, strFile));
467 }
468
469 // Call CheckUniqueFileid on the containing BDB environment to
470 // avoid BDB data consistency bugs that happen when different data
471 // files in the same environment have the same fileid.
472 CheckUniqueFileid(*env, strFile, *pdb_temp, this->env->m_fileids[strFile]);
473
474 m_db.reset(pdb_temp.release());
475
476 }
477 }
478 }
479
480 void BerkeleyBatch::Flush()
481 {
482 if (activeTxn)
483 return;
484
485 // Flush database activity from memory pool to disk log
486 unsigned int nMinutes = 0;
487 if (fReadOnly)
488 nMinutes = 1;
489
490 if (env) { // env is nullptr for dummy databases (i.e. in tests). Don't actually flush if env is nullptr so we don't segfault
491 env->dbenv->txn_checkpoint(nMinutes ? m_database.m_max_log_mb * 1024 : 0, nMinutes, 0);
492 }
493 }
494
495 void BerkeleyDatabase::IncrementUpdateCounter()
496 {
497 ++nUpdateCounter;
498 }
499
500 BerkeleyBatch::~BerkeleyBatch()
501 {
502 Close();
503 m_database.RemoveRef();
504 }
505
506 void BerkeleyBatch::Close()
507 {
508 if (!pdb)
509 return;
510 if (activeTxn)
511 activeTxn->abort();
512 activeTxn = nullptr;
513 pdb = nullptr;
514
515 if (fFlushOnClose)
516 Flush();
517 }
518
519 void BerkeleyEnvironment::CloseDb(const fs::path& filename)
520 {
521 {
522 LOCK(cs_db);
523 auto it = m_databases.find(filename);
524 assert(it != m_databases.end());
525 BerkeleyDatabase& database = it->second.get();
526 if (database.m_db) {
527 // Close the database handle
528 database.m_db->close(0);
529 database.m_db.reset();
530 }
531 }
532 }
533
534 void BerkeleyEnvironment::ReloadDbEnv()
535 {
536 // Make sure that no Db's are in use
537 AssertLockNotHeld(cs_db);
538 std::unique_lock<RecursiveMutex> lock(cs_db);
539 m_db_in_use.wait(lock, [this](){
540 for (auto& db : m_databases) {
541 if (db.second.get().m_refcount > 0) return false;
542 }
543 return true;
544 });
545
546 std::vector<fs::path> filenames;
547 filenames.reserve(m_databases.size());
548 for (const auto& it : m_databases) {
549 filenames.push_back(it.first);
550 }
551 // Close the individual Db's
552 for (const fs::path& filename : filenames) {
553 CloseDb(filename);
554 }
555 // Reset the environment
556 Flush(true); // This will flush and close the environment
557 Reset();
558 bilingual_str open_err;
559 Open(open_err);
560 }
561
562 DbTxn* BerkeleyEnvironment::TxnBegin(int flags)
563 {
564 DbTxn* ptxn = nullptr;
565 int ret = dbenv->txn_begin(nullptr, &ptxn, flags);
566 if (!ptxn || ret != 0)
567 return nullptr;
568 return ptxn;
569 }
570
571 bool BerkeleyDatabase::Rewrite(const char* pszSkip)
572 {
573 while (true) {
574 {
575 LOCK(cs_db);
576 const std::string strFile = fs::PathToString(m_filename);
577 if (m_refcount <= 0) {
578 // Flush log data to the dat file
579 env->CloseDb(m_filename);
580 if (!env->CheckpointLSN(strFile)) {
581 LogPrintLevel(BCLog::WALLETDB, BCLog::Level::Error, "%s: Failed to checkpoint database file %s\n", __func__, strFile);
582 return false;
583 }
584 m_refcount = -1;
585
586 bool fSuccess = true;
587 LogPrintf("BerkeleyBatch::Rewrite: Rewriting %s...\n", strFile);
588 std::string strFileRes = strFile + ".rewrite";
589 { // surround usage of db with extra {}
590 BerkeleyBatch db(*this, true);
591 std::unique_ptr<Db> pdbCopy = std::make_unique<Db>(env->dbenv.get(), 0);
592
593 if (m_byteswap) {
594 pdbCopy->set_lorder(REVERSE_BYTE_ORDER);
595 }
596
597 int ret = pdbCopy->open(nullptr, // Txn pointer
598 strFileRes.c_str(), // Filename
599 "main", // Logical db name
600 DB_BTREE, // Database type
601 DB_CREATE, // Flags
602 0);
603 if (ret > 0) {
604 LogWarning("BerkeleyBatch::Rewrite: Can't create database file %s", strFileRes);
605 fSuccess = false;
606 }
607
608 std::unique_ptr<DatabaseCursor> cursor = db.GetNewCursor();
609 if (cursor) {
610 while (fSuccess) {
611 DataStream ssKey{};
612 DataStream ssValue{};
613 DatabaseCursor::Status ret1 = cursor->Next(ssKey, ssValue);
614 if (ret1 == DatabaseCursor::Status::DONE) {
615 break;
616 } else if (ret1 == DatabaseCursor::Status::FAIL) {
617 fSuccess = false;
618 break;
619 }
620 if (pszSkip &&
621 strncmp((const char*)ssKey.data(), pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0)
622 continue;
623 if (strncmp((const char*)ssKey.data(), "\x07version", 8) == 0) {
624 // Update version:
625 ssValue.clear();
626 ssValue << CLIENT_VERSION;
627 }
628 Dbt datKey(ssKey.data(), ssKey.size());
629 Dbt datValue(ssValue.data(), ssValue.size());
630 int ret2 = pdbCopy->put(nullptr, &datKey, &datValue, DB_NOOVERWRITE);
631 if (ret2 > 0)
632 fSuccess = false;
633 }
634 cursor.reset();
635 }
636 if (fSuccess) {
637 db.Close();
638 env->CloseDb(m_filename);
639 if (pdbCopy->close(0))
640 fSuccess = false;
641 } else {
642 pdbCopy->close(0);
643 }
644 }
645 if (fSuccess) {
646 // Atomic rename: rename backup, then rename new to target, then remove backup
647 Db dbA(env->dbenv.get(), 0);
648 const std::string strFileBak = strFile + ".bak";
649 if (dbA.rename(strFile.c_str(), nullptr, strFileBak.c_str(), 0))
650 fSuccess = false;
651 if (fSuccess) {
652 Db dbB(env->dbenv.get(), 0);
653 if (dbB.rename(strFileRes.c_str(), nullptr, strFile.c_str(), 0))
654 fSuccess = false;
655 }
656 if (fSuccess) {
657 Db dbC(env->dbenv.get(), 0);
658 dbC.remove(strFileBak.c_str(), nullptr, 0);
659 }
660 }
661 if (!fSuccess)
662 LogWarning("BerkeleyBatch::Rewrite: Failed to rewrite database file %s", strFileRes);
663 return fSuccess;
664 }
665 }
666 UninterruptibleSleep(std::chrono::milliseconds{100});
667 }
668 }
669
670
671 void BerkeleyEnvironment::Flush(bool fShutdown)
672 {
673 const auto start{SteadyClock::now()};
674 // Flush log data to the actual data file on all files that are not in use
675 LogDebug(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: [%s] Flush(%s)%s\n", strPath, fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started");
676 if (!fDbEnvInit)
677 return;
678 {
679 LOCK(cs_db);
680 bool no_dbs_accessed = true;
681 for (auto& db_it : m_databases) {
682 const fs::path& filename = db_it.first;
683 BerkeleyDatabase& database = db_it.second.get();
684 const int nRefCount = database.m_refcount;
685 if (nRefCount < 0) continue;
686 const std::string strFile = fs::PathToString(filename);
687 LogDebug(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount);
688 if (nRefCount == 0) {
689 // Move log data to the dat file
690 CloseDb(filename);
691 LogDebug(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s checkpoint\n", strFile);
692 if (dbenv->txn_checkpoint(0, 0, 0) != 0) {
693 LogPrintLevel(BCLog::WALLETDB, BCLog::Level::Error, "%s: %s checkpoint FAILED\n", __func__, strFile);
694 no_dbs_accessed = false;
695 continue;
696 }
697 LogDebug(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s detach\n", strFile);
698 if (!fMockDb) {
699 if (dbenv->lsn_reset(strFile.c_str(), 0) != 0) {
700 LogPrintLevel(BCLog::WALLETDB, BCLog::Level::Error, "%s: %s detach FAILED\n", __func__, strFile);
701 no_dbs_accessed = false;
702 continue;
703 }
704 }
705 LogDebug(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s closed\n", strFile);
706 database.m_refcount = -1;
707 } else {
708 no_dbs_accessed = false;
709 }
710 }
711 LogDebug(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", Ticks<std::chrono::milliseconds>(SteadyClock::now() - start));
712 if (fShutdown) {
713 char** listp;
714 if (no_dbs_accessed) {
715 dbenv->log_archive(&listp, DB_ARCH_REMOVE);
716 Close();
717 }
718 }
719 }
720 }
721
722 bool BerkeleyDatabase::PeriodicFlush()
723 {
724 // Don't flush if we can't acquire the lock.
725 TRY_LOCK(cs_db, lockDb);
726 if (!lockDb) return false;
727
728 // Don't flush if any databases are in use
729 for (auto& it : env->m_databases) {
730 if (it.second.get().m_refcount > 0) return false;
731 }
732
733 // Don't flush if there haven't been any batch writes for this database.
734 if (m_refcount < 0) return false;
735
736 const std::string strFile = fs::PathToString(m_filename);
737 LogDebug(BCLog::WALLETDB, "Flushing %s\n", strFile);
738 const auto start{SteadyClock::now()};
739
740 // Flush wallet file so it's self contained
741 env->CloseDb(m_filename);
742 if (!env->CheckpointLSN(strFile)) {
743 LogPrintLevel(BCLog::WALLETDB, BCLog::Level::Error, "%s: FAILED to flush wallet %s\n", __func__, strFile);
744 return false;
745 }
746 m_refcount = -1;
747
748 LogDebug(BCLog::WALLETDB, "Flushed %s %dms\n", strFile, Ticks<std::chrono::milliseconds>(SteadyClock::now() - start));
749
750 return true;
751 }
752
753 bool BerkeleyDatabase::Backup(const std::string& strDest) const
754 {
755 const std::string strFile = fs::PathToString(m_filename);
756 while (true)
757 {
758 {
759 LOCK(cs_db);
760 if (m_refcount <= 0)
761 {
762 // Flush log data to the dat file
763 env->CloseDb(m_filename);
764 if (!env->CheckpointLSN(strFile)) {
765 LogPrintLevel(BCLog::WALLETDB, BCLog::Level::Error, "%s: FAILED to flush wallet %s\n", __func__, strFile);
766 return false;
767 }
768
769 // Copy wallet file
770 fs::path pathSrc = env->Directory() / m_filename;
771 fs::path pathDest(fs::PathFromString(strDest));
772 if (fs::is_directory(pathDest))
773 pathDest /= m_filename;
774
775 try {
776 if (fs::exists(pathDest) && fs::equivalent(pathSrc, pathDest)) {
777 LogWarning("cannot backup to wallet source file %s", fs::PathToString(pathDest));
778 return false;
779 }
780
781 fs::copy_file(pathSrc, pathDest, fs::copy_options::overwrite_existing);
782 LogPrintf("copied %s to %s\n", strFile, fs::PathToString(pathDest));
783 return true;
784 } catch (const fs::filesystem_error& e) {
785 LogWarning("error copying %s to %s - %s", strFile, fs::PathToString(pathDest), fsbridge::get_filesystem_error_message(e));
786 return false;
787 }
788 }
789 }
790 UninterruptibleSleep(std::chrono::milliseconds{100});
791 }
792 }
793
794 void BerkeleyDatabase::Flush()
795 {
796 env->Flush(false);
797 }
798
799 void BerkeleyDatabase::Close()
800 {
801 env->Flush(true);
802 }
803
804 void BerkeleyDatabase::ReloadDbEnv()
805 {
806 env->ReloadDbEnv();
807 }
808
809 BerkeleyCursor::BerkeleyCursor(BerkeleyDatabase& database, const BerkeleyBatch& batch, Span<const std::byte> prefix)
810 : m_key_prefix(prefix.begin(), prefix.end())
811 {
812 if (!database.m_db.get()) {
813 throw std::runtime_error(STR_INTERNAL_BUG("BerkeleyDatabase does not exist"));
814 }
815 // Transaction argument to cursor is only needed when using the cursor to
816 // write to the database. Read-only cursors do not need a txn pointer.
817 int ret = database.m_db->cursor(batch.txn(), &m_cursor, 0);
818 if (ret != 0) {
819 throw std::runtime_error(STR_INTERNAL_BUG(strprintf("BDB Cursor could not be created. Returned %d", ret)));
820 }
821 }
822
823 DatabaseCursor::Status BerkeleyCursor::Next(DataStream& ssKey, DataStream& ssValue)
824 {
825 if (m_cursor == nullptr) return Status::FAIL;
826 // Read at cursor
827 SafeDbt datKey(m_key_prefix.data(), m_key_prefix.size());
828 SafeDbt datValue;
829 int ret = -1;
830 if (m_first && !m_key_prefix.empty()) {
831 ret = m_cursor->get(datKey, datValue, DB_SET_RANGE);
832 } else {
833 ret = m_cursor->get(datKey, datValue, DB_NEXT);
834 }
835 m_first = false;
836 if (ret == DB_NOTFOUND) {
837 return Status::DONE;
838 }
839 if (ret != 0) {
840 return Status::FAIL;
841 }
842
843 Span<const std::byte> raw_key = SpanFromDbt(datKey);
844 if (!m_key_prefix.empty() && std::mismatch(raw_key.begin(), raw_key.end(), m_key_prefix.begin(), m_key_prefix.end()).second != m_key_prefix.end()) {
845 return Status::DONE;
846 }
847
848 // Convert to streams
849 ssKey.clear();
850 ssKey.write(raw_key);
851 ssValue.clear();
852 ssValue.write(SpanFromDbt(datValue));
853 return Status::MORE;
854 }
855
856 BerkeleyCursor::~BerkeleyCursor()
857 {
858 if (!m_cursor) return;
859 m_cursor->close();
860 m_cursor = nullptr;
861 }
862
863 std::unique_ptr<DatabaseCursor> BerkeleyBatch::GetNewCursor()
864 {
865 if (!pdb) return nullptr;
866 return std::make_unique<BerkeleyCursor>(m_database, *this);
867 }
868
869 std::unique_ptr<DatabaseCursor> BerkeleyBatch::GetNewPrefixCursor(Span<const std::byte> prefix)
870 {
871 if (!pdb) return nullptr;
872 return std::make_unique<BerkeleyCursor>(m_database, *this, prefix);
873 }
874
875 bool BerkeleyBatch::TxnBegin()
876 {
877 if (!pdb || activeTxn)
878 return false;
879 DbTxn* ptxn = env->TxnBegin(DB_TXN_WRITE_NOSYNC);
880 if (!ptxn)
881 return false;
882 activeTxn = ptxn;
883 return true;
884 }
885
886 bool BerkeleyBatch::TxnCommit()
887 {
888 if (!pdb || !activeTxn)
889 return false;
890 int ret = activeTxn->commit(0);
891 activeTxn = nullptr;
892 return (ret == 0);
893 }
894
895 bool BerkeleyBatch::TxnAbort()
896 {
897 if (!pdb || !activeTxn)
898 return false;
899 int ret = activeTxn->abort();
900 activeTxn = nullptr;
901 return (ret == 0);
902 }
903
904 bool BerkeleyDatabaseSanityCheck()
905 {
906 int major, minor;
907 DbEnv::version(&major, &minor, nullptr);
908
909 /* If the major version differs, or the minor version of library is *older*
910 * than the header that was compiled against, flag an error.
911 */
912 if (major != DB_VERSION_MAJOR || minor < DB_VERSION_MINOR) {
913 LogError("BerkeleyDB database version conflict: header version is %d.%d, library version is %d.%d",
914 DB_VERSION_MAJOR, DB_VERSION_MINOR, major, minor);
915 return false;
916 }
917
918 return true;
919 }
920
921 std::string BerkeleyDatabaseVersion()
922 {
923 return DbEnv::version(nullptr, nullptr, nullptr);
924 }
925
926 bool BerkeleyBatch::ReadKey(DataStream&& key, DataStream& value)
927 {
928 if (!pdb)
929 return false;
930
931 SafeDbt datKey(key.data(), key.size());
932
933 SafeDbt datValue;
934 int ret = pdb->get(activeTxn, datKey, datValue, 0);
935 if (ret == 0 && datValue.get_data() != nullptr) {
936 value.clear();
937 value.write(SpanFromDbt(datValue));
938 return true;
939 }
940 return false;
941 }
942
943 bool BerkeleyBatch::WriteKey(DataStream&& key, DataStream&& value, bool overwrite)
944 {
945 if (!pdb)
946 return false;
947 if (fReadOnly)
948 assert(!"Write called on database in read-only mode");
949
950 SafeDbt datKey(key.data(), key.size());
951
952 SafeDbt datValue(value.data(), value.size());
953
954 int ret = pdb->put(activeTxn, datKey, datValue, (overwrite ? 0 : DB_NOOVERWRITE));
955 return (ret == 0);
956 }
957
958 bool BerkeleyBatch::EraseKey(DataStream&& key)
959 {
960 if (!pdb)
961 return false;
962 if (fReadOnly)
963 assert(!"Erase called on database in read-only mode");
964
965 SafeDbt datKey(key.data(), key.size());
966
967 int ret = pdb->del(activeTxn, datKey, 0);
968 return (ret == 0 || ret == DB_NOTFOUND);
969 }
970
971 bool BerkeleyBatch::HasKey(DataStream&& key)
972 {
973 if (!pdb)
974 return false;
975
976 SafeDbt datKey(key.data(), key.size());
977
978 int ret = pdb->exists(activeTxn, datKey, 0);
979 return ret == 0;
980 }
981
982 bool BerkeleyBatch::ErasePrefix(Span<const std::byte> prefix)
983 {
984 // Because this function erases records one by one, ensure that it is executed within a txn context.
985 // Otherwise, consistency is at risk; it's possible that certain records are removed while others
986 // remain due to an internal failure during the procedure.
987 // Additionally, the Dbc::del() cursor delete call below would fail without an active transaction.
988 if (!Assume(activeTxn)) return false;
989
990 auto cursor{std::make_unique<BerkeleyCursor>(m_database, *this)};
991 // const_cast is safe below even though prefix_key is an in/out parameter,
992 // because we are not using the DB_DBT_USERMEM flag, so BDB will allocate
993 // and return a different output data pointer
994 Dbt prefix_key{const_cast<std::byte*>(prefix.data()), static_cast<uint32_t>(prefix.size())}, prefix_value{};
995 int ret{cursor->dbc()->get(&prefix_key, &prefix_value, DB_SET_RANGE)};
996 for (int flag{DB_CURRENT}; ret == 0; flag = DB_NEXT) {
997 SafeDbt key, value;
998 ret = cursor->dbc()->get(key, value, flag);
999 if (ret != 0 || key.get_size() < prefix.size() || memcmp(key.get_data(), prefix.data(), prefix.size()) != 0) break;
1000 ret = cursor->dbc()->del(0);
1001 }
1002 cursor.reset();
1003 return ret == 0 || ret == DB_NOTFOUND;
1004 }
1005
1006 void BerkeleyDatabase::AddRef()
1007 {
1008 LOCK(cs_db);
1009 if (m_refcount < 0) {
1010 m_refcount = 1;
1011 } else {
1012 m_refcount++;
1013 }
1014 }
1015
1016 void BerkeleyDatabase::RemoveRef()
1017 {
1018 LOCK(cs_db);
1019 m_refcount--;
1020 if (env) env->m_db_in_use.notify_all();
1021 }
1022
1023 std::unique_ptr<DatabaseBatch> BerkeleyDatabase::MakeBatch(bool flush_on_close)
1024 {
1025 return std::make_unique<BerkeleyBatch>(*this, false, flush_on_close);
1026 }
1027
1028 std::unique_ptr<BerkeleyDatabase> MakeBerkeleyDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error)
1029 {
1030 fs::path data_file = BDBDataFile(path);
1031 std::unique_ptr<BerkeleyDatabase> db;
1032 try {
1033 LOCK(cs_db); // Lock env.m_databases until insert in BerkeleyDatabase constructor
1034 fs::path data_filename = data_file.filename();
1035 std::shared_ptr<BerkeleyEnvironment> env = GetBerkeleyEnv(data_file.parent_path(), options.use_shared_memory);
1036 if (env->m_databases.count(data_filename)) {
1037 error = Untranslated(strprintf("Refusing to load database. Data file '%s' is already loaded.", fs::PathToString(env->Directory() / data_filename)));
1038 status = DatabaseStatus::FAILED_ALREADY_LOADED;
1039 return nullptr;
1040 }
1041 db = std::make_unique<BerkeleyDatabase>(std::move(env), std::move(data_filename), options);
1042 } catch (const std::runtime_error& e) {
1043 status = DatabaseStatus::FAILED_LOAD;
1044 error = Untranslated(e.what());
1045 return nullptr;
1046 }
1047
1048 try {
1049 if (options.verify && !db->Verify(error)) {
1050 status = DatabaseStatus::FAILED_VERIFY;
1051 return nullptr;
1052 }
1053 } catch (const std::runtime_error& e) {
1054 status = DatabaseStatus::FAILED_VERIFY;
1055 error = Untranslated(e.what());
1056 return nullptr;
1057 }
1058
1059 status = DatabaseStatus::SUCCESS;
1060 return db;
1061 }
1062 } // namespace wallet
1063