db_tests.cpp raw
1 // Copyright (c) 2018-2022 The Limenka developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4
5 #include <limenka-build-config.h> // IWYU pragma: keep
6
7 #include <boost/test/unit_test.hpp>
8
9 #include <test/util/setup_common.h>
10 #include <util/check.h>
11 #include <util/fs.h>
12 #include <util/translation.h>
13 #ifdef USE_BDB
14 #include <wallet/bdb.h>
15 #endif
16 #ifdef USE_SQLITE
17 #include <wallet/sqlite.h>
18 #endif
19 #include <wallet/migrate.h>
20 #include <wallet/test/util.h>
21 #include <wallet/walletutil.h> // for WALLET_FLAG_DESCRIPTORS
22
23 #include <fstream>
24 #include <memory>
25 #include <string>
26
27 inline std::ostream& operator<<(std::ostream& os, const std::pair<const SerializeData, SerializeData>& kv)
28 {
29 Span key{kv.first}, value{kv.second};
30 os << "(\"" << std::string_view{reinterpret_cast<const char*>(key.data()), key.size()} << "\", \""
31 << std::string_view{reinterpret_cast<const char*>(value.data()), value.size()} << "\")";
32 return os;
33 }
34
35 namespace wallet {
36
37 inline std::span<const std::byte> StringBytes(std::string_view str)
38 {
39 return std::as_bytes(std::span{str});
40 }
41
42 static SerializeData StringData(std::string_view str)
43 {
44 auto bytes = StringBytes(str);
45 return SerializeData{bytes.begin(), bytes.end()};
46 }
47
48 static void CheckPrefix(DatabaseBatch& batch, Span<const std::byte> prefix, MockableData expected)
49 {
50 std::unique_ptr<DatabaseCursor> cursor = batch.GetNewPrefixCursor(prefix);
51 MockableData actual;
52 while (true) {
53 DataStream key, value;
54 DatabaseCursor::Status status = cursor->Next(key, value);
55 if (status == DatabaseCursor::Status::DONE) break;
56 BOOST_CHECK(status == DatabaseCursor::Status::MORE);
57 BOOST_CHECK(
58 actual.emplace(SerializeData(key.begin(), key.end()), SerializeData(value.begin(), value.end())).second);
59 }
60 BOOST_CHECK_EQUAL_COLLECTIONS(actual.begin(), actual.end(), expected.begin(), expected.end());
61 }
62
63 BOOST_FIXTURE_TEST_SUITE(db_tests, BasicTestingSetup)
64
65 #ifdef USE_BDB
66 static std::shared_ptr<BerkeleyEnvironment> GetWalletEnv(const fs::path& path, fs::path& database_filename)
67 {
68 fs::path data_file = BDBDataFile(path);
69 database_filename = data_file.filename();
70 return GetBerkeleyEnv(data_file.parent_path(), false);
71 }
72
73 BOOST_AUTO_TEST_CASE(getwalletenv_file)
74 {
75 fs::path test_name = "test_name.dat";
76 const fs::path datadir = m_args.GetDataDirNet();
77 fs::path file_path = datadir / test_name;
78 std::ofstream f{file_path};
79 f.close();
80
81 fs::path filename;
82 std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, filename);
83 BOOST_CHECK_EQUAL(filename, test_name);
84 BOOST_CHECK_EQUAL(env->Directory(), datadir);
85 }
86
87 BOOST_AUTO_TEST_CASE(getwalletenv_directory)
88 {
89 fs::path expected_name = "wallet.dat";
90 const fs::path datadir = m_args.GetDataDirNet();
91
92 fs::path filename;
93 std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(datadir, filename);
94 BOOST_CHECK_EQUAL(filename, expected_name);
95 BOOST_CHECK_EQUAL(env->Directory(), datadir);
96 }
97
98 BOOST_AUTO_TEST_CASE(getwalletenv_g_dbenvs_multiple)
99 {
100 fs::path datadir = m_args.GetDataDirNet() / "1";
101 fs::path datadir_2 = m_args.GetDataDirNet() / "2";
102 fs::path filename;
103
104 std::shared_ptr<BerkeleyEnvironment> env_1 = GetWalletEnv(datadir, filename);
105 std::shared_ptr<BerkeleyEnvironment> env_2 = GetWalletEnv(datadir, filename);
106 std::shared_ptr<BerkeleyEnvironment> env_3 = GetWalletEnv(datadir_2, filename);
107
108 BOOST_CHECK(env_1 == env_2);
109 BOOST_CHECK(env_2 != env_3);
110 }
111
112 BOOST_AUTO_TEST_CASE(getwalletenv_g_dbenvs_free_instance)
113 {
114 fs::path datadir = gArgs.GetDataDirNet() / "1";
115 fs::path datadir_2 = gArgs.GetDataDirNet() / "2";
116 fs::path filename;
117
118 std::shared_ptr <BerkeleyEnvironment> env_1_a = GetWalletEnv(datadir, filename);
119 std::shared_ptr <BerkeleyEnvironment> env_2_a = GetWalletEnv(datadir_2, filename);
120 env_1_a.reset();
121
122 std::shared_ptr<BerkeleyEnvironment> env_1_b = GetWalletEnv(datadir, filename);
123 std::shared_ptr<BerkeleyEnvironment> env_2_b = GetWalletEnv(datadir_2, filename);
124
125 BOOST_CHECK(env_1_a != env_1_b);
126 BOOST_CHECK(env_2_a == env_2_b);
127 }
128 #endif
129
130 static std::vector<std::unique_ptr<WalletDatabase>> TestDatabases(const fs::path& path_root)
131 {
132 std::vector<std::unique_ptr<WalletDatabase>> dbs;
133 DatabaseOptions options;
134 DatabaseStatus status;
135 bilingual_str error;
136 #ifdef USE_BDB
137 dbs.emplace_back(MakeBerkeleyDatabase(path_root / "bdb", options, status, error));
138 // Needs BDB to make the DB to read
139 dbs.emplace_back(std::make_unique<BerkeleyRODatabase>(BDBDataFile(path_root / "bdb"), /*open=*/false));
140 #endif
141 #ifdef USE_SQLITE
142 dbs.emplace_back(MakeSQLiteDatabase(path_root / "sqlite", options, status, error));
143 #endif
144 dbs.emplace_back(CreateMockableWalletDatabase());
145 return dbs;
146 }
147
148 BOOST_AUTO_TEST_CASE(db_cursor_prefix_range_test)
149 {
150 // Test each supported db
151 for (const auto& database : TestDatabases(m_path_root)) {
152 std::vector<std::string> prefixes = {"", "FIRST", "SECOND", "P\xfe\xff", "P\xff\x01", "\xff\xff"};
153
154 std::unique_ptr<DatabaseBatch> handler = Assert(database)->MakeBatch();
155 if (dynamic_cast<BerkeleyRODatabase*>(database.get())) {
156 // For BerkeleyRO, open the file now. This must happen after BDB has written to the file
157 database->Open();
158 } else {
159 // Write elements to it if not berkeleyro
160 for (unsigned int i = 0; i < 10; i++) {
161 for (const auto& prefix : prefixes) {
162 BOOST_CHECK(handler->Write(std::make_pair(prefix, i), i));
163 }
164 }
165 }
166
167 // Now read all the items by prefix and verify that each element gets parsed correctly
168 for (const auto& prefix : prefixes) {
169 DataStream s_prefix;
170 s_prefix << prefix;
171 std::unique_ptr<DatabaseCursor> cursor = handler->GetNewPrefixCursor(s_prefix);
172 DataStream key;
173 DataStream value;
174 for (int i = 0; i < 10; i++) {
175 DatabaseCursor::Status status = cursor->Next(key, value);
176 BOOST_CHECK_EQUAL(status, DatabaseCursor::Status::MORE);
177
178 std::string key_back;
179 unsigned int i_back;
180 key >> key_back >> i_back;
181 BOOST_CHECK_EQUAL(key_back, prefix);
182
183 unsigned int value_back;
184 value >> value_back;
185 BOOST_CHECK_EQUAL(value_back, i_back);
186 }
187
188 // Let's now read it once more, it should return DONE
189 BOOST_CHECK(cursor->Next(key, value) == DatabaseCursor::Status::DONE);
190 }
191 handler.reset();
192 database->Close();
193 }
194 }
195
196 // Lower level DatabaseBase::GetNewPrefixCursor test, to cover cases that aren't
197 // covered in the higher level test above. The higher level test uses
198 // serialized strings which are prefixed with string length, so it doesn't test
199 // truly empty prefixes or prefixes that begin with \xff
200 BOOST_AUTO_TEST_CASE(db_cursor_prefix_byte_test)
201 {
202 const MockableData::value_type
203 e{StringData(""), StringData("e")},
204 p{StringData("prefix"), StringData("p")},
205 ps{StringData("prefixsuffix"), StringData("ps")},
206 f{StringData("\xff"), StringData("f")},
207 fs{StringData("\xffsuffix"), StringData("fs")},
208 ff{StringData("\xff\xff"), StringData("ff")},
209 ffs{StringData("\xff\xffsuffix"), StringData("ffs")};
210 for (const auto& database : TestDatabases(m_path_root)) {
211 std::unique_ptr<DatabaseBatch> batch = database->MakeBatch();
212
213 if (dynamic_cast<BerkeleyRODatabase*>(database.get())) {
214 // For BerkeleyRO, open the file now. This must happen after BDB has written to the file
215 database->Open();
216 } else {
217 // Write elements to it if not berkeleyro
218 for (const auto& [k, v] : {e, p, ps, f, fs, ff, ffs}) {
219 batch->Write(Span{k}, Span{v});
220 }
221 }
222
223 CheckPrefix(*batch, StringBytes(""), {e, p, ps, f, fs, ff, ffs});
224 CheckPrefix(*batch, StringBytes("prefix"), {p, ps});
225 CheckPrefix(*batch, StringBytes("\xff"), {f, fs, ff, ffs});
226 CheckPrefix(*batch, StringBytes("\xff\xff"), {ff, ffs});
227 batch.reset();
228 database->Close();
229 }
230 }
231
232 BOOST_AUTO_TEST_CASE(db_availability_after_write_error)
233 {
234 // Ensures the database remains accessible without deadlocking after a write error.
235 // To simulate the behavior, record overwrites are disallowed, and the test verifies
236 // that the database remains active after failing to store an existing record.
237 for (const auto& database : TestDatabases(m_path_root)) {
238 if (dynamic_cast<BerkeleyRODatabase*>(database.get())) {
239 // Skip this test if BerkeleyRO
240 continue;
241 }
242 // Write original record
243 std::unique_ptr<DatabaseBatch> batch = database->MakeBatch();
244 std::string key = "key";
245 std::string value = "value";
246 std::string value2 = "value_2";
247 BOOST_CHECK(batch->Write(key, value));
248 // Attempt to overwrite the record (expect failure)
249 BOOST_CHECK(!batch->Write(key, value2, /*fOverwrite=*/false));
250 // Successfully overwrite the record
251 BOOST_CHECK(batch->Write(key, value2, /*fOverwrite=*/true));
252 // Sanity-check; read and verify the overwritten value
253 std::string read_value;
254 BOOST_CHECK(batch->Read(key, read_value));
255 BOOST_CHECK_EQUAL(read_value, value2);
256 }
257 }
258
259 // Verify 'ErasePrefix' functionality using db keys similar to the ones used by the wallet.
260 // Keys are in the form of std::pair<TYPE, ENTRY_ID>
261 BOOST_AUTO_TEST_CASE(erase_prefix)
262 {
263 const std::string key = "key";
264 const std::string key2 = "key2";
265 const std::string value = "value";
266 const std::string value2 = "value_2";
267 auto make_key = [](std::string type, std::string id) { return std::make_pair(type, id); };
268
269 for (const auto& database : TestDatabases(m_path_root)) {
270 if (dynamic_cast<BerkeleyRODatabase*>(database.get())) {
271 // Skip this test if BerkeleyRO
272 continue;
273 }
274 std::unique_ptr<DatabaseBatch> batch = database->MakeBatch();
275
276 // Write two entries with the same key type prefix, a third one with a different prefix
277 // and a fourth one with the type-id values inverted
278 BOOST_CHECK(batch->Write(make_key(key, value), value));
279 BOOST_CHECK(batch->Write(make_key(key, value2), value2));
280 BOOST_CHECK(batch->Write(make_key(key2, value), value));
281 BOOST_CHECK(batch->Write(make_key(value, key), value));
282
283 // Erase the ones with the same prefix and verify result
284 BOOST_CHECK(batch->TxnBegin());
285 BOOST_CHECK(batch->ErasePrefix(DataStream() << key));
286 BOOST_CHECK(batch->TxnCommit());
287
288 BOOST_CHECK(!batch->Exists(make_key(key, value)));
289 BOOST_CHECK(!batch->Exists(make_key(key, value2)));
290 // Also verify that entries with a different prefix were not erased
291 BOOST_CHECK(batch->Exists(make_key(key2, value)));
292 BOOST_CHECK(batch->Exists(make_key(value, key)));
293 }
294 }
295
296 #ifdef USE_SQLITE
297
298 // Test-only statement execution error
299 constexpr int TEST_SQLITE_ERROR = -999;
300
301 class DbExecBlocker : public SQliteExecHandler
302 {
303 private:
304 SQliteExecHandler m_base_exec;
305 std::set<std::string> m_blocked_statements;
306 public:
307 DbExecBlocker(std::set<std::string> blocked_statements) : m_blocked_statements(blocked_statements) {}
308 int Exec(SQLiteDatabase& database, const std::string& statement) override {
309 if (m_blocked_statements.contains(statement)) return TEST_SQLITE_ERROR;
310 return m_base_exec.Exec(database, statement);
311 }
312 };
313
314 BOOST_AUTO_TEST_CASE(txn_close_failure_dangling_txn)
315 {
316 // Verifies that there is no active dangling, to-be-reversed db txn
317 // after the batch object that initiated it is destroyed.
318 DatabaseOptions options;
319 DatabaseStatus status;
320 bilingual_str error;
321 std::unique_ptr<SQLiteDatabase> database = MakeSQLiteDatabase(m_path_root / "sqlite", options, status, error);
322
323 std::string key = "key";
324 std::string value = "value";
325
326 std::unique_ptr<SQLiteBatch> batch = std::make_unique<SQLiteBatch>(*database);
327 BOOST_CHECK(batch->TxnBegin());
328 BOOST_CHECK(batch->Write(key, value));
329 // Set a handler to prevent txn abortion during destruction.
330 // Mimicking a db statement execution failure.
331 batch->SetExecHandler(std::make_unique<DbExecBlocker>(std::set<std::string>{"ROLLBACK TRANSACTION"}));
332 // Destroy batch
333 batch.reset();
334
335 // Ensure there is no dangling, to-be-reversed db txn
336 BOOST_CHECK(!database->HasActiveTxn());
337
338 // And, just as a sanity check; verify that new batchs only write what they suppose to write
339 // and nothing else.
340 std::string key2 = "key2";
341 std::unique_ptr<SQLiteBatch> batch2 = std::make_unique<SQLiteBatch>(*database);
342 BOOST_CHECK(batch2->Write(key2, value));
343 // The first key must not exist
344 BOOST_CHECK(!batch2->Exists(key));
345 }
346
347 BOOST_AUTO_TEST_CASE(concurrent_txn_dont_interfere)
348 {
349 std::string key = "key";
350 std::string value = "value";
351 std::string value2 = "value_2";
352
353 DatabaseOptions options;
354 DatabaseStatus status;
355 bilingual_str error;
356 const auto& database = MakeSQLiteDatabase(m_path_root / "sqlite", options, status, error);
357
358 std::unique_ptr<DatabaseBatch> handler = Assert(database)->MakeBatch();
359
360 // Verify concurrent db transactions does not interfere between each other.
361 // Start db txn, write key and check the key does exist within the db txn.
362 BOOST_CHECK(handler->TxnBegin());
363 BOOST_CHECK(handler->Write(key, value));
364 BOOST_CHECK(handler->Exists(key));
365
366 // But, the same key, does not exist in another handler
367 std::unique_ptr<DatabaseBatch> handler2 = Assert(database)->MakeBatch();
368 BOOST_CHECK(handler2->Exists(key));
369
370 // Attempt to commit the handler txn calling the handler2 methods.
371 // Which, must not be possible.
372 BOOST_CHECK(!handler2->TxnCommit());
373 BOOST_CHECK(!handler2->TxnAbort());
374
375 // Only the first handler can commit the changes.
376 BOOST_CHECK(handler->TxnCommit());
377 // And, once commit is completed, handler2 can read the record
378 std::string read_value;
379 BOOST_CHECK(handler2->Read(key, read_value));
380 BOOST_CHECK_EQUAL(read_value, value);
381
382 // Also, once txn is committed, single write statements are re-enabled.
383 // Which means that handler2 can read the record changes directly.
384 BOOST_CHECK(handler->Write(key, value2, /*fOverwrite=*/true));
385 BOOST_CHECK(handler2->Read(key, read_value));
386 BOOST_CHECK_EQUAL(read_value, value2);
387 }
388 #endif // USE_SQLITE
389
390 BOOST_AUTO_TEST_SUITE_END()
391 } // namespace wallet
392