dbwrapper.cpp raw
1 // Copyright (c) 2012-present The Bitcoin Core developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4
5 #include <dbwrapper.h>
6
7 #include <leveldb/cache.h>
8 #include <leveldb/db.h>
9 #include <leveldb/env.h>
10 #include <leveldb/filter_policy.h>
11 #include <leveldb/helpers/memenv/memenv.h>
12 #include <leveldb/iterator.h>
13 #include <leveldb/options.h>
14 #include <leveldb/slice.h>
15 #include <leveldb/status.h>
16 #include <leveldb/write_batch.h>
17 #include <random.h>
18 #include <serialize.h>
19 #include <span.h>
20 #include <streams.h>
21 #include <util/byte_units.h>
22 #include <util/fs.h>
23 #include <util/fs_helpers.h>
24 #include <util/log.h>
25 #include <util/obfuscation.h>
26 #include <util/strencodings.h>
27
28 #include <algorithm>
29 #include <cassert>
30 #include <cstdarg>
31 #include <cstdint>
32 #include <cstdio>
33 #include <memory>
34 #include <optional>
35 #include <utility>
36
37 static auto CharCast(const std::byte* data) { return reinterpret_cast<const char*>(data); }
38
39 bool DestroyDB(const std::string& path_str)
40 {
41 return leveldb::DestroyDB(path_str, {}).ok();
42 }
43
44 /** Handle database error by throwing dbwrapper_error exception.
45 */
46 static void HandleError(const leveldb::Status& status)
47 {
48 if (status.ok())
49 return;
50 const std::string errmsg = "Fatal LevelDB error: " + status.ToString();
51 LogError("%s", errmsg);
52 LogInfo("You can use -debug=leveldb to get more complete diagnostic messages");
53 throw dbwrapper_error(errmsg);
54 }
55
56 class CBitcoinLevelDBLogger : public leveldb::Logger {
57 public:
58 // This code is adapted from posix_logger.h, which is why it is using vsprintf.
59 // Please do not do this in normal code
60 void Logv(const char * format, va_list ap) override {
61 if (!util::log::ShouldDebugLog(BCLog::LEVELDB)) {
62 return;
63 }
64 char buffer[500];
65 for (int iter = 0; iter < 2; iter++) {
66 char* base;
67 int bufsize;
68 if (iter == 0) {
69 bufsize = sizeof(buffer);
70 base = buffer;
71 }
72 else {
73 bufsize = 30000;
74 base = new char[bufsize];
75 }
76 char* p = base;
77 char* limit = base + bufsize;
78
79 // Print the message
80 if (p < limit) {
81 va_list backup_ap;
82 va_copy(backup_ap, ap);
83 // Do not use vsnprintf elsewhere in bitcoin source code, see above.
84 p += vsnprintf(p, limit - p, format, backup_ap);
85 va_end(backup_ap);
86 }
87
88 // Truncate to available space if necessary
89 if (p >= limit) {
90 if (iter == 0) {
91 continue; // Try again with larger buffer
92 }
93 else {
94 p = limit - 1;
95 }
96 }
97
98 // Add newline if necessary
99 if (p == base || p[-1] != '\n') {
100 *p++ = '\n';
101 }
102
103 assert(p <= limit);
104 base[std::min(bufsize - 1, (int)(p - base))] = '\0';
105 LogDebug(BCLog::LEVELDB, "%s\n", util::RemoveSuffixView(base, "\n"));
106 if (base != buffer) {
107 delete[] base;
108 }
109 break;
110 }
111 }
112 };
113
114 static void SetMaxOpenFiles(leveldb::Options *options) {
115 // On most platforms the default setting of max_open_files (which is 1000)
116 // is optimal. On Windows using a large file count is OK because the handles
117 // do not interfere with select() loops. On 64-bit Unix hosts this value is
118 // also OK, because up to that amount LevelDB will use an mmap
119 // implementation that does not use extra file descriptors (the fds are
120 // closed after being mmap'ed).
121 //
122 // Increasing the value beyond the default is dangerous because LevelDB will
123 // fall back to a non-mmap implementation when the file count is too large.
124 // On 32-bit Unix host we should decrease the value because the handles use
125 // up real fds, and we want to avoid fd exhaustion issues.
126 //
127 // See PR #12495 for further discussion.
128
129 int default_open_files = options->max_open_files;
130 #ifndef WIN32
131 if (sizeof(void*) < 8) {
132 options->max_open_files = 64;
133 }
134 #endif
135 LogDebug(BCLog::LEVELDB, "LevelDB using max_open_files=%d (default=%d)\n",
136 options->max_open_files, default_open_files);
137 }
138
139 static leveldb::Options GetOptions(size_t nCacheSize, bool bloom_filter)
140 {
141 leveldb::Options options;
142 options.block_cache = leveldb::NewLRUCache(nCacheSize / 2);
143 options.write_buffer_size = nCacheSize / 4; // up to two write buffers may be held in memory simultaneously
144 options.filter_policy = bloom_filter ? leveldb::NewBloomFilterPolicy(10) : nullptr;
145 options.compression = leveldb::kNoCompression;
146 options.info_log = new CBitcoinLevelDBLogger();
147 if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) {
148 // LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error
149 // on corruption in later versions.
150 options.paranoid_checks = true;
151 }
152 SetMaxOpenFiles(&options);
153 return options;
154 }
155
156 struct CDBBatch::WriteBatchImpl {
157 leveldb::WriteBatch batch;
158 };
159
160 CDBBatch::CDBBatch(const CDBWrapper& _parent)
161 : parent{_parent},
162 m_impl_batch{std::make_unique<CDBBatch::WriteBatchImpl>()}
163 {
164 m_key_scratch.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
165 m_value_scratch.reserve(DBWRAPPER_PREALLOC_VALUE_SIZE);
166 Clear();
167 };
168
169 CDBBatch::~CDBBatch() = default;
170
171 void CDBBatch::Clear()
172 {
173 m_impl_batch->batch.Clear();
174 assert(m_key_scratch.empty());
175 assert(m_value_scratch.empty());
176 }
177
178 void CDBBatch::WriteImpl(std::span<const std::byte> key, DataStream& value)
179 {
180 leveldb::Slice slKey(CharCast(key.data()), key.size());
181 dbwrapper_private::GetObfuscation(parent)(value);
182 leveldb::Slice slValue(CharCast(value.data()), value.size());
183 m_impl_batch->batch.Put(slKey, slValue);
184 }
185
186 void CDBBatch::EraseImpl(std::span<const std::byte> key)
187 {
188 leveldb::Slice slKey(CharCast(key.data()), key.size());
189 m_impl_batch->batch.Delete(slKey);
190 }
191
192 size_t CDBBatch::ApproximateSize() const
193 {
194 return m_impl_batch->batch.ApproximateSize();
195 }
196
197 struct LevelDBContext {
198 //! custom environment this database is using (may be nullptr in case of default environment)
199 leveldb::Env* penv;
200
201 //! database options used
202 leveldb::Options options;
203
204 //! options used when reading from the database
205 leveldb::ReadOptions readoptions;
206
207 //! options used when iterating over values of the database
208 leveldb::ReadOptions iteroptions;
209
210 //! options used when writing to the database
211 leveldb::WriteOptions writeoptions;
212
213 //! options used when sync writing to the database
214 leveldb::WriteOptions syncoptions;
215
216 //! the database itself
217 leveldb::DB* pdb;
218 };
219
220 CDBWrapper::CDBWrapper(const DBParams& params)
221 : m_db_context{std::make_unique<LevelDBContext>()}, m_name{fs::PathToString(params.path.stem())}
222 {
223 DBContext().penv = nullptr;
224 DBContext().readoptions.verify_checksums = true;
225 DBContext().iteroptions.verify_checksums = true;
226 DBContext().iteroptions.fill_cache = false;
227 DBContext().syncoptions.sync = true;
228 DBContext().options = GetOptions(params.cache_bytes, params.bloom_filter);
229 DBContext().options.create_if_missing = true;
230 DBContext().options.max_file_size = params.max_file_size;
231 assert(!(params.testing_env && params.memory_only));
232 if (params.testing_env) {
233 DBContext().options.env = params.testing_env;
234 } else if (params.memory_only) {
235 DBContext().penv = leveldb::NewMemEnv(leveldb::Env::Default());
236 DBContext().options.env = DBContext().penv;
237 }
238 if (!params.memory_only) {
239 if (params.wipe_data) {
240 LogInfo("Wiping LevelDB in %s", fs::PathToString(params.path));
241 leveldb::Status result = leveldb::DestroyDB(fs::PathToString(params.path), DBContext().options);
242 HandleError(result);
243 }
244 if (!params.testing_env) {
245 TryCreateDirectories(params.path);
246 }
247 LogInfo("Opening LevelDB in %s", fs::PathToString(params.path));
248 }
249 // PathToString() return value is safe to pass to leveldb open function,
250 // because on POSIX leveldb passes the byte string directly to ::open(), and
251 // on Windows it converts from UTF-8 to UTF-16 before calling ::CreateFileW
252 // (see env_posix.cc and env_windows.cc).
253 leveldb::Status status = leveldb::DB::Open(DBContext().options, fs::PathToString(params.path), &DBContext().pdb);
254 HandleError(status);
255 LogInfo("Opened LevelDB successfully");
256
257 if (params.options.force_compact) {
258 LogInfo("Starting database compaction of %s", fs::PathToString(params.path));
259 CompactFull();
260 LogInfo("Finished database compaction of %s", fs::PathToString(params.path));
261 }
262
263 if (!Read(OBFUSCATION_KEY, m_obfuscation) && params.obfuscate && IsEmpty()) {
264 // Generate and write the new obfuscation key.
265 const Obfuscation obfuscation{FastRandomContext{}.randbytes<Obfuscation::KEY_SIZE>()};
266 assert(!m_obfuscation); // Make sure the key is written without obfuscation.
267 Write(OBFUSCATION_KEY, obfuscation);
268 m_obfuscation = obfuscation;
269 LogInfo("Wrote new obfuscation key for %s: %s", fs::PathToString(params.path), m_obfuscation.HexKey());
270 }
271 LogInfo("Using obfuscation key for %s: %s", fs::PathToString(params.path), m_obfuscation.HexKey());
272 }
273
274 CDBWrapper::~CDBWrapper()
275 {
276 delete DBContext().pdb;
277 DBContext().pdb = nullptr;
278 delete DBContext().options.filter_policy;
279 DBContext().options.filter_policy = nullptr;
280 delete DBContext().options.info_log;
281 DBContext().options.info_log = nullptr;
282 delete DBContext().options.block_cache;
283 DBContext().options.block_cache = nullptr;
284 delete DBContext().penv;
285 DBContext().options.env = nullptr;
286 }
287
288 void CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync)
289 {
290 const bool log_memory = util::log::ShouldDebugLog(BCLog::LEVELDB);
291 double mem_before = 0;
292 if (log_memory) {
293 mem_before = DynamicMemoryUsage() / double(1_MiB);
294 }
295 leveldb::Status status = DBContext().pdb->Write(fSync ? DBContext().syncoptions : DBContext().writeoptions, &batch.m_impl_batch->batch);
296 HandleError(status);
297 if (log_memory) {
298 double mem_after{DynamicMemoryUsage() / double(1_MiB)};
299 LogDebug(BCLog::LEVELDB, "WriteBatch memory usage: db=%s, before=%.1fMiB, after=%.1fMiB\n",
300 m_name, mem_before, mem_after);
301 }
302 }
303
304 std::optional<std::string> CDBWrapper::GetProperty(const std::string& property) const
305 {
306 if (std::string value; DBContext().pdb->GetProperty(property, &value)) return value;
307 return std::nullopt;
308 }
309
310 void CDBWrapper::CompactFull() { DBContext().pdb->CompactRange(nullptr, nullptr); }
311
312 size_t CDBWrapper::DynamicMemoryUsage() const
313 {
314 std::optional<size_t> parsed;
315 if (auto memory{GetProperty("leveldb.approximate-memory-usage")}; !memory || !(parsed = ToIntegral<size_t>(*memory))) {
316 LogDebug(BCLog::LEVELDB, "Failed to get approximate-memory-usage property\n");
317 return 0;
318 }
319 return parsed.value();
320 }
321
322 std::optional<std::string> CDBWrapper::ReadImpl(std::span<const std::byte> key) const
323 {
324 leveldb::Slice slKey(CharCast(key.data()), key.size());
325 std::string strValue;
326 leveldb::Status status = DBContext().pdb->Get(DBContext().readoptions, slKey, &strValue);
327 if (!status.ok()) {
328 if (status.IsNotFound())
329 return std::nullopt;
330 LogError("LevelDB read failure: %s", status.ToString());
331 HandleError(status);
332 }
333 return strValue;
334 }
335
336 bool CDBWrapper::ExistsImpl(std::span<const std::byte> key) const
337 {
338 leveldb::Slice slKey(CharCast(key.data()), key.size());
339
340 std::string strValue;
341 leveldb::Status status = DBContext().pdb->Get(DBContext().readoptions, slKey, &strValue);
342 if (!status.ok()) {
343 if (status.IsNotFound())
344 return false;
345 LogError("LevelDB read failure: %s", status.ToString());
346 HandleError(status);
347 }
348 return true;
349 }
350
351 size_t CDBWrapper::EstimateSizeImpl(std::span<const std::byte> key1, std::span<const std::byte> key2) const
352 {
353 leveldb::Slice slKey1(CharCast(key1.data()), key1.size());
354 leveldb::Slice slKey2(CharCast(key2.data()), key2.size());
355 uint64_t size = 0;
356 leveldb::Range range(slKey1, slKey2);
357 DBContext().pdb->GetApproximateSizes(&range, 1, &size);
358 return size;
359 }
360
361 bool CDBWrapper::IsEmpty()
362 {
363 std::unique_ptr<CDBIterator> it(NewIterator());
364 it->SeekToFirst();
365 return !(it->Valid());
366 }
367
368 struct CDBIterator::IteratorImpl {
369 const std::unique_ptr<leveldb::Iterator> iter;
370
371 explicit IteratorImpl(leveldb::Iterator* _iter) : iter{_iter} {}
372 };
373
374 CDBIterator::CDBIterator(const CDBWrapper& _parent, std::unique_ptr<IteratorImpl> _piter) : parent(_parent),
375 m_impl_iter(std::move(_piter))
376 {
377 m_scratch.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
378 }
379
380 CDBIterator* CDBWrapper::NewIterator()
381 {
382 return new CDBIterator{*this, std::make_unique<CDBIterator::IteratorImpl>(DBContext().pdb->NewIterator(DBContext().iteroptions))};
383 }
384
385 void CDBIterator::SeekImpl(std::span<const std::byte> key)
386 {
387 leveldb::Slice slKey(CharCast(key.data()), key.size());
388 m_impl_iter->iter->Seek(slKey);
389 }
390
391 std::span<const std::byte> CDBIterator::GetKeyImpl() const
392 {
393 // The returned span borrows from the current iterator entry and is only
394 // valid until the iterator is advanced.
395 return MakeByteSpan(m_impl_iter->iter->key());
396 }
397
398 std::span<const std::byte> CDBIterator::GetValueImpl() const
399 {
400 return MakeByteSpan(m_impl_iter->iter->value());
401 }
402
403 CDBIterator::~CDBIterator() = default;
404 bool CDBIterator::Valid() const { return m_impl_iter->iter->Valid(); }
405 void CDBIterator::SeekToFirst() { m_impl_iter->iter->SeekToFirst(); }
406 void CDBIterator::Next() { m_impl_iter->iter->Next(); }
407
408 namespace dbwrapper_private {
409
410 const Obfuscation& GetObfuscation(const CDBWrapper& w)
411 {
412 return w.m_obfuscation;
413 }
414
415 } // namespace dbwrapper_private
416