blockstorage.cpp raw
1 // Copyright (c) 2011-2022 The Limenka developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4
5 #include <node/blockstorage.h>
6
7 #include <arith_uint256.h>
8 #include <chain.h>
9 #include <consensus/params.h>
10 #include <consensus/validation.h>
11 #include <dbwrapper.h>
12 #include <flatfile.h>
13 #include <hash.h>
14 #include <kernel/blockmanager_opts.h>
15 #include <kernel/chainparams.h>
16 #include <kernel/messagestartchars.h>
17 #include <kernel/notifications_interface.h>
18 #include <logging.h>
19 #include <pow.h>
20 #include <primitives/block.h>
21 #include <primitives/transaction.h>
22 #include <random.h>
23 #include <serialize.h>
24 #include <signet.h>
25 #include <span.h>
26 #include <streams.h>
27 #include <sync.h>
28 #include <tinyformat.h>
29 #include <uint256.h>
30 #include <undo.h>
31 #include <util/batchpriority.h>
32 #include <util/check.h>
33 #include <util/fs.h>
34 #include <util/ioprio.h>
35 #include <util/obfuscation.h>
36 #include <util/overflow.h>
37 #include <util/signalinterrupt.h>
38 #include <util/strencodings.h>
39 #include <util/syserror.h>
40 #include <util/translation.h>
41 #include <validation.h>
42
43 #include <cstddef>
44 #include <map>
45 #include <optional>
46 #include <ranges>
47 #include <unordered_map>
48
49 namespace kernel {
50 static constexpr uint8_t DB_BLOCK_FILES{'f'};
51 static constexpr uint8_t DB_BLOCK_INDEX{'b'};
52 static constexpr uint8_t DB_FLAG{'F'};
53 static constexpr uint8_t DB_REINDEX_FLAG{'R'};
54 static constexpr uint8_t DB_LAST_BLOCK{'l'};
55 static constexpr uint8_t DB_PRUNE_LOCK{'L'};
56 // Keys used in previous version that might still be found in the DB:
57 // BlockTreeDB::DB_TXINDEX_BLOCK{'T'};
58 // BlockTreeDB::DB_TXINDEX{'t'}
59 // BlockTreeDB::ReadFlag("txindex")
60
61 bool BlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo& info)
62 {
63 return Read(std::make_pair(DB_BLOCK_FILES, nFile), info);
64 }
65
66 bool BlockTreeDB::WriteReindexing(bool fReindexing)
67 {
68 if (fReindexing) {
69 return Write(DB_REINDEX_FLAG, uint8_t{'1'});
70 } else {
71 return Erase(DB_REINDEX_FLAG);
72 }
73 }
74
75 void BlockTreeDB::ReadReindexing(bool& fReindexing)
76 {
77 fReindexing = Exists(DB_REINDEX_FLAG);
78 }
79
80 bool BlockTreeDB::ReadLastBlockFile(int& nFile)
81 {
82 return Read(DB_LAST_BLOCK, nFile);
83 }
84
85 bool BlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*>>& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo, const std::unordered_map<std::string, node::PruneLockInfo>& prune_locks)
86 {
87 CDBBatch batch(*this);
88 for (const auto& [file, info] : fileInfo) {
89 batch.Write(std::make_pair(DB_BLOCK_FILES, file), *info);
90 }
91 batch.Write(DB_LAST_BLOCK, nLastFile);
92 for (const CBlockIndex* bi : blockinfo) {
93 batch.Write(std::make_pair(DB_BLOCK_INDEX, bi->GetBlockHash()), CDiskBlockIndex{bi});
94 }
95 for (const auto& prune_lock : prune_locks) {
96 if (prune_lock.second.temporary) continue;
97 batch.Write(std::make_pair(DB_PRUNE_LOCK, prune_lock.first), prune_lock.second);
98 }
99 return WriteBatch(batch, true);
100 }
101
102 bool BlockTreeDB::WritePruneLock(const std::string& name, const node::PruneLockInfo& lock_info) {
103 if (lock_info.temporary) return true;
104 return Write(std::make_pair(DB_PRUNE_LOCK, name), lock_info);
105 }
106
107 bool BlockTreeDB::DeletePruneLock(const std::string& name) {
108 return Erase(std::make_pair(DB_PRUNE_LOCK, name));
109 }
110
111 bool BlockTreeDB::LoadPruneLocks(std::unordered_map<std::string, node::PruneLockInfo>& prune_locks, const util::SignalInterrupt& interrupt) {
112 std::unique_ptr<CDBIterator> pcursor(NewIterator());
113 for (pcursor->Seek(DB_PRUNE_LOCK); pcursor->Valid(); pcursor->Next()) {
114 if (interrupt) return false;
115
116 std::pair<uint8_t, std::string> key;
117 if ((!pcursor->GetKey(key)) || key.first != DB_PRUNE_LOCK) break;
118
119 node::PruneLockInfo& lock_info = prune_locks[key.second];
120 if (!pcursor->GetValue(lock_info)) {
121 LogError("%s: failed to %s prune lock '%s'\n", __func__, "read", key.second);
122 return false;
123 }
124 lock_info.temporary = false;
125 }
126
127 return true;
128 }
129
130 bool BlockTreeDB::WriteFlag(const std::string& name, bool fValue)
131 {
132 return Write(std::make_pair(DB_FLAG, name), fValue ? uint8_t{'1'} : uint8_t{'0'});
133 }
134
135 bool BlockTreeDB::ReadFlag(const std::string& name, bool& fValue)
136 {
137 uint8_t ch;
138 if (!Read(std::make_pair(DB_FLAG, name), ch)) {
139 return false;
140 }
141 fValue = ch == uint8_t{'1'};
142 return true;
143 }
144
145 bool BlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex, const util::SignalInterrupt& interrupt)
146 {
147 AssertLockHeld(::cs_main);
148 std::unique_ptr<CDBIterator> pcursor(NewIterator());
149 pcursor->Seek(std::make_pair(DB_BLOCK_INDEX, uint256()));
150
151 // Load m_block_index
152 while (pcursor->Valid()) {
153 if (interrupt) return false;
154 std::pair<uint8_t, uint256> key;
155 if (pcursor->GetKey(key) && key.first == DB_BLOCK_INDEX) {
156 CDiskBlockIndex diskindex;
157 if (pcursor->GetValue(diskindex)) {
158 // Construct block index object
159 CBlockIndex* pindexNew = insertBlockIndex(diskindex.ConstructBlockHash());
160 pindexNew->pprev = insertBlockIndex(diskindex.hashPrev);
161 pindexNew->nHeight = diskindex.nHeight;
162 if (pindexNew->nHeight < 0) {
163 LogError("%s: Invalid nHeight %d\n", __func__, pindexNew->nHeight);
164 return false;
165 }
166 pindexNew->nFile = diskindex.nFile;
167 pindexNew->nDataPos = diskindex.nDataPos;
168 pindexNew->nUndoPos = diskindex.nUndoPos;
169 pindexNew->nVersion = diskindex.nVersion;
170 pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot;
171 pindexNew->nTime = diskindex.nTime;
172 pindexNew->nBits = diskindex.nBits;
173 pindexNew->nNonce = diskindex.nNonce;
174 pindexNew->nStatus = diskindex.nStatus;
175 pindexNew->nTx = diskindex.nTx;
176
177 if (!CheckProofOfWork(pindexNew->GetBlockHash(), pindexNew->nBits, consensusParams)) {
178 LogError("%s: CheckProofOfWork failed: %s\n", __func__, pindexNew->ToString());
179 return false;
180 }
181
182 pcursor->Next();
183 } else {
184 LogError("%s: failed to read value\n", __func__);
185 return false;
186 }
187 } else {
188 break;
189 }
190 }
191
192 return true;
193 }
194 } // namespace kernel
195
196 namespace node {
197
198 bool CBlockIndexWorkComparator::operator()(const CBlockIndex* pa, const CBlockIndex* pb) const
199 {
200 // First sort by most total work, ...
201 if (pa->nChainWork != pb->nChainWork) {
202 return pa->nChainWork < pb->nChainWork;
203 }
204
205 // ... then by earliest activatable time, ...
206 if (pa->nSequenceId != pb->nSequenceId) {
207 return pa->nSequenceId > pb->nSequenceId;
208 }
209
210 // Use pointer address as tie breaker (should only happen with blocks
211 // loaded from disk, as those share the same id: 0 for blocks on the
212 // best chain, 1 for all others).
213 return pa > pb;
214 }
215
216 bool CBlockIndexHeightOnlyComparator::operator()(const CBlockIndex* pa, const CBlockIndex* pb) const
217 {
218 return pa->nHeight < pb->nHeight;
219 }
220
221 /** The number of blocks to keep below the deepest prune lock.
222 * There is nothing special about this number. It is higher than what we
223 * expect to see in regular mainnet reorgs, but not so high that it would
224 * noticeably interfere with the pruning mechanism.
225 * */
226 static constexpr int PRUNE_LOCK_BUFFER{10};
227
228 std::vector<CBlockIndex*> BlockManager::GetAllBlockIndices()
229 {
230 AssertLockHeld(cs_main);
231 std::vector<CBlockIndex*> rv;
232 rv.reserve(m_block_index.size());
233 for (auto& [_, block_index] : m_block_index) {
234 rv.push_back(&block_index);
235 }
236 return rv;
237 }
238
239 CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash)
240 {
241 AssertLockHeld(cs_main);
242 BlockMap::iterator it = m_block_index.find(hash);
243 return it == m_block_index.end() ? nullptr : &it->second;
244 }
245
246 const CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash) const
247 {
248 AssertLockHeld(cs_main);
249 BlockMap::const_iterator it = m_block_index.find(hash);
250 return it == m_block_index.end() ? nullptr : &it->second;
251 }
252
253 CBlockIndex* BlockManager::AddToBlockIndex(const CBlockHeader& block, CBlockIndex*& best_header)
254 {
255 AssertLockHeld(cs_main);
256
257 auto [mi, inserted] = m_block_index.try_emplace(block.GetHash(), block);
258 if (!inserted) {
259 return &mi->second;
260 }
261 CBlockIndex* pindexNew = &(*mi).second;
262
263 // We assign the sequence id to blocks only when the full data is available,
264 // to avoid miners withholding blocks but broadcasting headers, to get a
265 // competitive advantage.
266 pindexNew->nSequenceId = SEQ_ID_INIT_FROM_DISK;
267
268 pindexNew->phashBlock = &((*mi).first);
269 BlockMap::iterator miPrev = m_block_index.find(block.hashPrevBlock);
270 if (miPrev != m_block_index.end()) {
271 pindexNew->pprev = &(*miPrev).second;
272 pindexNew->nHeight = pindexNew->pprev->nHeight + 1;
273 pindexNew->BuildSkip();
274 }
275 pindexNew->nTimeMax = (pindexNew->pprev ? std::max(pindexNew->pprev->nTimeMax, pindexNew->nTime) : pindexNew->nTime);
276 pindexNew->nChainWork = (pindexNew->pprev ? pindexNew->pprev->nChainWork : 0) + GetBlockProof(*pindexNew);
277 pindexNew->RaiseValidity(BLOCK_VALID_TREE);
278 if (best_header == nullptr || best_header->nChainWork < pindexNew->nChainWork) {
279 best_header = pindexNew;
280 }
281
282 m_dirty_blockindex.insert(pindexNew);
283
284 return pindexNew;
285 }
286
287 void BlockManager::PruneOneBlockFile(const int fileNumber)
288 {
289 AssertLockHeld(cs_main);
290 LOCK(cs_LastBlockFile);
291
292 for (auto& entry : m_block_index) {
293 CBlockIndex* pindex = &entry.second;
294 if (pindex->nFile == fileNumber) {
295 pindex->nStatus &= ~BLOCK_HAVE_DATA;
296 pindex->nStatus &= ~BLOCK_HAVE_UNDO;
297 pindex->nFile = 0;
298 pindex->nDataPos = 0;
299 pindex->nUndoPos = 0;
300 m_dirty_blockindex.insert(pindex);
301
302 // Prune from m_blocks_unlinked -- any block we prune would have
303 // to be downloaded again in order to consider its chain, at which
304 // point it would be considered as a candidate for
305 // m_blocks_unlinked or setBlockIndexCandidates.
306 auto range = m_blocks_unlinked.equal_range(pindex->pprev);
307 while (range.first != range.second) {
308 std::multimap<CBlockIndex*, CBlockIndex*>::iterator _it = range.first;
309 range.first++;
310 if (_it->second == pindex) {
311 m_blocks_unlinked.erase(_it);
312 }
313 }
314 }
315 }
316
317 m_blockfile_info.at(fileNumber) = CBlockFileInfo{};
318 m_dirty_fileinfo.insert(fileNumber);
319 }
320
321 bool BlockManager::DoPruneLocksForbidPruning(const CBlockFileInfo& block_file_info)
322 {
323 AssertLockHeld(cs_main);
324 for (const auto& prune_lock : m_prune_locks) {
325 if (prune_lock.second.height_first == std::numeric_limits<uint64_t>::max()) continue;
326 // Remove the buffer and one additional block here to get actual height that is outside of the buffer
327 const uint64_t lock_height{(prune_lock.second.height_first <= PRUNE_LOCK_BUFFER + 1) ? 1 : (prune_lock.second.height_first - PRUNE_LOCK_BUFFER - 1)};
328 const uint64_t lock_height_last{SaturatingAdd(prune_lock.second.height_last, (uint64_t)PRUNE_LOCK_BUFFER)};
329 if (block_file_info.nHeightFirst > lock_height_last) continue;
330 if (block_file_info.nHeightLast <= lock_height) continue;
331 // TODO: Check each block within the file against the prune_lock range
332
333 LogDebug(BCLog::PRUNE, "%s limited pruning to height %d\n", prune_lock.first, lock_height);
334 return true;
335 }
336 return false;
337 }
338
339 void BlockManager::FindFilesToPruneManual(
340 std::set<int>& setFilesToPrune,
341 int nManualPruneHeight,
342 const Chainstate& chain,
343 ChainstateManager& chainman)
344 {
345 assert(IsPruneMode() && nManualPruneHeight > 0);
346
347 LOCK2(cs_main, cs_LastBlockFile);
348 if (chain.m_chain.Height() < 0) {
349 return;
350 }
351
352 const auto [min_block_to_prune, last_block_can_prune] = chainman.GetPruneRange(chain, nManualPruneHeight);
353
354 int count = 0;
355 for (int fileNumber = 0; fileNumber < this->MaxBlockfileNum(); fileNumber++) {
356 const auto& fileinfo = m_blockfile_info[fileNumber];
357 if (fileinfo.nSize == 0 || fileinfo.nHeightLast > (unsigned)last_block_can_prune || fileinfo.nHeightFirst < (unsigned)min_block_to_prune) {
358 continue;
359 }
360
361 if (DoPruneLocksForbidPruning(m_blockfile_info[fileNumber])) continue;
362
363 PruneOneBlockFile(fileNumber);
364 setFilesToPrune.insert(fileNumber);
365 count++;
366 }
367 LogPrintf("[%s] Prune (Manual): prune_height=%d removed %d blk/rev pairs\n",
368 chain.GetRole(), last_block_can_prune, count);
369 }
370
371 uint64_t BlockManager::GetPruneTargetForChainstate(const Chainstate& chain, ChainstateManager& chainman) const
372 {
373 const auto number_of_chainstates{chainman.GetAll().size()};
374 const uint64_t min_overall_target{MIN_DISK_SPACE_FOR_BLOCK_FILES * number_of_chainstates};
375 auto target = std::max(min_overall_target, GetPruneTarget());
376 uint64_t target_boost{0};
377 if (m_opts.prune_target_during_init > -1 && chainman.IsInitialBlockDownload()) {
378 if ((uint64_t)m_opts.prune_target_during_init <= target) {
379 target = std::max(min_overall_target, (uint64_t)m_opts.prune_target_during_init);
380 } else if (chain.GetRole() != ChainstateRole::ASSUMEDVALID) {
381 // Only the background/normal gets the benefit
382 // NOTE: This assumes only one such chainstate exists
383 target_boost = m_opts.prune_target_during_init - target;
384 }
385 }
386 // Distribute our -prune budget over all chainstates.
387 target = (target / number_of_chainstates) + target_boost;
388 return target;
389 }
390
391 void BlockManager::FindFilesToPrune(
392 std::set<int>& setFilesToPrune,
393 int last_prune,
394 const Chainstate& chain,
395 ChainstateManager& chainman)
396 {
397 LOCK2(cs_main, cs_LastBlockFile);
398 const auto target{GetPruneTargetForChainstate(chain, chainman)};
399 const uint64_t target_sync_height = chainman.m_best_header->nHeight;
400
401 if (chain.m_chain.Height() < 0 || target == 0) {
402 return;
403 }
404 if (static_cast<uint64_t>(chain.m_chain.Height()) <= chainman.GetParams().PruneAfterHeight()) {
405 return;
406 }
407
408 const auto [min_block_to_prune, last_block_can_prune] = chainman.GetPruneRange(chain, last_prune);
409
410 uint64_t nCurrentUsage = CalculateCurrentUsage();
411 // We don't check to prune until after we've allocated new space for files
412 // So we should leave a buffer under our target to account for another allocation
413 // before the next pruning.
414 uint64_t nBuffer = BLOCKFILE_CHUNK_SIZE + UNDOFILE_CHUNK_SIZE;
415 uint64_t nBytesToPrune;
416 int count = 0;
417
418 if (nCurrentUsage + nBuffer >= target) {
419 // On a prune event, the chainstate DB is flushed.
420 // To avoid excessive prune events negating the benefit of high dbcache
421 // values, we should not prune too rapidly.
422 // So when pruning in IBD, increase the buffer to avoid a re-prune too soon.
423 const auto chain_tip_height = chain.m_chain.Height();
424 if (chainman.IsInitialBlockDownload() && target_sync_height > (uint64_t)chain_tip_height) {
425 // Since this is only relevant during IBD, we assume blocks are at least 1 MB on average
426 static constexpr uint64_t average_block_size = 1000000; /* 1 MB */
427 const uint64_t remaining_blocks = target_sync_height - chain_tip_height;
428 nBuffer += average_block_size * remaining_blocks;
429 }
430
431 for (int fileNumber = 0; fileNumber < this->MaxBlockfileNum(); fileNumber++) {
432 const auto& fileinfo = m_blockfile_info[fileNumber];
433 nBytesToPrune = fileinfo.nSize + fileinfo.nUndoSize;
434
435 if (fileinfo.nSize == 0) {
436 continue;
437 }
438
439 if (nCurrentUsage + nBuffer < target) { // are we below our target?
440 break;
441 }
442
443 // don't prune files that could have a block that's not within the allowable
444 // prune range for the chain being pruned.
445 if (fileinfo.nHeightLast > (unsigned)last_block_can_prune || fileinfo.nHeightFirst < (unsigned)min_block_to_prune) {
446 continue;
447 }
448
449 if (DoPruneLocksForbidPruning(m_blockfile_info[fileNumber])) continue;
450
451 PruneOneBlockFile(fileNumber);
452 // Queue up the files for removal
453 setFilesToPrune.insert(fileNumber);
454 nCurrentUsage -= nBytesToPrune;
455 count++;
456 }
457 }
458
459 LogDebug(BCLog::PRUNE, "[%s] target=%dMiB actual=%dMiB diff=%dMiB min_height=%d max_prune_height=%d removed %d blk/rev pairs\n",
460 chain.GetRole(), target / 1024 / 1024, nCurrentUsage / 1024 / 1024,
461 (int64_t(target) - int64_t(nCurrentUsage)) / 1024 / 1024,
462 min_block_to_prune, last_block_can_prune, count);
463 }
464
465 bool BlockManager::PruneLockExists(const std::string& name) const {
466 return m_prune_locks.count(name);
467 }
468
469 bool BlockManager::UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info, const bool sync) {
470 AssertLockHeld(::cs_main);
471 if (sync) {
472 if (!m_block_tree_db->WritePruneLock(name, lock_info)) {
473 LogError("%s: failed to %s prune lock '%s'\n", __func__, "write", name);
474 return false;
475 }
476 }
477 PruneLockInfo& stored_lock_info = m_prune_locks[name];
478 if (lock_info.temporary && !stored_lock_info.temporary) {
479 // Erase non-temporary lock from disk
480 if (!m_block_tree_db->DeletePruneLock(name)) {
481 LogError("%s: failed to %s prune lock '%s'\n", __func__, "erase", name);
482 return false;
483 }
484 }
485 stored_lock_info = lock_info;
486 return true;
487 }
488
489 bool BlockManager::DeletePruneLock(const std::string& name)
490 {
491 AssertLockHeld(::cs_main);
492 m_prune_locks.erase(name);
493
494 // Since there is no reasonable expectation for any follow-up to this prune lock, actually ensure it gets committed to disk immediately
495 if (!m_block_tree_db->DeletePruneLock(name)) {
496 LogError("%s: failed to %s prune lock '%s'\n", __func__, "erase", name);
497 return false;
498 }
499 return true;
500 }
501
502 CBlockIndex* BlockManager::InsertBlockIndex(const uint256& hash)
503 {
504 AssertLockHeld(cs_main);
505
506 if (hash.IsNull()) {
507 return nullptr;
508 }
509
510 const auto [mi, inserted]{m_block_index.try_emplace(hash)};
511 CBlockIndex* pindex = &(*mi).second;
512 if (inserted) {
513 pindex->phashBlock = &((*mi).first);
514 }
515 return pindex;
516 }
517
518 bool BlockManager::LoadBlockIndex(const std::optional<uint256>& snapshot_blockhash)
519 {
520 if (!m_block_tree_db->LoadBlockIndexGuts(
521 GetConsensus(), [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); }, m_interrupt)) {
522 return false;
523 }
524
525 if (!m_block_tree_db->LoadPruneLocks(m_prune_locks, m_interrupt)) return false;
526
527 if (snapshot_blockhash) {
528 const std::optional<AssumeutxoData> maybe_au_data = GetParams().AssumeutxoForBlockhash(*snapshot_blockhash);
529 if (!maybe_au_data) {
530 m_opts.notifications.fatalError(strprintf(_("Assumeutxo data not found for the given blockhash '%s'."), snapshot_blockhash->ToString()));
531 return false;
532 }
533 const AssumeutxoData& au_data = *Assert(maybe_au_data);
534 m_snapshot_height = au_data.height;
535 CBlockIndex* base{LookupBlockIndex(*snapshot_blockhash)};
536
537 // Since m_chain_tx_count (responsible for estimated progress) isn't persisted
538 // to disk, we must bootstrap the value for assumedvalid chainstates
539 // from the hardcoded assumeutxo chainparams.
540 base->m_chain_tx_count = au_data.m_chain_tx_count;
541 LogPrintf("[snapshot] set m_chain_tx_count=%d for %s\n", au_data.m_chain_tx_count, snapshot_blockhash->ToString());
542 } else {
543 // If this isn't called with a snapshot blockhash, make sure the cached snapshot height
544 // is null. This is relevant during snapshot completion, when the blockman may be loaded
545 // with a height that then needs to be cleared after the snapshot is fully validated.
546 m_snapshot_height.reset();
547 }
548
549 Assert(m_snapshot_height.has_value() == snapshot_blockhash.has_value());
550
551 // Calculate nChainWork
552 std::vector<CBlockIndex*> vSortedByHeight{GetAllBlockIndices()};
553 std::sort(vSortedByHeight.begin(), vSortedByHeight.end(),
554 CBlockIndexHeightOnlyComparator());
555
556 CBlockIndex* previous_index{nullptr};
557 for (CBlockIndex* pindex : vSortedByHeight) {
558 if (m_interrupt) return false;
559 if (previous_index && pindex->nHeight > previous_index->nHeight + 1) {
560 LogError("%s: block index is non-contiguous, index of height %d missing\n", __func__, previous_index->nHeight + 1);
561 return false;
562 }
563 previous_index = pindex;
564 pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex);
565 pindex->nTimeMax = (pindex->pprev ? std::max(pindex->pprev->nTimeMax, pindex->nTime) : pindex->nTime);
566
567 // We can link the chain of blocks for which we've received transactions at some point, or
568 // blocks that are assumed-valid on the basis of snapshot load (see
569 // PopulateAndValidateSnapshot()).
570 // Pruned nodes may have deleted the block.
571 if (pindex->nTx > 0) {
572 if (pindex->pprev) {
573 if (m_snapshot_height && pindex->nHeight == *m_snapshot_height &&
574 pindex->GetBlockHash() == *snapshot_blockhash) {
575 // Should have been set above; don't disturb it with code below.
576 Assert(pindex->m_chain_tx_count > 0);
577 } else if (pindex->pprev->m_chain_tx_count > 0) {
578 pindex->m_chain_tx_count = pindex->pprev->m_chain_tx_count + pindex->nTx;
579 } else {
580 pindex->m_chain_tx_count = 0;
581 m_blocks_unlinked.insert(std::make_pair(pindex->pprev, pindex));
582 }
583 } else {
584 pindex->m_chain_tx_count = pindex->nTx;
585 }
586 }
587 if (!(pindex->nStatus & BLOCK_FAILED_MASK) && pindex->pprev && (pindex->pprev->nStatus & BLOCK_FAILED_MASK)) {
588 pindex->nStatus |= BLOCK_FAILED_CHILD;
589 m_dirty_blockindex.insert(pindex);
590 }
591 if (pindex->pprev) {
592 pindex->BuildSkip();
593 }
594 }
595
596 return true;
597 }
598
599 bool BlockManager::WriteBlockIndexDB()
600 {
601 AssertLockHeld(::cs_main);
602 std::vector<std::pair<int, const CBlockFileInfo*>> vFiles;
603 vFiles.reserve(m_dirty_fileinfo.size());
604 for (std::set<int>::iterator it = m_dirty_fileinfo.begin(); it != m_dirty_fileinfo.end(); ++it) {
605 vFiles.emplace_back(*it, &m_blockfile_info[*it]);
606 }
607 std::vector<const CBlockIndex*> vBlocks;
608 vBlocks.reserve(m_dirty_blockindex.size());
609 for (std::set<CBlockIndex*>::iterator it = m_dirty_blockindex.begin(); it != m_dirty_blockindex.end(); ++it) {
610 vBlocks.push_back(*it);
611 }
612 int max_blockfile = WITH_LOCK(cs_LastBlockFile, return this->MaxBlockfileNum());
613 if (!m_block_tree_db->WriteBatchSync(vFiles, max_blockfile, vBlocks, m_prune_locks)) {
614 return false;
615 }
616 m_dirty_fileinfo.clear();
617 m_dirty_blockindex.clear();
618 return true;
619 }
620
621 bool BlockManager::LoadBlockIndexDB(const std::optional<uint256>& snapshot_blockhash)
622 {
623 if (!LoadBlockIndex(snapshot_blockhash)) {
624 return false;
625 }
626 int max_blockfile_num{0};
627
628 // Load block file info
629 m_block_tree_db->ReadLastBlockFile(max_blockfile_num);
630 m_blockfile_info.resize(max_blockfile_num + 1);
631 LogPrintf("%s: last block file = %i\n", __func__, max_blockfile_num);
632 for (int nFile = 0; nFile <= max_blockfile_num; nFile++) {
633 m_block_tree_db->ReadBlockFileInfo(nFile, m_blockfile_info[nFile]);
634 }
635 LogPrintf("%s: last block file info: %s\n", __func__, m_blockfile_info[max_blockfile_num].ToString());
636 for (int nFile = max_blockfile_num + 1; true; nFile++) {
637 CBlockFileInfo info;
638 if (m_block_tree_db->ReadBlockFileInfo(nFile, info)) {
639 m_blockfile_info.push_back(info);
640 } else {
641 break;
642 }
643 }
644
645 // Check presence of blk files
646 LogPrintf("Checking all blk files are present...\n");
647 std::set<int> setBlkDataFiles;
648 for (const auto& [_, block_index] : m_block_index) {
649 if (block_index.nStatus & BLOCK_HAVE_DATA) {
650 setBlkDataFiles.insert(block_index.nFile);
651 }
652 }
653 for (std::set<int>::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++) {
654 FlatFilePos pos(*it, 0);
655 if (OpenBlockFile(pos, true).IsNull()) {
656 return false;
657 }
658 }
659
660 {
661 // Initialize the blockfile cursors.
662 LOCK(cs_LastBlockFile);
663 for (size_t i = 0; i < m_blockfile_info.size(); ++i) {
664 const auto last_height_in_file = m_blockfile_info[i].nHeightLast;
665 m_blockfile_cursors[BlockfileTypeForHeight(last_height_in_file)] = {static_cast<int>(i), 0};
666 }
667 }
668
669 // Check whether we have ever pruned block & undo files
670 m_block_tree_db->ReadFlag("prunedblockfiles", m_have_pruned);
671 if (m_have_pruned) {
672 LogPrintf("LoadBlockIndexDB(): Block files have previously been pruned\n");
673 }
674
675 // Check whether we need to continue reindexing
676 bool fReindexing = false;
677 m_block_tree_db->ReadReindexing(fReindexing);
678 if (fReindexing) m_blockfiles_indexed = false;
679
680 return true;
681 }
682
683 void BlockManager::ScanAndUnlinkAlreadyPrunedFiles()
684 {
685 AssertLockHeld(::cs_main);
686 int max_blockfile = WITH_LOCK(cs_LastBlockFile, return this->MaxBlockfileNum());
687 if (!m_have_pruned) {
688 return;
689 }
690
691 std::set<int> block_files_to_prune;
692 for (int file_number = 0; file_number < max_blockfile; file_number++) {
693 if (m_blockfile_info[file_number].nSize == 0) {
694 block_files_to_prune.insert(file_number);
695 }
696 }
697
698 UnlinkPrunedFiles(block_files_to_prune);
699 }
700
701 const CBlockIndex* BlockManager::GetLastCheckpoint(const CCheckpointData& data)
702 {
703 const MapCheckpoints& checkpoints = data.mapCheckpoints;
704
705 for (const MapCheckpoints::value_type& i : checkpoints | std::views::reverse) {
706 const uint256& hash = i.second;
707 const CBlockIndex* pindex = LookupBlockIndex(hash);
708 if (pindex) {
709 return pindex;
710 }
711 }
712 return nullptr;
713 }
714
715 bool BlockManager::IsBlockPruned(const CBlockIndex& block) const
716 {
717 AssertLockHeld(::cs_main);
718 return m_have_pruned && !(block.nStatus & BLOCK_HAVE_DATA) && (block.nTx > 0);
719 }
720
721 const CBlockIndex* BlockManager::GetFirstBlock(const CBlockIndex& upper_block, uint32_t status_mask, const CBlockIndex* lower_block) const
722 {
723 AssertLockHeld(::cs_main);
724 const CBlockIndex* last_block = &upper_block;
725 assert((last_block->nStatus & status_mask) == status_mask); // 'upper_block' must satisfy the status mask
726 while (last_block->pprev && ((last_block->pprev->nStatus & status_mask) == status_mask)) {
727 if (lower_block) {
728 // Return if we reached the lower_block
729 if (last_block == lower_block) return lower_block;
730 // if range was surpassed, means that 'lower_block' is not part of the 'upper_block' chain
731 // and so far this is not allowed.
732 assert(last_block->nHeight >= lower_block->nHeight);
733 }
734 last_block = last_block->pprev;
735 }
736 assert(last_block != nullptr);
737 return last_block;
738 }
739
740 bool BlockManager::CheckBlockDataAvailability(const CBlockIndex& upper_block, const CBlockIndex& lower_block)
741 {
742 if (!(upper_block.nStatus & BLOCK_HAVE_DATA)) return false;
743 return GetFirstBlock(upper_block, BLOCK_HAVE_DATA, &lower_block) == &lower_block;
744 }
745
746 // If we're using -prune with -reindex, then delete block files that will be ignored by the
747 // reindex. Since reindexing works by starting at block file 0 and looping until a blockfile
748 // is missing, do the same here to delete any later block files after a gap. Also delete all
749 // rev files since they'll be rewritten by the reindex anyway. This ensures that m_blockfile_info
750 // is in sync with what's actually on disk by the time we start downloading, so that pruning
751 // works correctly.
752 void BlockManager::CleanupBlockRevFiles() const
753 {
754 std::map<std::string, fs::path> mapBlockFiles;
755
756 // Glob all blk?????.dat and rev?????.dat files from the blocks directory.
757 // Remove the rev files immediately and insert the blk file paths into an
758 // ordered map keyed by block file index.
759 LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n");
760 for (fs::directory_iterator it(m_opts.blocks_dir); it != fs::directory_iterator(); it++) {
761 const std::string path = fs::PathToString(it->path().filename());
762 if (fs::is_regular_file(*it) &&
763 path.length() == 12 &&
764 path.substr(8,4) == ".dat")
765 {
766 if (path.substr(0, 3) == "blk") {
767 mapBlockFiles[path.substr(3, 5)] = it->path();
768 } else if (path.substr(0, 3) == "rev") {
769 remove(it->path());
770 }
771 }
772 }
773
774 // Remove all block files that aren't part of a contiguous set starting at
775 // zero by walking the ordered map (keys are block file indices) by
776 // keeping a separate counter. Once we hit a gap (or if 0 doesn't exist)
777 // start removing block files.
778 int nContigCounter = 0;
779 for (const std::pair<const std::string, fs::path>& item : mapBlockFiles) {
780 if (LocaleIndependentAtoi<int>(item.first) == nContigCounter) {
781 nContigCounter++;
782 continue;
783 }
784 remove(item.second);
785 }
786 }
787
788 CBlockFileInfo* BlockManager::GetBlockFileInfo(size_t n)
789 {
790 LOCK(cs_LastBlockFile);
791
792 if (n >= m_blockfile_info.size()) return nullptr;
793 return &m_blockfile_info.at(n);
794 }
795
796 bool BlockManager::ReadBlockUndo(CBlockUndo& blockundo, const CBlockIndex& index) const
797 {
798 const FlatFilePos pos{WITH_LOCK(::cs_main, return index.GetUndoPos())};
799
800 // Open history file to read
801 AutoFile file{OpenUndoFile(pos, true)};
802 if (file.IsNull()) {
803 LogError("OpenUndoFile failed for %s while reading block undo", pos.ToString());
804 return false;
805 }
806 BufferedReader filein{std::move(file)};
807
808 try {
809 // Read block
810 uint256 hashChecksum;
811 HashVerifier verifier{filein}; // Use HashVerifier as reserializing may lose data, c.f. commit d342424301013ec47dc146a4beb49d5c9319d80a
812 verifier << index.pprev->GetBlockHash();
813 verifier >> blockundo;
814 filein >> hashChecksum;
815
816 // Verify checksum
817 if (hashChecksum != verifier.GetHash()) {
818 LogError("%s: Checksum mismatch at %s\n", __func__, pos.ToString());
819 return false;
820 }
821 } catch (const std::exception& e) {
822 LogError("Deserialize or I/O error - %s at %s while reading block undo", e.what(), pos.ToString());
823 return false;
824 }
825
826 return true;
827 }
828
829 bool BlockManager::FlushUndoFile(int block_file, bool finalize)
830 {
831 FlatFilePos undo_pos_old(block_file, m_blockfile_info[block_file].nUndoSize);
832 if (!m_undo_file_seq.Flush(undo_pos_old, finalize)) {
833 m_opts.notifications.flushError(_("Flushing undo file to disk failed. This is likely the result of an I/O error."));
834 return false;
835 }
836 return true;
837 }
838
839 bool BlockManager::FlushBlockFile(int blockfile_num, bool fFinalize, bool finalize_undo)
840 {
841 bool success = true;
842 LOCK(cs_LastBlockFile);
843
844 if (m_blockfile_info.size() < 1) {
845 // Return if we haven't loaded any blockfiles yet. This happens during
846 // chainstate init, when we call ChainstateManager::MaybeRebalanceCaches() (which
847 // then calls FlushStateToDisk()), resulting in a call to this function before we
848 // have populated `m_blockfile_info` via LoadBlockIndexDB().
849 return true;
850 }
851 assert(static_cast<int>(m_blockfile_info.size()) > blockfile_num);
852
853 FlatFilePos block_pos_old(blockfile_num, m_blockfile_info[blockfile_num].nSize);
854 if (!m_block_file_seq.Flush(block_pos_old, fFinalize)) {
855 m_opts.notifications.flushError(_("Flushing block file to disk failed. This is likely the result of an I/O error."));
856 success = false;
857 }
858 // we do not always flush the undo file, as the chain tip may be lagging behind the incoming blocks,
859 // e.g. during IBD or a sync after a node going offline
860 if (!fFinalize || finalize_undo) {
861 if (!FlushUndoFile(blockfile_num, finalize_undo)) {
862 success = false;
863 }
864 }
865 return success;
866 }
867
868 BlockfileType BlockManager::BlockfileTypeForHeight(int height)
869 {
870 if (!m_snapshot_height) {
871 return BlockfileType::NORMAL;
872 }
873 return (height >= *m_snapshot_height) ? BlockfileType::ASSUMED : BlockfileType::NORMAL;
874 }
875
876 bool BlockManager::FlushChainstateBlockFile(int tip_height)
877 {
878 LOCK(cs_LastBlockFile);
879 auto& cursor = m_blockfile_cursors[BlockfileTypeForHeight(tip_height)];
880 // If the cursor does not exist, it means an assumeutxo snapshot is loaded,
881 // but no blocks past the snapshot height have been written yet, so there
882 // is no data associated with the chainstate, and it is safe not to flush.
883 if (cursor) {
884 return FlushBlockFile(cursor->file_num, /*fFinalize=*/false, /*finalize_undo=*/false);
885 }
886 // No need to log warnings in this case.
887 return true;
888 }
889
890 uint64_t BlockManager::CalculateCurrentUsage()
891 {
892 LOCK(cs_LastBlockFile);
893
894 uint64_t retval = 0;
895 for (const CBlockFileInfo& file : m_blockfile_info) {
896 retval += file.nSize + file.nUndoSize;
897 }
898 return retval;
899 }
900
901 void BlockManager::UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) const
902 {
903 std::error_code ec;
904 for (std::set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) {
905 FlatFilePos pos(*it, 0);
906 const bool removed_blockfile{fs::remove(m_block_file_seq.FileName(pos), ec)};
907 const bool removed_undofile{fs::remove(m_undo_file_seq.FileName(pos), ec)};
908 if (removed_blockfile || removed_undofile) {
909 LogDebug(BCLog::BLOCKSTORAGE, "Prune: %s deleted blk/rev (%05u)\n", __func__, *it);
910 }
911 }
912 }
913
914 AutoFile BlockManager::OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) const
915 {
916 return AutoFile{m_block_file_seq.Open(pos, fReadOnly), m_xor_key};
917 }
918
919 /** Open an undo file (rev?????.dat) */
920 AutoFile BlockManager::OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) const
921 {
922 return AutoFile{m_undo_file_seq.Open(pos, fReadOnly), m_xor_key};
923 }
924
925 fs::path BlockManager::GetBlockPosFilename(const FlatFilePos& pos) const
926 {
927 return m_block_file_seq.FileName(pos);
928 }
929
930 FlatFilePos BlockManager::FindNextBlockPos(unsigned int nAddSize, unsigned int nHeight, uint64_t nTime)
931 {
932 LOCK(cs_LastBlockFile);
933
934 const BlockfileType chain_type = BlockfileTypeForHeight(nHeight);
935
936 if (!m_blockfile_cursors[chain_type]) {
937 // If a snapshot is loaded during runtime, we may not have initialized this cursor yet.
938 assert(chain_type == BlockfileType::ASSUMED);
939 const auto new_cursor = BlockfileCursor{this->MaxBlockfileNum() + 1};
940 m_blockfile_cursors[chain_type] = new_cursor;
941 LogDebug(BCLog::BLOCKSTORAGE, "[%s] initializing blockfile cursor to %s\n", chain_type, new_cursor);
942 }
943 const int last_blockfile = m_blockfile_cursors[chain_type]->file_num;
944
945 int nFile = last_blockfile;
946 if (static_cast<int>(m_blockfile_info.size()) <= nFile) {
947 m_blockfile_info.resize(nFile + 1);
948 }
949
950 bool finalize_undo = false;
951 unsigned int max_blockfile_size{MAX_BLOCKFILE_SIZE};
952 // Use smaller blockfiles in test-only -fastprune mode - but avoid
953 // the possibility of having a block not fit into the block file.
954 if (m_opts.fast_prune) {
955 max_blockfile_size = 0x10000; // 64kiB
956 if (nAddSize >= max_blockfile_size) {
957 // dynamically adjust the blockfile size to be larger than the added size
958 max_blockfile_size = nAddSize + 1;
959 }
960 }
961 assert(nAddSize < max_blockfile_size);
962
963 while (m_blockfile_info[nFile].nSize + nAddSize >= max_blockfile_size) {
964 // when the undo file is keeping up with the block file, we want to flush it explicitly
965 // when it is lagging behind (more blocks arrive than are being connected), we let the
966 // undo block write case handle it
967 finalize_undo = (static_cast<int>(m_blockfile_info[nFile].nHeightLast) ==
968 Assert(m_blockfile_cursors[chain_type])->undo_height);
969
970 // Try the next unclaimed blockfile number
971 nFile = this->MaxBlockfileNum() + 1;
972 // Set to increment MaxBlockfileNum() for next iteration
973 m_blockfile_cursors[chain_type] = BlockfileCursor{nFile};
974
975 if (static_cast<int>(m_blockfile_info.size()) <= nFile) {
976 m_blockfile_info.resize(nFile + 1);
977 }
978 }
979 FlatFilePos pos;
980 pos.nFile = nFile;
981 pos.nPos = m_blockfile_info[nFile].nSize;
982
983 if (nFile != last_blockfile) {
984 LogDebug(BCLog::BLOCKSTORAGE, "Leaving block file %i: %s (onto %i) (height %i)\n",
985 last_blockfile, m_blockfile_info[last_blockfile].ToString(), nFile, nHeight);
986
987 // Do not propagate the return code. The flush concerns a previous block
988 // and undo file that has already been written to. If a flush fails
989 // here, and we crash, there is no expected additional block data
990 // inconsistency arising from the flush failure here. However, the undo
991 // data may be inconsistent after a crash if the flush is called during
992 // a reindex. A flush error might also leave some of the data files
993 // untrimmed.
994 if (!FlushBlockFile(last_blockfile, /*fFinalize=*/true, finalize_undo)) {
995 LogPrintLevel(BCLog::BLOCKSTORAGE, BCLog::Level::Warning,
996 "Failed to flush previous block file %05i (finalize=1, finalize_undo=%i) before opening new block file %05i\n",
997 last_blockfile, finalize_undo, nFile);
998 }
999 // No undo data yet in the new file, so reset our undo-height tracking.
1000 m_blockfile_cursors[chain_type] = BlockfileCursor{nFile};
1001 }
1002
1003 m_blockfile_info[nFile].AddBlock(nHeight, nTime);
1004 m_blockfile_info[nFile].nSize += nAddSize;
1005
1006 bool out_of_space;
1007 size_t bytes_allocated = m_block_file_seq.Allocate(pos, nAddSize, out_of_space);
1008 if (out_of_space) {
1009 m_opts.notifications.fatalError(_("Disk space is too low!"));
1010 return {};
1011 }
1012 if (bytes_allocated != 0 && IsPruneMode()) {
1013 m_check_for_pruning = true;
1014 }
1015
1016 m_dirty_fileinfo.insert(nFile);
1017 return pos;
1018 }
1019
1020 void BlockManager::UpdateBlockInfo(const CBlock& block, unsigned int nHeight, const FlatFilePos& pos)
1021 {
1022 LOCK(cs_LastBlockFile);
1023
1024 // Update the cursor so it points to the last file.
1025 const BlockfileType chain_type{BlockfileTypeForHeight(nHeight)};
1026 auto& cursor{m_blockfile_cursors[chain_type]};
1027 if (!cursor || cursor->file_num < pos.nFile) {
1028 m_blockfile_cursors[chain_type] = BlockfileCursor{pos.nFile};
1029 }
1030
1031 // Update the file information with the current block.
1032 const unsigned int added_size = ::GetSerializeSize(TX_WITH_WITNESS(block));
1033 const int nFile = pos.nFile;
1034 if (static_cast<int>(m_blockfile_info.size()) <= nFile) {
1035 m_blockfile_info.resize(nFile + 1);
1036 }
1037 m_blockfile_info[nFile].AddBlock(nHeight, block.GetBlockTime());
1038 m_blockfile_info[nFile].nSize = std::max(pos.nPos + added_size, m_blockfile_info[nFile].nSize);
1039 m_dirty_fileinfo.insert(nFile);
1040 }
1041
1042 bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize)
1043 {
1044 pos.nFile = nFile;
1045
1046 LOCK(cs_LastBlockFile);
1047
1048 pos.nPos = m_blockfile_info[nFile].nUndoSize;
1049 m_blockfile_info[nFile].nUndoSize += nAddSize;
1050 m_dirty_fileinfo.insert(nFile);
1051
1052 bool out_of_space;
1053 size_t bytes_allocated = m_undo_file_seq.Allocate(pos, nAddSize, out_of_space);
1054 if (out_of_space) {
1055 return FatalError(m_opts.notifications, state, _("Disk space is too low!"));
1056 }
1057 if (bytes_allocated != 0 && IsPruneMode()) {
1058 m_check_for_pruning = true;
1059 }
1060
1061 return true;
1062 }
1063
1064 bool BlockManager::WriteBlockUndo(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block)
1065 {
1066 AssertLockHeld(::cs_main);
1067 const BlockfileType type = BlockfileTypeForHeight(block.nHeight);
1068 auto& cursor = *Assert(WITH_LOCK(cs_LastBlockFile, return m_blockfile_cursors[type]));
1069
1070 // Write undo information to disk
1071 if (block.GetUndoPos().IsNull()) {
1072 FlatFilePos pos;
1073 const auto blockundo_size{static_cast<uint32_t>(GetSerializeSize(blockundo))};
1074 if (!FindUndoPos(state, block.nFile, pos, blockundo_size + UNDO_DATA_DISK_OVERHEAD)) {
1075 LogError("FindUndoPos failed for %s while writing block undo", pos.ToString());
1076 return false;
1077 }
1078
1079 // Open history file to append
1080 AutoFile file{OpenUndoFile(pos)};
1081 if (file.IsNull()) {
1082 LogError("OpenUndoFile failed for %s while writing block undo", pos.ToString());
1083 return FatalError(m_opts.notifications, state, _("Failed to write undo data."));
1084 }
1085 {
1086 BufferedWriter fileout{file};
1087
1088 // Write index header
1089 fileout << GetParams().MessageStart() << blockundo_size;
1090 pos.nPos += BLOCK_SERIALIZATION_HEADER_SIZE;
1091 {
1092 // Calculate checksum
1093 HashWriter hasher{};
1094 hasher << block.pprev->GetBlockHash() << blockundo;
1095 // Write undo data & checksum
1096 fileout << blockundo << hasher.GetHash();
1097 }
1098 // BufferedWriter will flush pending data to file when fileout goes out of scope.
1099 }
1100
1101 // Make sure that the file is closed before we call `FlushUndoFile`.
1102 if (file.fclose() != 0) {
1103 LogError("Failed to close block undo file %s: %s", pos.ToString(), SysErrorString(errno));
1104 return FatalError(m_opts.notifications, state, _("Failed to close block undo file."));
1105 }
1106
1107 // rev files are written in block height order, whereas blk files are written as blocks come in (often out of order)
1108 // we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height
1109 // in the block file info as below; note that this does not catch the case where the undo writes are keeping up
1110 // with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in
1111 // the FindNextBlockPos function
1112 if (pos.nFile < cursor.file_num && static_cast<uint32_t>(block.nHeight) == m_blockfile_info[pos.nFile].nHeightLast) {
1113 // Do not propagate the return code, a failed flush here should not
1114 // be an indication for a failed write. If it were propagated here,
1115 // the caller would assume the undo data not to be written, when in
1116 // fact it is. Note though, that a failed flush might leave the data
1117 // file untrimmed.
1118 if (!FlushUndoFile(pos.nFile, true)) {
1119 LogPrintLevel(BCLog::BLOCKSTORAGE, BCLog::Level::Warning, "Failed to flush undo file %05i\n", pos.nFile);
1120 }
1121 } else if (pos.nFile == cursor.file_num && block.nHeight > cursor.undo_height) {
1122 cursor.undo_height = block.nHeight;
1123 }
1124 // update nUndoPos in block index
1125 block.nUndoPos = pos.nPos;
1126 block.nStatus |= BLOCK_HAVE_UNDO;
1127 m_dirty_blockindex.insert(&block);
1128 }
1129
1130 return true;
1131 }
1132
1133 bool BlockManager::ReadBlock(CBlock& block, const FlatFilePos& pos, const std::optional<uint256>& expected_hash, const bool lowprio) const
1134 {
1135 block.SetNull();
1136
1137 // Open history file to read
1138 std::vector<uint8_t> block_data;
1139 if (!ReadRawBlock(block_data, pos, /*lowprio=*/lowprio)) {
1140 return false;
1141 }
1142
1143 // Read block
1144 try {
1145 SpanReader{block_data} >> TX_WITH_WITNESS(block);
1146 } catch (const std::exception& e) {
1147 LogError("%s: Deserialize or I/O error - %s at %s\n", __func__, e.what(), pos.ToString());
1148 return false;
1149 }
1150
1151 const auto block_hash{block.GetHash()};
1152
1153 // Check the header (sanity check on an already-validated block).
1154 if (!CheckProofOfWork(block_hash, block.nBits, GetConsensus())) {
1155 LogError("%s: Errors in block header at %s\n", __func__, pos.ToString());
1156 return false;
1157 }
1158
1159 // Signet only: check block solution
1160 if (GetConsensus().signet_blocks && !CheckSignetBlockSolution(block, GetConsensus())) {
1161 LogError("%s: Errors in block solution at %s\n", __func__, pos.ToString());
1162 return false;
1163 }
1164
1165 if (expected_hash && block_hash != *expected_hash) {
1166 LogError("GetHash() doesn't match index at %s while reading block (%s != %s)",
1167 pos.ToString(), block_hash.ToString(), expected_hash->ToString());
1168 return false;
1169 }
1170
1171 return true;
1172 }
1173
1174 bool BlockManager::ReadBlock(CBlock& block, const CBlockIndex& index, const bool lowprio) const
1175 {
1176 const FlatFilePos block_pos{WITH_LOCK(cs_main, return index.GetBlockPos())};
1177 return ReadBlock(block, block_pos, index.GetBlockHash(), /*lowprio=*/ lowprio);
1178 }
1179
1180 bool BlockManager::ReadRawBlock(std::vector<uint8_t>& block, const FlatFilePos& pos, const bool lowprio) const
1181 {
1182 if (pos.nPos < BLOCK_SERIALIZATION_HEADER_SIZE) {
1183 // If nPos is less than BLOCK_SERIALIZATION_HEADER_SIZE, we can't read the header that precedes the block data
1184 // This would cause an unsigned integer underflow when trying to position the file cursor
1185 // This can happen after pruning or default constructed positions
1186 LogError("%s: OpenBlockFile failed for %s\n", __func__, pos.ToString());
1187 return false;
1188 }
1189
1190 IOPRIO_IDLER(lowprio);
1191
1192 AutoFile filein{OpenBlockFile({pos.nFile, pos.nPos - BLOCK_SERIALIZATION_HEADER_SIZE}, /*fReadOnly=*/true)};
1193 if (filein.IsNull()) {
1194 LogError("%s: OpenBlockFile failed for %s\n", __func__, pos.ToString());
1195 return false;
1196 }
1197
1198 if (lowprio) filein.SetIdlePriority();
1199
1200 try {
1201 MessageStartChars blk_start;
1202 unsigned int blk_size;
1203
1204 filein >> blk_start >> blk_size;
1205
1206 if (blk_start != GetParams().MessageStart()) {
1207 LogError("%s: Block magic mismatch for %s: %s versus expected %s\n", __func__, pos.ToString(),
1208 HexStr(blk_start),
1209 HexStr(GetParams().MessageStart()));
1210 return false;
1211 }
1212
1213 if (blk_size > MAX_SIZE) {
1214 LogError("%s: Block data is larger than maximum deserialization size for %s: %s versus %s\n", __func__, pos.ToString(),
1215 blk_size, MAX_SIZE);
1216 return false;
1217 }
1218
1219 block.resize(blk_size); // Zeroing of memory is intentional here
1220 filein.read(MakeWritableByteSpan(block));
1221 } catch (const std::exception& e) {
1222 LogError("%s: Read from block file failed: %s for %s\n", __func__, e.what(), pos.ToString());
1223 return false;
1224 }
1225
1226 return true;
1227 }
1228
1229 FlatFilePos BlockManager::WriteBlock(const CBlock& block, int nHeight)
1230 {
1231 const unsigned int block_size{static_cast<unsigned int>(GetSerializeSize(TX_WITH_WITNESS(block)))};
1232 FlatFilePos pos{FindNextBlockPos(block_size + BLOCK_SERIALIZATION_HEADER_SIZE, nHeight, block.GetBlockTime())};
1233 if (pos.IsNull()) {
1234 LogError("FindNextBlockPos failed for %s while writing block", pos.ToString());
1235 return FlatFilePos();
1236 }
1237 AutoFile file{OpenBlockFile(pos, /*fReadOnly=*/false)};
1238 if (file.IsNull()) {
1239 LogError("OpenBlockFile failed for %s while writing block", pos.ToString());
1240 m_opts.notifications.fatalError(_("Failed to write block."));
1241 return FlatFilePos();
1242 }
1243 {
1244 BufferedWriter fileout{file};
1245
1246 // Write index header
1247 fileout << GetParams().MessageStart() << block_size;
1248 // Write block
1249 pos.nPos += BLOCK_SERIALIZATION_HEADER_SIZE;
1250 fileout << TX_WITH_WITNESS(block);
1251 }
1252
1253 if (file.fclose() != 0) {
1254 LogError("Failed to close block file %s: %s", pos.ToString(), SysErrorString(errno));
1255 m_opts.notifications.fatalError(_("Failed to close file when writing block."));
1256 return FlatFilePos();
1257 }
1258
1259 return pos;
1260 }
1261
1262 static auto InitBlocksdirXorKey(const BlockManager::Options& opts)
1263 {
1264 // Bytes are serialized without length indicator, so this is also the exact
1265 // size of the XOR-key file.
1266 std::array<std::byte, 8> xor_key{};
1267
1268 // Consider this to be the first run if the blocksdir contains only hidden
1269 // files (those which start with a .). Checking for a fully-empty dir would
1270 // be too aggressive as a .lock file may have already been written.
1271 bool first_run = true;
1272 for (const auto& entry : fs::directory_iterator(opts.blocks_dir)) {
1273 const std::string path = fs::PathToString(entry.path().filename());
1274 if (!entry.is_regular_file() || !path.starts_with('.')) {
1275 first_run = false;
1276 break;
1277 }
1278 }
1279
1280 if (opts.use_xor && first_run) {
1281 // Only use random fresh key when the boolean option is set and on the
1282 // very first start of the program.
1283 FastRandomContext{}.fillrand(xor_key);
1284 }
1285
1286 const fs::path xor_key_path{opts.blocks_dir / "xor.dat"};
1287 if (fs::exists(xor_key_path)) {
1288 // A pre-existing xor key file has priority.
1289 AutoFile xor_key_file{fsbridge::fopen(xor_key_path, "rb")};
1290 xor_key_file >> xor_key;
1291 } else {
1292 // Create initial or missing xor key file
1293 AutoFile xor_key_file{fsbridge::fopen(xor_key_path,
1294 #if 0
1295 "wb" // Temporary workaround for https://github.com/limenka/limenka/issues/30210
1296 #else
1297 "wbx"
1298 #endif
1299 )};
1300 xor_key_file << xor_key;
1301 if (xor_key_file.fclose() != 0) {
1302 throw std::runtime_error{strprintf("Error closing XOR key file %s: %s",
1303 fs::PathToString(xor_key_path),
1304 SysErrorString(errno))};
1305 }
1306 }
1307 // If the user disabled the key, it must be zero.
1308 if (!opts.use_xor && xor_key != decltype(xor_key){}) {
1309 throw std::runtime_error{
1310 strprintf("The blocksdir XOR-key can not be disabled when a random key was already stored! "
1311 "Stored key: '%s', stored path: '%s'.",
1312 HexStr(xor_key), fs::PathToString(xor_key_path)),
1313 };
1314 }
1315 LogInfo("Using obfuscation key for blocksdir *.dat files (%s): '%s'\n", fs::PathToString(opts.blocks_dir), HexStr(xor_key));
1316 return Obfuscation{xor_key};
1317 }
1318
1319 BlockManager::BlockManager(const util::SignalInterrupt& interrupt, Options opts)
1320 : m_prune_mode{opts.prune_target > 0},
1321 m_xor_key{InitBlocksdirXorKey(opts)},
1322 m_opts{std::move(opts)},
1323 m_block_file_seq{FlatFileSeq{m_opts.blocks_dir, "blk", m_opts.fast_prune ? 0x4000 /* 16kB */ : BLOCKFILE_CHUNK_SIZE}},
1324 m_undo_file_seq{FlatFileSeq{m_opts.blocks_dir, "rev", UNDOFILE_CHUNK_SIZE}},
1325 m_interrupt{interrupt}
1326 {
1327 m_block_tree_db = std::make_unique<BlockTreeDB>(m_opts.block_tree_db_params);
1328
1329 if (m_opts.block_tree_db_params.wipe_data) {
1330 m_block_tree_db->WriteReindexing(true);
1331 m_blockfiles_indexed = false;
1332 // If we're reindexing in prune mode, wipe away unusable block files and all undo data files
1333 if (m_prune_mode) {
1334 CleanupBlockRevFiles();
1335 }
1336 }
1337 }
1338
1339 class ImportingNow
1340 {
1341 std::atomic<bool>& m_importing;
1342
1343 public:
1344 ImportingNow(std::atomic<bool>& importing) : m_importing{importing}
1345 {
1346 assert(m_importing == false);
1347 m_importing = true;
1348 }
1349 ~ImportingNow()
1350 {
1351 assert(m_importing == true);
1352 m_importing = false;
1353 }
1354 };
1355
1356 void ImportBlocks(ChainstateManager& chainman, std::span<const fs::path> import_paths)
1357 {
1358 ImportingNow imp{chainman.m_blockman.m_importing};
1359
1360 // -reindex
1361 if (!chainman.m_blockman.m_blockfiles_indexed) {
1362 int nFile = 0;
1363 // Map of disk positions for blocks with unknown parent (only used for reindex);
1364 // parent hash -> child disk position, multiple children can have the same parent.
1365 std::multimap<uint256, FlatFilePos> blocks_with_unknown_parent;
1366 while (true) {
1367 FlatFilePos pos(nFile, 0);
1368 if (!fs::exists(chainman.m_blockman.GetBlockPosFilename(pos))) {
1369 break; // No block files left to reindex
1370 }
1371 AutoFile file{chainman.m_blockman.OpenBlockFile(pos, true)};
1372 if (file.IsNull()) {
1373 break; // This error is logged in OpenBlockFile
1374 }
1375 LogPrintf("Reindexing block file blk%05u.dat...\n", (unsigned int)nFile);
1376 chainman.LoadExternalBlockFile(file, &pos, &blocks_with_unknown_parent);
1377 if (chainman.m_interrupt) {
1378 LogPrintf("Interrupt requested. Exit %s\n", __func__);
1379 return;
1380 }
1381 nFile++;
1382 }
1383 WITH_LOCK(::cs_main, chainman.m_blockman.m_block_tree_db->WriteReindexing(false));
1384 chainman.m_blockman.m_blockfiles_indexed = true;
1385 LogPrintf("Reindexing finished\n");
1386 // To avoid ending up in a situation without genesis block, re-try initializing (no-op if reindexing worked):
1387 chainman.ActiveChainstate().LoadGenesisBlock();
1388 }
1389
1390 // -loadblock=
1391 for (const fs::path& path : import_paths) {
1392 AutoFile file{fsbridge::fopen(path, "rb")};
1393 if (!file.IsNull()) {
1394 LogPrintf("Importing blocks file %s...\n", fs::PathToString(path));
1395 chainman.LoadExternalBlockFile(file);
1396 if (chainman.m_interrupt) {
1397 LogPrintf("Interrupt requested. Exit %s\n", __func__);
1398 return;
1399 }
1400 } else {
1401 LogWarning("Could not open blocks file %s", fs::PathToString(path));
1402 }
1403 }
1404
1405 // scan for better chains in the block chain database, that are not yet connected in the active best chain
1406
1407 // We can't hold cs_main during ActivateBestChain even though we're accessing
1408 // the chainman unique_ptrs since ABC requires us not to be holding cs_main, so retrieve
1409 // the relevant pointers before the ABC call.
1410 for (Chainstate* chainstate : WITH_LOCK(::cs_main, return chainman.GetAll())) {
1411 BlockValidationState state;
1412 if (!chainstate->ActivateBestChain(state, nullptr)) {
1413 chainman.GetNotifications().fatalError(strprintf(_("Failed to connect best block (%s)."), state.ToString()));
1414 return;
1415 }
1416 }
1417 // End scope of ImportingNow
1418 }
1419
1420 std::ostream& operator<<(std::ostream& os, const BlockfileType& type) {
1421 switch(type) {
1422 case BlockfileType::NORMAL: os << "normal"; break;
1423 case BlockfileType::ASSUMED: os << "assumed"; break;
1424 default: os.setstate(std::ios_base::failbit);
1425 }
1426 return os;
1427 }
1428
1429 std::ostream& operator<<(std::ostream& os, const BlockfileCursor& cursor) {
1430 os << strprintf("BlockfileCursor(file_num=%d, undo_height=%d)", cursor.file_num, cursor.undo_height);
1431 return os;
1432 }
1433 } // namespace node
1434