blockstorage.cpp raw
1 // Copyright (c) 2011-present The Bitcoin Core developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4
5 #include <node/blockstorage.h>
6
7 #include <arith_uint256.h>
8 #include <chain.h>
9 #include <consensus/params.h>
10 #include <crypto/hex_base.h>
11 #include <dbwrapper.h>
12 #include <flatfile.h>
13 #include <hash.h>
14 #include <kernel/blockmanager_opts.h>
15 #include <kernel/chainparams.h>
16 #include <kernel/messagestartchars.h>
17 #include <kernel/notifications_interface.h>
18 #include <kernel/types.h>
19 #include <pow.h>
20 #include <primitives/block.h>
21 #include <primitives/transaction.h>
22 #include <random.h>
23 #include <serialize.h>
24 #include <signet.h>
25 #include <streams.h>
26 #include <sync.h>
27 #include <tinyformat.h>
28 #include <uint256.h>
29 #include <undo.h>
30 #include <util/check.h>
31 #include <util/expected.h>
32 #include <util/fs.h>
33 #include <util/log.h>
34 #include <util/obfuscation.h>
35 #include <util/overflow.h>
36 #include <util/result.h>
37 #include <util/signalinterrupt.h>
38 #include <util/strencodings.h>
39 #include <util/syserror.h>
40 #include <util/time.h>
41 #include <util/translation.h>
42 #include <validation.h>
43
44 #include <cerrno>
45 #include <compare>
46 #include <cstddef>
47 #include <cstdio>
48 #include <exception>
49 #include <map>
50 #include <optional>
51 #include <ostream>
52 #include <span>
53 #include <stdexcept>
54 #include <system_error>
55 #include <unordered_map>
56
57 namespace kernel {
58 static constexpr uint8_t DB_BLOCK_FILES{'f'};
59 static constexpr uint8_t DB_BLOCK_INDEX{'b'};
60 static constexpr uint8_t DB_FLAG{'F'};
61 static constexpr uint8_t DB_REINDEX_FLAG{'R'};
62 static constexpr uint8_t DB_LAST_BLOCK{'l'};
63 // Keys used in previous version that might still be found in the DB:
64 // BlockTreeDB::DB_TXINDEX_BLOCK{'T'};
65 // BlockTreeDB::DB_TXINDEX{'t'}
66 // BlockTreeDB::ReadFlag("txindex")
67
68 bool BlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo& info)
69 {
70 return Read(std::make_pair(DB_BLOCK_FILES, nFile), info);
71 }
72
73 void BlockTreeDB::WriteReindexing(bool fReindexing)
74 {
75 if (fReindexing) {
76 Write(DB_REINDEX_FLAG, uint8_t{'1'});
77 } else {
78 Erase(DB_REINDEX_FLAG);
79 }
80 }
81
82 void BlockTreeDB::ReadReindexing(bool& fReindexing)
83 {
84 fReindexing = Exists(DB_REINDEX_FLAG);
85 }
86
87 bool BlockTreeDB::ReadLastBlockFile(int& nFile)
88 {
89 return Read(DB_LAST_BLOCK, nFile);
90 }
91
92 void BlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*>>& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo)
93 {
94 CDBBatch batch(*this);
95 for (const auto& [file, info] : fileInfo) {
96 batch.Write(std::make_pair(DB_BLOCK_FILES, file), *info);
97 }
98 batch.Write(DB_LAST_BLOCK, nLastFile);
99 for (const CBlockIndex* bi : blockinfo) {
100 batch.Write(std::make_pair(DB_BLOCK_INDEX, bi->GetBlockHash()), CDiskBlockIndex{bi});
101 }
102 WriteBatch(batch, true);
103 }
104
105 void BlockTreeDB::WriteFlag(const std::string& name, bool fValue)
106 {
107 Write(std::make_pair(DB_FLAG, name), fValue ? uint8_t{'1'} : uint8_t{'0'});
108 }
109
110 bool BlockTreeDB::ReadFlag(const std::string& name, bool& fValue)
111 {
112 uint8_t ch;
113 if (!Read(std::make_pair(DB_FLAG, name), ch)) {
114 return false;
115 }
116 fValue = ch == uint8_t{'1'};
117 return true;
118 }
119
120 bool BlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex, const util::SignalInterrupt& interrupt)
121 {
122 AssertLockHeld(::cs_main);
123 std::unique_ptr<CDBIterator> pcursor(NewIterator());
124 pcursor->Seek(std::make_pair(DB_BLOCK_INDEX, uint256()));
125
126 // Load m_block_index
127 while (pcursor->Valid()) {
128 if (interrupt) return false;
129 std::pair<uint8_t, uint256> key;
130 if (pcursor->GetKey(key) && key.first == DB_BLOCK_INDEX) {
131 CDiskBlockIndex diskindex;
132 if (pcursor->GetValue(diskindex)) {
133 // Construct block index object
134 CBlockIndex* pindexNew = insertBlockIndex(diskindex.ConstructBlockHash());
135 pindexNew->pprev = insertBlockIndex(diskindex.hashPrev);
136 pindexNew->nHeight = diskindex.nHeight;
137 pindexNew->nFile = diskindex.nFile;
138 pindexNew->nDataPos = diskindex.nDataPos;
139 pindexNew->nUndoPos = diskindex.nUndoPos;
140 pindexNew->nVersion = diskindex.nVersion;
141 pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot;
142 pindexNew->nTime = diskindex.nTime;
143 pindexNew->nBits = diskindex.nBits;
144 pindexNew->nNonce = diskindex.nNonce;
145 pindexNew->nStatus = diskindex.nStatus;
146 pindexNew->nTx = diskindex.nTx;
147
148 if (!CheckProofOfWork(pindexNew->GetBlockHash(), pindexNew->nBits, consensusParams)) {
149 LogError("%s: CheckProofOfWork failed: %s\n", __func__, pindexNew->ToString());
150 return false;
151 }
152
153 pcursor->Next();
154 } else {
155 LogError("%s: failed to read value\n", __func__);
156 return false;
157 }
158 } else {
159 break;
160 }
161 }
162
163 return true;
164 }
165
166 std::string CBlockFileInfo::ToString() const
167 {
168 return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, FormatISO8601Date(nTimeFirst), FormatISO8601Date(nTimeLast));
169 }
170 } // namespace kernel
171
172 namespace node {
173
174 bool CBlockIndexWorkComparator::operator()(const CBlockIndex* pa, const CBlockIndex* pb) const
175 {
176 // First sort by most total work, ...
177 if (pa->nChainWork > pb->nChainWork) return false;
178 if (pa->nChainWork < pb->nChainWork) return true;
179
180 // ... then by earliest activatable time, ...
181 if (pa->nSequenceId < pb->nSequenceId) return false;
182 if (pa->nSequenceId > pb->nSequenceId) return true;
183
184 // Use pointer address as tie breaker (should only happen with blocks
185 // loaded from disk, as those share the same id: 0 for blocks on the
186 // best chain, 1 for all others).
187 if (pa < pb) return false;
188 if (pa > pb) return true;
189
190 // Identical blocks.
191 return false;
192 }
193
194 bool CBlockIndexHeightOnlyComparator::operator()(const CBlockIndex* pa, const CBlockIndex* pb) const
195 {
196 return pa->nHeight < pb->nHeight;
197 }
198
199 std::vector<CBlockIndex*> BlockManager::GetAllBlockIndices()
200 {
201 AssertLockHeld(cs_main);
202 std::vector<CBlockIndex*> rv;
203 rv.reserve(m_block_index.size());
204 for (auto& [_, block_index] : m_block_index) {
205 rv.push_back(&block_index);
206 }
207 return rv;
208 }
209
210 CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash)
211 {
212 AssertLockHeld(cs_main);
213 BlockMap::iterator it = m_block_index.find(hash);
214 return it == m_block_index.end() ? nullptr : &it->second;
215 }
216
217 const CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash) const
218 {
219 AssertLockHeld(cs_main);
220 BlockMap::const_iterator it = m_block_index.find(hash);
221 return it == m_block_index.end() ? nullptr : &it->second;
222 }
223
224 CBlockIndex* BlockManager::AddToBlockIndex(const CBlockHeader& block, CBlockIndex*& best_header)
225 {
226 AssertLockHeld(cs_main);
227
228 auto [mi, inserted] = m_block_index.try_emplace(block.GetHash(), block);
229 if (!inserted) {
230 return &mi->second;
231 }
232 CBlockIndex* pindexNew = &(*mi).second;
233
234 // We assign the sequence id to blocks only when the full data is available,
235 // to avoid miners withholding blocks but broadcasting headers, to get a
236 // competitive advantage.
237 pindexNew->nSequenceId = SEQ_ID_INIT_FROM_DISK;
238
239 pindexNew->phashBlock = &((*mi).first);
240 BlockMap::iterator miPrev = m_block_index.find(block.hashPrevBlock);
241 if (miPrev != m_block_index.end()) {
242 pindexNew->pprev = &(*miPrev).second;
243 pindexNew->nHeight = pindexNew->pprev->nHeight + 1;
244 pindexNew->BuildSkip();
245 }
246 pindexNew->nTimeMax = (pindexNew->pprev ? std::max(pindexNew->pprev->nTimeMax, pindexNew->nTime) : pindexNew->nTime);
247 pindexNew->nChainWork = (pindexNew->pprev ? pindexNew->pprev->nChainWork : 0) + GetBlockProof(*pindexNew);
248 pindexNew->RaiseValidity(BLOCK_VALID_TREE);
249 if (best_header == nullptr || best_header->nChainWork < pindexNew->nChainWork) {
250 best_header = pindexNew;
251 }
252
253 m_dirty_blockindex.insert(pindexNew);
254
255 return pindexNew;
256 }
257
258 void BlockManager::AddUnlinkedBlock(CBlockIndex* block)
259 {
260 AssertLockHeld(cs_main);
261 Assume(block != nullptr);
262 Assume(block->nStatus & BLOCK_HAVE_DATA);
263 auto range = m_blocks_unlinked.equal_range(block->pprev);
264 for (auto it = range.first; it != range.second; ++it) {
265 if (it->second == block) return; // don't insert duplicates
266 }
267 m_blocks_unlinked.emplace(block->pprev, block);
268 }
269
270 void BlockManager::PruneOneBlockFile(const int fileNumber)
271 {
272 AssertLockHeld(cs_main);
273
274 for (auto& entry : m_block_index) {
275 CBlockIndex* pindex = &entry.second;
276 if (pindex->nFile == fileNumber) {
277 pindex->nStatus &= ~BLOCK_HAVE_DATA;
278 pindex->nStatus &= ~BLOCK_HAVE_UNDO;
279 pindex->nFile = 0;
280 pindex->nDataPos = 0;
281 pindex->nUndoPos = 0;
282 m_dirty_blockindex.insert(pindex);
283
284 // Prune from m_blocks_unlinked -- any block we prune would have
285 // to be downloaded again in order to consider its chain, at which
286 // point it would be considered as a candidate for
287 // m_blocks_unlinked or setBlockIndexCandidates.
288 auto range = m_blocks_unlinked.equal_range(pindex->pprev);
289 while (range.first != range.second) {
290 std::multimap<CBlockIndex*, CBlockIndex*>::iterator _it = range.first;
291 range.first++;
292 if (_it->second == pindex) {
293 m_blocks_unlinked.erase(_it);
294 }
295 }
296 }
297 }
298
299 m_blockfile_info.at(fileNumber) = CBlockFileInfo{};
300 m_dirty_fileinfo.insert(fileNumber);
301 }
302
303 void BlockManager::FindFilesToPruneManual(
304 std::set<int>& setFilesToPrune,
305 int nManualPruneHeight,
306 const Chainstate& chain)
307 {
308 assert(IsPruneMode() && nManualPruneHeight > 0);
309
310 LOCK(::cs_main);
311 if (chain.m_chain.Height() < 0) {
312 return;
313 }
314
315 const auto [min_block_to_prune, last_block_can_prune] = chain.GetPruneRange(nManualPruneHeight);
316
317 int count = 0;
318 for (int fileNumber = 0; fileNumber < this->MaxBlockfileNum(); fileNumber++) {
319 const auto& fileinfo = m_blockfile_info[fileNumber];
320 if (fileinfo.nSize == 0 || fileinfo.nHeightLast > (unsigned)last_block_can_prune || fileinfo.nHeightFirst < (unsigned)min_block_to_prune) {
321 continue;
322 }
323
324 PruneOneBlockFile(fileNumber);
325 setFilesToPrune.insert(fileNumber);
326 count++;
327 }
328 LogInfo("[%s] Prune (Manual): prune_height=%d removed %d blk/rev pairs",
329 chain.GetRole(), last_block_can_prune, count);
330 }
331
332 void BlockManager::FindFilesToPrune(
333 std::set<int>& setFilesToPrune,
334 int last_prune,
335 const Chainstate& chain,
336 ChainstateManager& chainman)
337 {
338 LOCK(::cs_main);
339 // Compute `target` value with maximum size (in bytes) of blocks below the
340 // `last_prune` height which should be preserved and not pruned. The
341 // `target` value will be derived from the -prune preference provided by the
342 // user. If there is a historical chainstate being used to populate indexes
343 // and validate the snapshot, the target is divided by two so half of the
344 // block storage will be reserved for the historical chainstate, and the
345 // other half will be reserved for the most-work chainstate.
346 const int num_chainstates{chainman.HistoricalChainstate() ? 2 : 1};
347 const auto target = std::max(
348 MIN_DISK_SPACE_FOR_BLOCK_FILES, GetPruneTarget() / num_chainstates);
349 const uint64_t target_sync_height = chainman.m_best_header->nHeight;
350
351 if (chain.m_chain.Height() < 0 || target == 0) {
352 return;
353 }
354 if (static_cast<uint64_t>(chain.m_chain.Height()) <= chainman.GetParams().PruneAfterHeight()) {
355 return;
356 }
357
358 const auto [min_block_to_prune, last_block_can_prune] = chain.GetPruneRange(last_prune);
359
360 uint64_t nCurrentUsage = CalculateCurrentUsage();
361 // We don't check to prune until after we've allocated new space for files
362 // So we should leave a buffer under our target to account for another allocation
363 // before the next pruning.
364 uint64_t nBuffer = BLOCKFILE_CHUNK_SIZE + UNDOFILE_CHUNK_SIZE;
365 uint64_t nBytesToPrune;
366 int count = 0;
367
368 if (nCurrentUsage + nBuffer >= target) {
369 // On a prune event, the chainstate DB is flushed.
370 // To avoid excessive prune events negating the benefit of high dbcache
371 // values, we should not prune too rapidly.
372 // So when pruning in IBD, increase the buffer to avoid a re-prune too soon.
373 const auto chain_tip_height = chain.m_chain.Height();
374 if (chainman.IsInitialBlockDownload() && target_sync_height > (uint64_t)chain_tip_height) {
375 // Since this is only relevant during IBD, we assume blocks are at least 1 MB on average
376 static constexpr uint64_t average_block_size = 1000000; /* 1 MB */
377 const uint64_t remaining_blocks = target_sync_height - chain_tip_height;
378 nBuffer += average_block_size * remaining_blocks;
379 }
380
381 for (int fileNumber = 0; fileNumber < this->MaxBlockfileNum(); fileNumber++) {
382 const auto& fileinfo = m_blockfile_info[fileNumber];
383 nBytesToPrune = fileinfo.nSize + fileinfo.nUndoSize;
384
385 if (fileinfo.nSize == 0) {
386 continue;
387 }
388
389 if (nCurrentUsage + nBuffer < target) { // are we below our target?
390 break;
391 }
392
393 // don't prune files that could have a block that's not within the allowable
394 // prune range for the chain being pruned.
395 if (fileinfo.nHeightLast > (unsigned)last_block_can_prune || fileinfo.nHeightFirst < (unsigned)min_block_to_prune) {
396 continue;
397 }
398
399 PruneOneBlockFile(fileNumber);
400 // Queue up the files for removal
401 setFilesToPrune.insert(fileNumber);
402 nCurrentUsage -= nBytesToPrune;
403 count++;
404 }
405 }
406
407 LogDebug(BCLog::PRUNE, "[%s] target=%dMiB actual=%dMiB diff=%dMiB min_height=%d max_prune_height=%d removed %d blk/rev pairs\n",
408 chain.GetRole(), target / 1_MiB, nCurrentUsage / 1_MiB,
409 (int64_t(target) - int64_t(nCurrentUsage)) / int64_t(1_MiB),
410 min_block_to_prune, last_block_can_prune, count);
411 }
412
413 void BlockManager::UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) {
414 AssertLockHeld(::cs_main);
415 m_prune_locks[name] = lock_info;
416 }
417
418 bool BlockManager::DeletePruneLock(const std::string& name)
419 {
420 AssertLockHeld(::cs_main);
421 return m_prune_locks.erase(name) > 0;
422 }
423
424 CBlockIndex* BlockManager::InsertBlockIndex(const uint256& hash)
425 {
426 AssertLockHeld(cs_main);
427
428 if (hash.IsNull()) {
429 return nullptr;
430 }
431
432 const auto [mi, inserted]{m_block_index.try_emplace(hash)};
433 CBlockIndex* pindex = &(*mi).second;
434 if (inserted) {
435 pindex->phashBlock = &((*mi).first);
436 }
437 return pindex;
438 }
439
440 bool BlockManager::LoadBlockIndex(const std::optional<uint256>& snapshot_blockhash)
441 {
442 if (!m_block_tree_db->LoadBlockIndexGuts(
443 GetConsensus(), [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); }, m_interrupt)) {
444 return false;
445 }
446
447 if (snapshot_blockhash) {
448 const std::optional<AssumeutxoData> maybe_au_data = GetParams().AssumeutxoForBlockhash(*snapshot_blockhash);
449 if (!maybe_au_data) {
450 m_opts.notifications.fatalError(strprintf(_("Assumeutxo data not found for the given blockhash '%s'."), snapshot_blockhash->ToString()));
451 return false;
452 }
453 const AssumeutxoData& au_data = *Assert(maybe_au_data);
454 m_snapshot_height = au_data.height;
455 CBlockIndex* base{LookupBlockIndex(*snapshot_blockhash)};
456
457 // Since m_chain_tx_count (responsible for estimated progress) isn't persisted
458 // to disk, we must bootstrap the value for assumedvalid chainstates
459 // from the hardcoded assumeutxo chainparams.
460 base->m_chain_tx_count = au_data.m_chain_tx_count;
461 LogInfo("[snapshot] set m_chain_tx_count=%d for %s", au_data.m_chain_tx_count, snapshot_blockhash->ToString());
462 } else {
463 // If this isn't called with a snapshot blockhash, make sure the cached snapshot height
464 // is null. This is relevant during snapshot completion, when the blockman may be loaded
465 // with a height that then needs to be cleared after the snapshot is fully validated.
466 m_snapshot_height.reset();
467 }
468
469 Assert(m_snapshot_height.has_value() == snapshot_blockhash.has_value());
470
471 // Calculate nChainWork
472 std::vector<CBlockIndex*> vSortedByHeight{GetAllBlockIndices()};
473 std::sort(vSortedByHeight.begin(), vSortedByHeight.end(),
474 CBlockIndexHeightOnlyComparator());
475
476 CBlockIndex* previous_index{nullptr};
477 for (CBlockIndex* pindex : vSortedByHeight) {
478 if (m_interrupt) return false;
479 if (previous_index && pindex->nHeight > previous_index->nHeight + 1) {
480 LogError("%s: block index is non-contiguous, index of height %d missing\n", __func__, previous_index->nHeight + 1);
481 return false;
482 }
483 previous_index = pindex;
484 pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex);
485 pindex->nTimeMax = (pindex->pprev ? std::max(pindex->pprev->nTimeMax, pindex->nTime) : pindex->nTime);
486
487 // We can link the chain of blocks for which we've received transactions at some point, or
488 // blocks that are assumed-valid on the basis of snapshot load (see
489 // PopulateAndValidateSnapshot()).
490 // Pruned nodes may have deleted the block.
491 if (pindex->nTx > 0) {
492 if (pindex->pprev) {
493 if (m_snapshot_height && pindex->nHeight == *m_snapshot_height &&
494 pindex->GetBlockHash() == *snapshot_blockhash) {
495 // Should have been set above; don't disturb it with code below.
496 Assert(pindex->m_chain_tx_count > 0);
497 } else if (pindex->pprev->m_chain_tx_count > 0) {
498 pindex->m_chain_tx_count = pindex->pprev->m_chain_tx_count + pindex->nTx;
499 } else {
500 pindex->m_chain_tx_count = 0;
501 if (pindex->nStatus & BLOCK_HAVE_DATA) {
502 AddUnlinkedBlock(pindex);
503 }
504 }
505 } else {
506 pindex->m_chain_tx_count = pindex->nTx;
507 }
508 }
509
510 if (pindex->nStatus & BLOCK_FAILED_CHILD) {
511 // BLOCK_FAILED_CHILD is deprecated, but may still exist on disk. Replace it with BLOCK_FAILED_VALID.
512 pindex->nStatus = (pindex->nStatus & ~BLOCK_FAILED_CHILD) | BLOCK_FAILED_VALID;
513 m_dirty_blockindex.insert(pindex);
514 }
515 if (!(pindex->nStatus & BLOCK_FAILED_VALID) && pindex->pprev && (pindex->pprev->nStatus & BLOCK_FAILED_VALID)) {
516 // All descendants of invalid blocks are invalid too.
517 pindex->nStatus |= BLOCK_FAILED_VALID;
518 m_dirty_blockindex.insert(pindex);
519 }
520
521 if (pindex->pprev) {
522 pindex->BuildSkip();
523 }
524 }
525
526 return true;
527 }
528
529 void BlockManager::WriteBlockIndexDB()
530 {
531 AssertLockHeld(::cs_main);
532 std::vector<std::pair<int, const CBlockFileInfo*>> vFiles;
533 vFiles.reserve(m_dirty_fileinfo.size());
534 for (std::set<int>::iterator it = m_dirty_fileinfo.begin(); it != m_dirty_fileinfo.end();) {
535 vFiles.emplace_back(*it, &m_blockfile_info[*it]);
536 m_dirty_fileinfo.erase(it++);
537 }
538 std::vector<const CBlockIndex*> vBlocks;
539 vBlocks.reserve(m_dirty_blockindex.size());
540 for (std::set<CBlockIndex*>::iterator it = m_dirty_blockindex.begin(); it != m_dirty_blockindex.end();) {
541 vBlocks.push_back(*it);
542 m_dirty_blockindex.erase(it++);
543 }
544 int max_blockfile{this->MaxBlockfileNum()};
545 m_block_tree_db->WriteBatchSync(vFiles, max_blockfile, vBlocks);
546 }
547
548 bool BlockManager::LoadBlockIndexDB(const std::optional<uint256>& snapshot_blockhash)
549 {
550 AssertLockHeld(::cs_main);
551 if (!LoadBlockIndex(snapshot_blockhash)) {
552 return false;
553 }
554 int max_blockfile_num{0};
555
556 // Load block file info
557 m_block_tree_db->ReadLastBlockFile(max_blockfile_num);
558 m_blockfile_info.resize(max_blockfile_num + 1);
559 LogInfo("Loading block index db: last block file = %i", max_blockfile_num);
560 for (int nFile = 0; nFile <= max_blockfile_num; nFile++) {
561 m_block_tree_db->ReadBlockFileInfo(nFile, m_blockfile_info[nFile]);
562 }
563 LogInfo("Loading block index db: last block file info: %s", m_blockfile_info[max_blockfile_num].ToString());
564 for (int nFile = max_blockfile_num + 1; true; nFile++) {
565 CBlockFileInfo info;
566 if (m_block_tree_db->ReadBlockFileInfo(nFile, info)) {
567 m_blockfile_info.push_back(info);
568 } else {
569 break;
570 }
571 }
572
573 // Check presence of blk files
574 LogInfo("Checking all blk files are present...");
575 std::set<int> setBlkDataFiles;
576 for (const auto& [_, block_index] : m_block_index) {
577 if (block_index.nStatus & BLOCK_HAVE_DATA) {
578 setBlkDataFiles.insert(block_index.nFile);
579 }
580 }
581 for (std::set<int>::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++) {
582 FlatFilePos pos(*it, 0);
583 if (OpenBlockFile(pos, /*fReadOnly=*/true).IsNull()) {
584 return false;
585 }
586 }
587
588 {
589 // Initialize the blockfile cursors.
590 for (size_t i = 0; i < m_blockfile_info.size(); ++i) {
591 const auto last_height_in_file = m_blockfile_info[i].nHeightLast;
592 m_blockfile_cursors[BlockfileTypeForHeight(last_height_in_file)] = {static_cast<int>(i), 0};
593 }
594 }
595
596 // Check whether we have ever pruned block & undo files
597 m_block_tree_db->ReadFlag("prunedblockfiles", m_have_pruned);
598 if (m_have_pruned) {
599 LogInfo("Loading block index db: Block files have previously been pruned");
600 }
601
602 // Check whether we need to continue reindexing
603 bool fReindexing = false;
604 m_block_tree_db->ReadReindexing(fReindexing);
605 if (fReindexing) m_blockfiles_indexed = false;
606
607 return true;
608 }
609
610 void BlockManager::ScanAndUnlinkAlreadyPrunedFiles()
611 {
612 AssertLockHeld(::cs_main);
613 int max_blockfile{this->MaxBlockfileNum()};
614 if (!m_have_pruned) {
615 return;
616 }
617
618 std::set<int> block_files_to_prune;
619 for (int file_number = 0; file_number < max_blockfile; file_number++) {
620 if (m_blockfile_info[file_number].nSize == 0) {
621 block_files_to_prune.insert(file_number);
622 }
623 }
624
625 UnlinkPrunedFiles(block_files_to_prune);
626 }
627
628 bool BlockManager::IsBlockPruned(const CBlockIndex& block) const
629 {
630 AssertLockHeld(::cs_main);
631 return m_have_pruned && !(block.nStatus & BLOCK_HAVE_DATA) && (block.nTx > 0);
632 }
633
634 const CBlockIndex& BlockManager::GetFirstBlock(const CBlockIndex& upper_block, uint32_t status_mask, const CBlockIndex* lower_block) const
635 {
636 AssertLockHeld(::cs_main);
637 const CBlockIndex* last_block = &upper_block;
638 assert((last_block->nStatus & status_mask) == status_mask); // 'upper_block' must satisfy the status mask
639 while (last_block->pprev && ((last_block->pprev->nStatus & status_mask) == status_mask)) {
640 if (lower_block) {
641 // Return if we reached the lower_block
642 if (last_block == lower_block) return *lower_block;
643 // if range was surpassed, means that 'lower_block' is not part of the 'upper_block' chain
644 // and so far this is not allowed.
645 assert(last_block->nHeight >= lower_block->nHeight);
646 }
647 last_block = last_block->pprev;
648 }
649 assert(last_block != nullptr);
650 return *last_block;
651 }
652
653 bool BlockManager::CheckBlockDataAvailability(const CBlockIndex& upper_block, const CBlockIndex& lower_block, BlockStatus block_status)
654 {
655 if (!(upper_block.nStatus & block_status)) return false;
656 const auto& first_block = GetFirstBlock(upper_block, block_status, &lower_block);
657 // Special case: the genesis block has no undo data
658 if (block_status & BLOCK_HAVE_UNDO && lower_block.nHeight == 0 && first_block.nHeight == 1) {
659 // This might indicate missing data, or it could simply reflect the expected absence of undo data for the genesis block.
660 // To distinguish between the two, check if all required block data *except* undo is available up to the genesis block.
661 BlockStatus flags{block_status & ~BLOCK_HAVE_UNDO};
662 return first_block.pprev && first_block.pprev->nStatus & flags;
663 }
664 return &first_block == &lower_block;
665 }
666
667 // If we're using -prune with -reindex, then delete block files that will be ignored by the
668 // reindex. Since reindexing works by starting at block file 0 and looping until a blockfile
669 // is missing, do the same here to delete any later block files after a gap. Also delete all
670 // rev files since they'll be rewritten by the reindex anyway. This ensures that m_blockfile_info
671 // is in sync with what's actually on disk by the time we start downloading, so that pruning
672 // works correctly.
673 void BlockManager::CleanupBlockRevFiles() const
674 {
675 std::map<std::string, fs::path> mapBlockFiles;
676
677 // Glob all blk?????.dat and rev?????.dat files from the blocks directory.
678 // Remove the rev files immediately and insert the blk file paths into an
679 // ordered map keyed by block file index.
680 LogInfo("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune");
681 for (fs::directory_iterator it(m_opts.blocks_dir); it != fs::directory_iterator(); it++) {
682 const std::string path = fs::PathToString(it->path().filename());
683 if (fs::is_regular_file(*it) &&
684 path.length() == 12 &&
685 path.ends_with(".dat"))
686 {
687 if (path.starts_with("blk")) {
688 mapBlockFiles[path.substr(3, 5)] = it->path();
689 } else if (path.starts_with("rev")) {
690 remove(it->path());
691 }
692 }
693 }
694
695 // Remove all block files that aren't part of a contiguous set starting at
696 // zero by walking the ordered map (keys are block file indices) by
697 // keeping a separate counter. Once we hit a gap (or if 0 doesn't exist)
698 // start removing block files.
699 int nContigCounter = 0;
700 for (const std::pair<const std::string, fs::path>& item : mapBlockFiles) {
701 if (LocaleIndependentAtoi<int>(item.first) == nContigCounter) {
702 nContigCounter++;
703 continue;
704 }
705 remove(item.second);
706 }
707 }
708
709 CBlockFileInfo* BlockManager::GetBlockFileInfo(size_t n)
710 {
711 AssertLockHeld(::cs_main);
712 return &m_blockfile_info.at(n);
713 }
714
715 bool BlockManager::ReadBlockUndo(CBlockUndo& blockundo, const CBlockIndex& index) const
716 {
717 const FlatFilePos pos{WITH_LOCK(::cs_main, return index.GetUndoPos())};
718
719 // Open history file to read
720 AutoFile file{OpenUndoFile(pos, true)};
721 if (file.IsNull()) {
722 LogError("OpenUndoFile failed for %s while reading block undo", pos.ToString());
723 return false;
724 }
725 BufferedReader filein{std::move(file)};
726
727 try {
728 // Read block
729 HashVerifier verifier{filein}; // Use HashVerifier, as reserializing may lose data, c.f. commit d3424243
730
731 verifier << index.pprev->GetBlockHash();
732 verifier >> blockundo;
733
734 uint256 hashChecksum;
735 filein >> hashChecksum;
736
737 // Verify checksum
738 if (hashChecksum != verifier.GetHash()) {
739 LogError("Checksum mismatch at %s while reading block undo", pos.ToString());
740 return false;
741 }
742 } catch (const std::exception& e) {
743 LogError("Deserialize or I/O error - %s at %s while reading block undo", e.what(), pos.ToString());
744 return false;
745 }
746
747 return true;
748 }
749
750 bool BlockManager::FlushUndoFile(int block_file, bool finalize)
751 {
752 FlatFilePos undo_pos_old(block_file, m_blockfile_info[block_file].nUndoSize);
753 if (!m_undo_file_seq.Flush(undo_pos_old, finalize)) {
754 m_opts.notifications.flushError(_("Flushing undo file to disk failed. This is likely the result of an I/O error."));
755 return false;
756 }
757 return true;
758 }
759
760 bool BlockManager::FlushBlockFile(int blockfile_num, bool fFinalize, bool finalize_undo)
761 {
762 AssertLockHeld(::cs_main);
763 bool success = true;
764
765 if (m_blockfile_info.size() < 1) {
766 // Return if we haven't loaded any blockfiles yet. This happens during
767 // chainstate init, when we call ChainstateManager::MaybeRebalanceCaches() (which
768 // then calls FlushStateToDisk()), resulting in a call to this function before we
769 // have populated `m_blockfile_info` via LoadBlockIndexDB().
770 return true;
771 }
772 assert(static_cast<int>(m_blockfile_info.size()) > blockfile_num);
773
774 FlatFilePos block_pos_old(blockfile_num, m_blockfile_info[blockfile_num].nSize);
775 if (!m_block_file_seq.Flush(block_pos_old, fFinalize)) {
776 m_opts.notifications.flushError(_("Flushing block file to disk failed. This is likely the result of an I/O error."));
777 success = false;
778 }
779 // we do not always flush the undo file, as the chain tip may be lagging behind the incoming blocks,
780 // e.g. during IBD or a sync after a node going offline
781 if (!fFinalize || finalize_undo) {
782 if (!FlushUndoFile(blockfile_num, finalize_undo)) {
783 success = false;
784 }
785 }
786 return success;
787 }
788
789 BlockfileType BlockManager::BlockfileTypeForHeight(int height)
790 {
791 if (!m_snapshot_height) {
792 return BlockfileType::NORMAL;
793 }
794 return (height >= *m_snapshot_height) ? BlockfileType::ASSUMED : BlockfileType::NORMAL;
795 }
796
797 bool BlockManager::FlushChainstateBlockFile(int tip_height)
798 {
799 AssertLockHeld(::cs_main);
800 auto& cursor = m_blockfile_cursors[BlockfileTypeForHeight(tip_height)];
801 // If the cursor does not exist, it means an assumeutxo snapshot is loaded,
802 // but no blocks past the snapshot height have been written yet, so there
803 // is no data associated with the chainstate, and it is safe not to flush.
804 if (cursor) {
805 return FlushBlockFile(cursor->file_num, /*fFinalize=*/false, /*finalize_undo=*/false);
806 }
807 // No need to log warnings in this case.
808 return true;
809 }
810
811 uint64_t BlockManager::CalculateCurrentUsage()
812 {
813 AssertLockHeld(::cs_main);
814 uint64_t retval = 0;
815 for (const CBlockFileInfo& file : m_blockfile_info) {
816 retval += file.nSize + file.nUndoSize;
817 }
818 return retval;
819 }
820
821 void BlockManager::UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) const
822 {
823 std::error_code ec;
824 for (std::set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) {
825 FlatFilePos pos(*it, 0);
826 const bool removed_blockfile{fs::remove(m_block_file_seq.FileName(pos), ec)};
827 const bool removed_undofile{fs::remove(m_undo_file_seq.FileName(pos), ec)};
828 if (removed_blockfile || removed_undofile) {
829 LogDebug(BCLog::BLOCKSTORAGE, "Prune: %s deleted blk/rev (%05u)\n", __func__, *it);
830 }
831 }
832 }
833
834 AutoFile BlockManager::OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) const
835 {
836 return AutoFile{m_block_file_seq.Open(pos, fReadOnly), m_obfuscation};
837 }
838
839 /** Open an undo file (rev?????.dat) */
840 AutoFile BlockManager::OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) const
841 {
842 return AutoFile{m_undo_file_seq.Open(pos, fReadOnly), m_obfuscation};
843 }
844
845 fs::path BlockManager::GetBlockPosFilename(const FlatFilePos& pos) const
846 {
847 return m_block_file_seq.FileName(pos);
848 }
849
850 FlatFilePos BlockManager::FindNextBlockPos(unsigned int nAddSize, unsigned int nHeight, uint64_t nTime)
851 {
852 AssertLockHeld(::cs_main);
853 const BlockfileType chain_type = BlockfileTypeForHeight(nHeight);
854
855 if (!m_blockfile_cursors[chain_type]) {
856 // If a snapshot is loaded during runtime, we may not have initialized this cursor yet.
857 assert(chain_type == BlockfileType::ASSUMED);
858 const auto new_cursor = BlockfileCursor{this->MaxBlockfileNum() + 1};
859 m_blockfile_cursors[chain_type] = new_cursor;
860 LogDebug(BCLog::BLOCKSTORAGE, "[%s] initializing blockfile cursor to %s\n", chain_type, new_cursor);
861 }
862 const int last_blockfile = m_blockfile_cursors[chain_type]->file_num;
863
864 int nFile = last_blockfile;
865 if (static_cast<int>(m_blockfile_info.size()) <= nFile) {
866 m_blockfile_info.resize(nFile + 1);
867 }
868
869 bool finalize_undo = false;
870 unsigned int max_blockfile_size{MAX_BLOCKFILE_SIZE};
871 // Use smaller blockfiles in test-only -fastprune mode - but avoid
872 // the possibility of having a block not fit into the block file.
873 if (m_opts.fast_prune) {
874 max_blockfile_size = 0x10000; // 64kiB
875 if (nAddSize >= max_blockfile_size) {
876 // dynamically adjust the blockfile size to be larger than the added size
877 max_blockfile_size = nAddSize + 1;
878 }
879 }
880 assert(nAddSize < max_blockfile_size);
881
882 while (m_blockfile_info[nFile].nSize + nAddSize >= max_blockfile_size) {
883 // when the undo file is keeping up with the block file, we want to flush it explicitly
884 // when it is lagging behind (more blocks arrive than are being connected), we let the
885 // undo block write case handle it
886 finalize_undo = (static_cast<int>(m_blockfile_info[nFile].nHeightLast) ==
887 Assert(m_blockfile_cursors[chain_type])->undo_height);
888
889 // Try the next unclaimed blockfile number
890 nFile = this->MaxBlockfileNum() + 1;
891 // Set to increment MaxBlockfileNum() for next iteration
892 m_blockfile_cursors[chain_type] = BlockfileCursor{nFile};
893
894 if (static_cast<int>(m_blockfile_info.size()) <= nFile) {
895 m_blockfile_info.resize(nFile + 1);
896 }
897 }
898 FlatFilePos pos;
899 pos.nFile = nFile;
900 pos.nPos = m_blockfile_info[nFile].nSize;
901
902 if (nFile != last_blockfile) {
903 LogDebug(BCLog::BLOCKSTORAGE, "Leaving block file %i: %s (onto %i) (height %i)\n",
904 last_blockfile, m_blockfile_info[last_blockfile].ToString(), nFile, nHeight);
905
906 // Do not propagate the return code. The flush concerns a previous block
907 // and undo file that has already been written to. If a flush fails
908 // here, and we crash, there is no expected additional block data
909 // inconsistency arising from the flush failure here. However, the undo
910 // data may be inconsistent after a crash if the flush is called during
911 // a reindex. A flush error might also leave some of the data files
912 // untrimmed.
913 if (!FlushBlockFile(last_blockfile, /*fFinalize=*/true, finalize_undo)) {
914 LogWarning(
915 "Failed to flush previous block file %05i (finalize=1, finalize_undo=%i) before opening new block file %05i\n",
916 last_blockfile, finalize_undo, nFile);
917 }
918 // No undo data yet in the new file, so reset our undo-height tracking.
919 m_blockfile_cursors[chain_type] = BlockfileCursor{nFile};
920 }
921
922 m_blockfile_info[nFile].AddBlock(nHeight, nTime);
923 m_blockfile_info[nFile].nSize += nAddSize;
924
925 bool out_of_space;
926 size_t bytes_allocated = m_block_file_seq.Allocate(pos, nAddSize, out_of_space);
927 if (out_of_space) {
928 m_opts.notifications.fatalError(_("Disk space is too low!"));
929 return {};
930 }
931 if (bytes_allocated != 0 && IsPruneMode()) {
932 m_check_for_pruning = true;
933 }
934
935 m_dirty_fileinfo.insert(nFile);
936 return pos;
937 }
938
939 void BlockManager::UpdateBlockInfo(const CBlock& block, unsigned int nHeight, const FlatFilePos& pos)
940 {
941 AssertLockHeld(::cs_main);
942 // Update the cursor so it points to the last file.
943 const BlockfileType chain_type{BlockfileTypeForHeight(nHeight)};
944 auto& cursor{m_blockfile_cursors[chain_type]};
945 if (!cursor || cursor->file_num < pos.nFile) {
946 m_blockfile_cursors[chain_type] = BlockfileCursor{pos.nFile};
947 }
948
949 // Update the file information with the current block.
950 const unsigned int added_size = ::GetSerializeSize(TX_WITH_WITNESS(block));
951 const int nFile = pos.nFile;
952 if (static_cast<int>(m_blockfile_info.size()) <= nFile) {
953 m_blockfile_info.resize(nFile + 1);
954 }
955 m_blockfile_info[nFile].AddBlock(nHeight, block.GetBlockTime());
956 m_blockfile_info[nFile].nSize = std::max(pos.nPos + added_size, m_blockfile_info[nFile].nSize);
957 m_dirty_fileinfo.insert(nFile);
958 }
959
960 bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize)
961 {
962 AssertLockHeld(::cs_main);
963 pos.nFile = nFile;
964
965 pos.nPos = m_blockfile_info[nFile].nUndoSize;
966 m_blockfile_info[nFile].nUndoSize += nAddSize;
967 m_dirty_fileinfo.insert(nFile);
968
969 bool out_of_space;
970 size_t bytes_allocated = m_undo_file_seq.Allocate(pos, nAddSize, out_of_space);
971 if (out_of_space) {
972 return FatalError(m_opts.notifications, state, _("Disk space is too low!"));
973 }
974 if (bytes_allocated != 0 && IsPruneMode()) {
975 m_check_for_pruning = true;
976 }
977
978 return true;
979 }
980
981 bool BlockManager::WriteBlockUndo(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block)
982 {
983 AssertLockHeld(::cs_main);
984 const BlockfileType type = BlockfileTypeForHeight(block.nHeight);
985 auto& cursor = *Assert(m_blockfile_cursors[type]);
986
987 // Write undo information to disk
988 if (block.GetUndoPos().IsNull()) {
989 FlatFilePos pos;
990 const auto blockundo_size{static_cast<uint32_t>(GetSerializeSize(blockundo))};
991 if (!FindUndoPos(state, block.nFile, pos, blockundo_size + UNDO_DATA_DISK_OVERHEAD)) {
992 LogError("FindUndoPos failed for %s while writing block undo", pos.ToString());
993 return false;
994 }
995
996 // Open history file to append
997 AutoFile file{OpenUndoFile(pos)};
998 if (file.IsNull()) {
999 LogError("OpenUndoFile failed for %s while writing block undo", pos.ToString());
1000 return FatalError(m_opts.notifications, state, _("Failed to write undo data."));
1001 }
1002 {
1003 BufferedWriter fileout{file};
1004
1005 // Write index header
1006 fileout << GetParams().MessageStart() << blockundo_size;
1007 pos.nPos += STORAGE_HEADER_BYTES;
1008 {
1009 // Calculate checksum
1010 HashWriter hasher{};
1011 hasher << block.pprev->GetBlockHash() << blockundo;
1012 // Write undo data & checksum
1013 fileout << blockundo << hasher.GetHash();
1014 }
1015 // BufferedWriter will flush pending data to file when fileout goes out of scope.
1016 }
1017
1018 // Make sure that the file is closed before we call `FlushUndoFile`.
1019 if (file.fclose() != 0) {
1020 LogError("Failed to close block undo file %s: %s", pos.ToString(), SysErrorString(errno));
1021 return FatalError(m_opts.notifications, state, _("Failed to close block undo file."));
1022 }
1023
1024 // rev files are written in block height order, whereas blk files are written as blocks come in (often out of order)
1025 // we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height
1026 // in the block file info as below; note that this does not catch the case where the undo writes are keeping up
1027 // with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in
1028 // the FindNextBlockPos function
1029 if (pos.nFile < cursor.file_num && static_cast<uint32_t>(block.nHeight) == m_blockfile_info[pos.nFile].nHeightLast) {
1030 // Do not propagate the return code, a failed flush here should not
1031 // be an indication for a failed write. If it were propagated here,
1032 // the caller would assume the undo data not to be written, when in
1033 // fact it is. Note though, that a failed flush might leave the data
1034 // file untrimmed.
1035 if (!FlushUndoFile(pos.nFile, true)) {
1036 LogWarning("Failed to flush undo file %05i\n", pos.nFile);
1037 }
1038 } else if (pos.nFile == cursor.file_num && block.nHeight > cursor.undo_height) {
1039 cursor.undo_height = block.nHeight;
1040 }
1041 // update nUndoPos in block index
1042 block.nUndoPos = pos.nPos;
1043 block.nStatus |= BLOCK_HAVE_UNDO;
1044 m_dirty_blockindex.insert(&block);
1045 }
1046
1047 return true;
1048 }
1049
1050 bool BlockManager::ReadBlock(CBlock& block, const FlatFilePos& pos, const std::optional<uint256>& expected_hash) const
1051 {
1052 block.SetNull();
1053
1054 // Open history file to read
1055 const auto block_data{ReadRawBlock(pos)};
1056 if (!block_data) {
1057 return false;
1058 }
1059
1060 try {
1061 // Read block
1062 SpanReader{*block_data} >> TX_WITH_WITNESS(block);
1063 } catch (const std::exception& e) {
1064 LogError("Deserialize or I/O error - %s at %s while reading block", e.what(), pos.ToString());
1065 return false;
1066 }
1067
1068 const auto block_hash{block.GetHash()};
1069
1070 // Check the header
1071 if (!CheckProofOfWork(block_hash, block.nBits, GetConsensus())) {
1072 LogError("Errors in block header at %s while reading block", pos.ToString());
1073 return false;
1074 }
1075
1076 // Signet only: check block solution
1077 if (GetConsensus().signet_blocks && !CheckSignetBlockSolution(block, GetConsensus())) {
1078 LogError("Errors in block solution at %s while reading block", pos.ToString());
1079 return false;
1080 }
1081
1082 if (expected_hash && block_hash != *expected_hash) {
1083 LogError("GetHash() doesn't match index at %s while reading block (%s != %s)",
1084 pos.ToString(), block_hash.ToString(), expected_hash->ToString());
1085 return false;
1086 }
1087
1088 return true;
1089 }
1090
1091 bool BlockManager::ReadBlock(CBlock& block, const CBlockIndex& index) const
1092 {
1093 const FlatFilePos block_pos{WITH_LOCK(cs_main, return index.GetBlockPos())};
1094 return ReadBlock(block, block_pos, index.GetBlockHash());
1095 }
1096
1097 BlockManager::ReadRawBlockResult BlockManager::ReadRawBlock(const FlatFilePos& pos, std::optional<std::pair<size_t, size_t>> block_part) const
1098 {
1099 if (pos.nPos < STORAGE_HEADER_BYTES) {
1100 // If nPos is less than STORAGE_HEADER_BYTES, we can't read the header that precedes the block data
1101 // This would cause an unsigned integer underflow when trying to position the file cursor
1102 // This can happen after pruning or default constructed positions
1103 LogError("Failed for %s while reading raw block storage header", pos.ToString());
1104 return util::Unexpected{ReadRawError::IO};
1105 }
1106 AutoFile filein{OpenBlockFile({pos.nFile, pos.nPos - STORAGE_HEADER_BYTES}, /*fReadOnly=*/true)};
1107 if (filein.IsNull()) {
1108 LogError("OpenBlockFile failed for %s while reading raw block", pos.ToString());
1109 return util::Unexpected{ReadRawError::IO};
1110 }
1111
1112 try {
1113 MessageStartChars blk_start;
1114 unsigned int blk_size;
1115
1116 filein >> blk_start >> blk_size;
1117
1118 if (blk_start != GetParams().MessageStart()) {
1119 LogError("Block magic mismatch for %s: %s versus expected %s while reading raw block",
1120 pos.ToString(), HexStr(blk_start), HexStr(GetParams().MessageStart()));
1121 return util::Unexpected{ReadRawError::IO};
1122 }
1123
1124 if (blk_size > MAX_SIZE) {
1125 LogError("Block data is larger than maximum deserialization size for %s: %s versus %s while reading raw block",
1126 pos.ToString(), blk_size, MAX_SIZE);
1127 return util::Unexpected{ReadRawError::IO};
1128 }
1129
1130 if (block_part) {
1131 const auto [offset, size]{*block_part};
1132 if (size == 0 || SaturatingAdd(offset, size) > blk_size) {
1133 return util::Unexpected{ReadRawError::BadPartRange}; // Avoid logging - offset/size come from untrusted REST input
1134 }
1135 filein.seek(offset, SEEK_CUR);
1136 blk_size = size;
1137 }
1138
1139 std::vector<std::byte> data(blk_size); // Zeroing of memory is intentional here
1140 filein.read(data);
1141 return data;
1142 } catch (const std::exception& e) {
1143 LogError("Read from block file failed: %s for %s while reading raw block", e.what(), pos.ToString());
1144 return util::Unexpected{ReadRawError::IO};
1145 }
1146 }
1147
1148 FlatFilePos BlockManager::WriteBlock(const CBlock& block, int nHeight)
1149 {
1150 AssertLockHeld(::cs_main);
1151 const unsigned int block_size{static_cast<unsigned int>(GetSerializeSize(TX_WITH_WITNESS(block)))};
1152 FlatFilePos pos{FindNextBlockPos(block_size + STORAGE_HEADER_BYTES, nHeight, block.GetBlockTime())};
1153 if (pos.IsNull()) {
1154 LogError("FindNextBlockPos failed for %s while writing block", pos.ToString());
1155 return FlatFilePos();
1156 }
1157 AutoFile file{OpenBlockFile(pos, /*fReadOnly=*/false)};
1158 if (file.IsNull()) {
1159 LogError("OpenBlockFile failed for %s while writing block", pos.ToString());
1160 m_opts.notifications.fatalError(_("Failed to write block."));
1161 return FlatFilePos();
1162 }
1163 {
1164 BufferedWriter fileout{file};
1165
1166 // Write index header
1167 fileout << GetParams().MessageStart() << block_size;
1168 pos.nPos += STORAGE_HEADER_BYTES;
1169 // Write block
1170 fileout << TX_WITH_WITNESS(block);
1171 }
1172
1173 if (file.fclose() != 0) {
1174 LogError("Failed to close block file %s: %s", pos.ToString(), SysErrorString(errno));
1175 m_opts.notifications.fatalError(_("Failed to close file when writing block."));
1176 return FlatFilePos();
1177 }
1178
1179 return pos;
1180 }
1181
1182 static auto InitBlocksdirXorKey(const BlockManager::Options& opts)
1183 {
1184 // Bytes are serialized without length indicator, so this is also the exact
1185 // size of the XOR-key file.
1186 std::array<std::byte, Obfuscation::KEY_SIZE> obfuscation{};
1187
1188 // Consider this to be the first run if the blocksdir contains only hidden
1189 // files (those which start with a .). Checking for a fully-empty dir would
1190 // be too aggressive as a .lock file may have already been written.
1191 bool first_run = true;
1192 for (const auto& entry : fs::directory_iterator(opts.blocks_dir)) {
1193 const std::string path = fs::PathToString(entry.path().filename());
1194 if (!entry.is_regular_file() || !path.starts_with('.')) {
1195 first_run = false;
1196 break;
1197 }
1198 }
1199
1200 if (opts.use_xor && first_run) {
1201 // Only use random fresh key when the boolean option is set and on the
1202 // very first start of the program.
1203 FastRandomContext{}.fillrand(obfuscation);
1204 }
1205
1206 const fs::path xor_key_path{opts.blocks_dir / "xor.dat"};
1207 if (fs::exists(xor_key_path)) {
1208 // A pre-existing xor key file has priority.
1209 AutoFile xor_key_file{fsbridge::fopen(xor_key_path, "rb")};
1210 xor_key_file >> obfuscation;
1211 } else {
1212 // Create initial or missing xor key file
1213 AutoFile xor_key_file{fsbridge::fopen(xor_key_path,
1214 #ifdef __MINGW64__
1215 "wb" // Temporary workaround for https://github.com/bitcoin/bitcoin/issues/30210
1216 #else
1217 "wbx"
1218 #endif
1219 )};
1220 xor_key_file << obfuscation;
1221 if (xor_key_file.fclose() != 0) {
1222 throw std::runtime_error{strprintf("Error closing XOR key file %s: %s",
1223 fs::PathToString(xor_key_path),
1224 SysErrorString(errno))};
1225 }
1226 }
1227 // If the user disabled the key, it must be zero.
1228 if (!opts.use_xor && obfuscation != decltype(obfuscation){}) {
1229 throw std::runtime_error{
1230 strprintf("The blocksdir XOR-key can not be disabled when a random key was already stored! "
1231 "Stored key: '%s', stored path: '%s'.",
1232 HexStr(obfuscation), fs::PathToString(xor_key_path)),
1233 };
1234 }
1235 LogInfo("Using obfuscation key for blocksdir *.dat files (%s): '%s'\n", fs::PathToString(opts.blocks_dir), HexStr(obfuscation));
1236 return Obfuscation{obfuscation};
1237 }
1238
1239 BlockManager::BlockManager(const util::SignalInterrupt& interrupt, Options opts)
1240 : m_prune_mode{opts.prune_target > 0},
1241 m_obfuscation{InitBlocksdirXorKey(opts)},
1242 m_opts{std::move(opts)},
1243 m_block_file_seq{FlatFileSeq{m_opts.blocks_dir, "blk", m_opts.fast_prune ? 0x4000 /* 16kB */ : BLOCKFILE_CHUNK_SIZE}},
1244 m_undo_file_seq{FlatFileSeq{m_opts.blocks_dir, "rev", UNDOFILE_CHUNK_SIZE}},
1245 m_interrupt{interrupt}
1246 {
1247 m_block_tree_db = std::make_unique<BlockTreeDB>(m_opts.block_tree_db_params);
1248
1249 if (m_opts.block_tree_db_params.wipe_data) {
1250 m_block_tree_db->WriteReindexing(true);
1251 m_blockfiles_indexed = false;
1252 // If we're reindexing in prune mode, wipe away unusable block files and all undo data files
1253 if (m_prune_mode) {
1254 CleanupBlockRevFiles();
1255 }
1256 }
1257 }
1258
1259 class ImportingNow
1260 {
1261 std::atomic<bool>& m_importing;
1262
1263 public:
1264 ImportingNow(std::atomic<bool>& importing) : m_importing{importing}
1265 {
1266 assert(m_importing == false);
1267 m_importing = true;
1268 }
1269 ~ImportingNow()
1270 {
1271 assert(m_importing == true);
1272 m_importing = false;
1273 }
1274 };
1275
1276 void ImportBlocks(ChainstateManager& chainman, std::span<const fs::path> import_paths)
1277 {
1278 ImportingNow imp{chainman.m_blockman.m_importing};
1279
1280 // -reindex
1281 if (!chainman.m_blockman.m_blockfiles_indexed) {
1282 int total_files{0};
1283 while (fs::exists(chainman.m_blockman.GetBlockPosFilename(FlatFilePos(total_files, 0)))) {
1284 total_files++;
1285 }
1286
1287 // Map of disk positions for blocks with unknown parent (only used for reindex);
1288 // parent hash -> child disk position, multiple children can have the same parent.
1289 std::multimap<uint256, FlatFilePos> blocks_with_unknown_parent;
1290
1291 for (int nFile{0}; nFile < total_files; ++nFile) {
1292 FlatFilePos pos(nFile, 0);
1293 AutoFile file{chainman.m_blockman.OpenBlockFile(pos, /*fReadOnly=*/true)};
1294 if (file.IsNull()) {
1295 break; // This error is logged in OpenBlockFile
1296 }
1297 LogInfo("Reindexing block file blk%05u.dat (%d%% complete)...", (unsigned int)nFile, nFile * 100 / total_files);
1298 chainman.LoadExternalBlockFile(file, &pos, &blocks_with_unknown_parent);
1299 if (chainman.m_interrupt) {
1300 LogInfo("Interrupt requested. Exit reindexing.");
1301 return;
1302 }
1303 }
1304 WITH_LOCK(::cs_main, chainman.m_blockman.m_block_tree_db->WriteReindexing(false));
1305 chainman.m_blockman.m_blockfiles_indexed = true;
1306 LogInfo("Reindexing finished");
1307 // To avoid ending up in a situation without genesis block, re-try initializing (no-op if reindexing worked):
1308 (void)chainman.LoadGenesisBlock();
1309 }
1310
1311 // -loadblock=
1312 for (const fs::path& path : import_paths) {
1313 AutoFile file{fsbridge::fopen(path, "rb")};
1314 if (!file.IsNull()) {
1315 LogInfo("Importing blocks file %s...", fs::PathToString(path));
1316 chainman.LoadExternalBlockFile(file);
1317 if (chainman.m_interrupt) {
1318 LogInfo("Interrupt requested. Exit block importing.");
1319 return;
1320 }
1321 } else {
1322 LogWarning("Could not open blocks file %s", fs::PathToString(path));
1323 }
1324 }
1325
1326 // scan for better chains in the block chain database, that are not yet connected in the active best chain
1327 if (auto result = chainman.ActivateBestChains(); !result) {
1328 chainman.GetNotifications().fatalError(util::ErrorString(result));
1329 }
1330 // End scope of ImportingNow
1331 }
1332
1333 std::ostream& operator<<(std::ostream& os, const BlockfileType& type) {
1334 switch(type) {
1335 case BlockfileType::NORMAL: os << "normal"; break;
1336 case BlockfileType::ASSUMED: os << "assumed"; break;
1337 default: os.setstate(std::ios_base::failbit);
1338 }
1339 return os;
1340 }
1341
1342 std::ostream& operator<<(std::ostream& os, const BlockfileCursor& cursor) {
1343 os << strprintf("BlockfileCursor(file_num=%d, undo_height=%d)", cursor.file_num, cursor.undo_height);
1344 return os;
1345 }
1346 } // namespace node
1347