coinstatsindex.cpp raw
1 // Copyright (c) 2020-2022 The Limenka developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4
5 #include <chainparams.h>
6 #include <coins.h>
7 #include <common/args.h>
8 #include <crypto/muhash.h>
9 #include <index/coinstatsindex.h>
10 #include <kernel/coinstats.h>
11 #include <logging.h>
12 #include <node/blockstorage.h>
13 #include <serialize.h>
14 #include <txdb.h>
15 #include <undo.h>
16 #include <validation.h>
17
18 using kernel::ApplyCoinHash;
19 using kernel::CCoinsStats;
20 using kernel::GetBogoSize;
21 using kernel::RemoveCoinHash;
22
23 static constexpr uint8_t DB_BLOCK_HASH{'s'};
24 static constexpr uint8_t DB_BLOCK_HEIGHT{'t'};
25 static constexpr uint8_t DB_MUHASH{'M'};
26
27 namespace {
28
29 struct DBVal {
30 uint256 muhash;
31 uint64_t transaction_output_count;
32 uint64_t bogo_size;
33 CAmount total_amount;
34 CAmount total_subsidy;
35 CAmount total_unspendable_amount;
36 CAmount total_prevout_spent_amount;
37 CAmount total_new_outputs_ex_coinbase_amount;
38 CAmount total_coinbase_amount;
39 CAmount total_unspendables_genesis_block;
40 CAmount total_unspendables_bip30;
41 CAmount total_unspendables_scripts;
42 CAmount total_unspendables_unclaimed_rewards;
43
44 SERIALIZE_METHODS(DBVal, obj)
45 {
46 READWRITE(obj.muhash);
47 READWRITE(obj.transaction_output_count);
48 READWRITE(obj.bogo_size);
49 READWRITE(obj.total_amount);
50 READWRITE(obj.total_subsidy);
51 READWRITE(obj.total_unspendable_amount);
52 READWRITE(obj.total_prevout_spent_amount);
53 READWRITE(obj.total_new_outputs_ex_coinbase_amount);
54 READWRITE(obj.total_coinbase_amount);
55 READWRITE(obj.total_unspendables_genesis_block);
56 READWRITE(obj.total_unspendables_bip30);
57 READWRITE(obj.total_unspendables_scripts);
58 READWRITE(obj.total_unspendables_unclaimed_rewards);
59 }
60 };
61
62 struct DBHeightKey {
63 int height;
64
65 explicit DBHeightKey(int height_in) : height(height_in) {}
66
67 template <typename Stream>
68 void Serialize(Stream& s) const
69 {
70 ser_writedata8(s, DB_BLOCK_HEIGHT);
71 ser_writedata32be(s, height);
72 }
73
74 template <typename Stream>
75 void Unserialize(Stream& s)
76 {
77 const uint8_t prefix{ser_readdata8(s)};
78 if (prefix != DB_BLOCK_HEIGHT) {
79 throw std::ios_base::failure("Invalid format for coinstatsindex DB height key");
80 }
81 height = ser_readdata32be(s);
82 }
83 };
84
85 struct DBHashKey {
86 uint256 block_hash;
87
88 explicit DBHashKey(const uint256& hash_in) : block_hash(hash_in) {}
89
90 SERIALIZE_METHODS(DBHashKey, obj)
91 {
92 uint8_t prefix{DB_BLOCK_HASH};
93 READWRITE(prefix);
94 if (prefix != DB_BLOCK_HASH) {
95 throw std::ios_base::failure("Invalid format for coinstatsindex DB hash key");
96 }
97
98 READWRITE(obj.block_hash);
99 }
100 };
101
102 }; // namespace
103
104 std::unique_ptr<CoinStatsIndex> g_coin_stats_index;
105
106 CoinStatsIndex::CoinStatsIndex(std::unique_ptr<interfaces::Chain> chain, size_t n_cache_size, bool f_memory, bool f_wipe)
107 : BaseIndex(std::move(chain), "coinstatsindex")
108 {
109 fs::path path{gArgs.GetDataDirNet() / "indexes" / "coinstats"};
110 fs::create_directories(path);
111
112 m_db = std::make_unique<CoinStatsIndex::DB>(path / "db", n_cache_size, f_memory, f_wipe);
113 }
114
115 bool CoinStatsIndex::CustomAppend(const interfaces::BlockInfo& block)
116 {
117 CBlockUndo block_undo;
118 const CAmount block_subsidy{GetBlockSubsidy(block.height, Params().GetConsensus())};
119 m_total_subsidy += block_subsidy;
120
121 // Ignore genesis block
122 if (block.height > 0) {
123 // pindex variable gives indexing code access to node internals. It
124 // will be removed in upcoming commit
125 const CBlockIndex* pindex = WITH_LOCK(cs_main, return m_chainstate->m_blockman.LookupBlockIndex(block.hash));
126 if (!m_chainstate->m_blockman.ReadBlockUndo(block_undo, *pindex)) {
127 return false;
128 }
129
130 std::pair<uint256, DBVal> read_out;
131 if (!m_db->Read(DBHeightKey(block.height - 1), read_out)) {
132 return false;
133 }
134
135 uint256 expected_block_hash{*Assert(block.prev_hash)};
136 if (read_out.first != expected_block_hash) {
137 LogPrintf("WARNING: previous block header belongs to unexpected block %s; expected %s\n",
138 read_out.first.ToString(), expected_block_hash.ToString());
139
140 if (!m_db->Read(DBHashKey(expected_block_hash), read_out)) {
141 LogError("%s: previous block header not found; expected %s\n",
142 __func__, expected_block_hash.ToString());
143 return false;
144 }
145 }
146
147 // Add the new utxos created from the block
148 assert(block.data);
149 for (size_t i = 0; i < block.data->vtx.size(); ++i) {
150 const auto& tx{block.data->vtx.at(i)};
151
152 // Skip duplicate txid coinbase transactions (BIP30).
153 if (IsBIP30Unspendable(*pindex) && tx->IsCoinBase()) {
154 m_total_unspendable_amount += block_subsidy;
155 m_total_unspendables_bip30 += block_subsidy;
156 continue;
157 }
158
159 for (uint32_t j = 0; j < tx->vout.size(); ++j) {
160 const CTxOut& out{tx->vout[j]};
161 Coin coin{out, block.height, tx->IsCoinBase()};
162 COutPoint outpoint{tx->GetHash(), j};
163
164 // Skip unspendable coins
165 if (coin.out.scriptPubKey.IsUnspendable()) {
166 m_total_unspendable_amount += coin.out.nValue;
167 m_total_unspendables_scripts += coin.out.nValue;
168 continue;
169 }
170
171 ApplyCoinHash(m_muhash, outpoint, coin);
172
173 if (tx->IsCoinBase()) {
174 m_total_coinbase_amount += coin.out.nValue;
175 } else {
176 m_total_new_outputs_ex_coinbase_amount += coin.out.nValue;
177 }
178
179 ++m_transaction_output_count;
180 m_total_amount += coin.out.nValue;
181 m_bogo_size += GetBogoSize(coin.out.scriptPubKey);
182 }
183
184 // The coinbase tx has no undo data since no former output is spent
185 if (!tx->IsCoinBase()) {
186 const auto& tx_undo{block_undo.vtxundo.at(i - 1)};
187
188 for (size_t j = 0; j < tx_undo.vprevout.size(); ++j) {
189 Coin coin{tx_undo.vprevout[j]};
190 COutPoint outpoint{tx->vin[j].prevout.hash, tx->vin[j].prevout.n};
191
192 RemoveCoinHash(m_muhash, outpoint, coin);
193
194 m_total_prevout_spent_amount += coin.out.nValue;
195
196 --m_transaction_output_count;
197 m_total_amount -= coin.out.nValue;
198 m_bogo_size -= GetBogoSize(coin.out.scriptPubKey);
199 }
200 }
201 }
202 } else {
203 // genesis block
204 m_total_unspendable_amount += block_subsidy;
205 m_total_unspendables_genesis_block += block_subsidy;
206 }
207
208 // If spent prevouts + block subsidy are still a higher amount than
209 // new outputs + coinbase + current unspendable amount this means
210 // the miner did not claim the full block reward. Unclaimed block
211 // rewards are also unspendable.
212 const CAmount unclaimed_rewards{(m_total_prevout_spent_amount + m_total_subsidy) - (m_total_new_outputs_ex_coinbase_amount + m_total_coinbase_amount + m_total_unspendable_amount)};
213 m_total_unspendable_amount += unclaimed_rewards;
214 m_total_unspendables_unclaimed_rewards += unclaimed_rewards;
215
216 std::pair<uint256, DBVal> value;
217 value.first = block.hash;
218 value.second.transaction_output_count = m_transaction_output_count;
219 value.second.bogo_size = m_bogo_size;
220 value.second.total_amount = m_total_amount;
221 value.second.total_subsidy = m_total_subsidy;
222 value.second.total_unspendable_amount = m_total_unspendable_amount;
223 value.second.total_prevout_spent_amount = m_total_prevout_spent_amount;
224 value.second.total_new_outputs_ex_coinbase_amount = m_total_new_outputs_ex_coinbase_amount;
225 value.second.total_coinbase_amount = m_total_coinbase_amount;
226 value.second.total_unspendables_genesis_block = m_total_unspendables_genesis_block;
227 value.second.total_unspendables_bip30 = m_total_unspendables_bip30;
228 value.second.total_unspendables_scripts = m_total_unspendables_scripts;
229 value.second.total_unspendables_unclaimed_rewards = m_total_unspendables_unclaimed_rewards;
230
231 uint256 out;
232 m_muhash.Finalize(out);
233 value.second.muhash = out;
234
235 // Intentionally do not update DB_MUHASH here so it stays in sync with
236 // DB_BEST_BLOCK, and the index is not corrupted if there is an unclean shutdown.
237 return m_db->Write(DBHeightKey(block.height), value);
238 }
239
240 [[nodiscard]] static bool CopyHeightIndexToHashIndex(CDBIterator& db_it, CDBBatch& batch,
241 const std::string& index_name,
242 int start_height, int stop_height)
243 {
244 DBHeightKey key{start_height};
245 db_it.Seek(key);
246
247 for (int height = start_height; height <= stop_height; ++height) {
248 if (!db_it.GetKey(key) || key.height != height) {
249 LogError("%s: unexpected key in %s: expected (%c, %d)\n",
250 __func__, index_name, DB_BLOCK_HEIGHT, height);
251 return false;
252 }
253
254 std::pair<uint256, DBVal> value;
255 if (!db_it.GetValue(value)) {
256 LogError("%s: unable to read value in %s at key (%c, %d)\n",
257 __func__, index_name, DB_BLOCK_HEIGHT, height);
258 return false;
259 }
260
261 batch.Write(DBHashKey(value.first), std::move(value.second));
262
263 db_it.Next();
264 }
265 return true;
266 }
267
268 bool CoinStatsIndex::CustomRewind(const interfaces::BlockRef& current_tip, const interfaces::BlockRef& new_tip)
269 {
270 CDBBatch batch(*m_db);
271 std::unique_ptr<CDBIterator> db_it(m_db->NewIterator());
272
273 // During a reorg, we need to copy all hash digests for blocks that are
274 // getting disconnected from the height index to the hash index so we can
275 // still find them when the height index entries are overwritten.
276 if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip.height, current_tip.height)) {
277 return false;
278 }
279
280 if (!m_db->WriteBatch(batch)) return false;
281
282 {
283 LOCK(cs_main);
284 const CBlockIndex* iter_tip{m_chainstate->m_blockman.LookupBlockIndex(current_tip.hash)};
285 const CBlockIndex* new_tip_index{m_chainstate->m_blockman.LookupBlockIndex(new_tip.hash)};
286
287 do {
288 CBlock block;
289
290 if (!m_chainstate->m_blockman.ReadBlock(block, *iter_tip)) {
291 LogError("%s: Failed to read block %s from disk\n",
292 __func__, iter_tip->GetBlockHash().ToString());
293 return false;
294 }
295
296 if (!ReverseBlock(block, iter_tip)) {
297 return false; // failure cause logged internally
298 }
299
300 iter_tip = iter_tip->GetAncestor(iter_tip->nHeight - 1);
301 } while (new_tip_index != iter_tip);
302 }
303
304 return true;
305 }
306
307 static bool LookUpOne(const CDBWrapper& db, const interfaces::BlockRef& block, DBVal& result)
308 {
309 // First check if the result is stored under the height index and the value
310 // there matches the block hash. This should be the case if the block is on
311 // the active chain.
312 std::pair<uint256, DBVal> read_out;
313 if (!db.Read(DBHeightKey(block.height), read_out)) {
314 return false;
315 }
316 if (read_out.first == block.hash) {
317 result = std::move(read_out.second);
318 return true;
319 }
320
321 // If value at the height index corresponds to an different block, the
322 // result will be stored in the hash index.
323 return db.Read(DBHashKey(block.hash), result);
324 }
325
326 std::optional<CCoinsStats> CoinStatsIndex::LookUpStats(const CBlockIndex& block_index) const
327 {
328 CCoinsStats stats{block_index.nHeight, block_index.GetBlockHash()};
329 stats.index_used = true;
330
331 DBVal entry;
332 if (!LookUpOne(*m_db, {block_index.GetBlockHash(), block_index.nHeight}, entry)) {
333 return std::nullopt;
334 }
335
336 stats.hashSerialized = entry.muhash;
337 stats.nTransactionOutputs = entry.transaction_output_count;
338 stats.nBogoSize = entry.bogo_size;
339 stats.total_amount = entry.total_amount;
340 stats.total_subsidy = entry.total_subsidy;
341 stats.total_unspendable_amount = entry.total_unspendable_amount;
342 stats.total_prevout_spent_amount = entry.total_prevout_spent_amount;
343 stats.total_new_outputs_ex_coinbase_amount = entry.total_new_outputs_ex_coinbase_amount;
344 stats.total_coinbase_amount = entry.total_coinbase_amount;
345 stats.total_unspendables_genesis_block = entry.total_unspendables_genesis_block;
346 stats.total_unspendables_bip30 = entry.total_unspendables_bip30;
347 stats.total_unspendables_scripts = entry.total_unspendables_scripts;
348 stats.total_unspendables_unclaimed_rewards = entry.total_unspendables_unclaimed_rewards;
349
350 return stats;
351 }
352
353 bool CoinStatsIndex::CustomInit(const std::optional<interfaces::BlockRef>& block)
354 {
355 if (!m_db->Read(DB_MUHASH, m_muhash)) {
356 // Check that the cause of the read failure is that the key does not
357 // exist. Any other errors indicate database corruption or a disk
358 // failure, and starting the index would cause further corruption.
359 if (m_db->Exists(DB_MUHASH)) {
360 LogError("%s: Cannot read current %s state; index may be corrupted\n",
361 __func__, GetName());
362 return false;
363 }
364 }
365
366 if (block) {
367 DBVal entry;
368 if (!LookUpOne(*m_db, *block, entry)) {
369 LogError("%s: Cannot read current %s state; index may be corrupted\n",
370 __func__, GetName());
371 return false;
372 }
373
374 uint256 out;
375 m_muhash.Finalize(out);
376 if (entry.muhash != out) {
377 LogError("%s: Cannot read current %s state; index may be corrupted\n",
378 __func__, GetName());
379 return false;
380 }
381
382 m_transaction_output_count = entry.transaction_output_count;
383 m_bogo_size = entry.bogo_size;
384 m_total_amount = entry.total_amount;
385 m_total_subsidy = entry.total_subsidy;
386 m_total_unspendable_amount = entry.total_unspendable_amount;
387 m_total_prevout_spent_amount = entry.total_prevout_spent_amount;
388 m_total_new_outputs_ex_coinbase_amount = entry.total_new_outputs_ex_coinbase_amount;
389 m_total_coinbase_amount = entry.total_coinbase_amount;
390 m_total_unspendables_genesis_block = entry.total_unspendables_genesis_block;
391 m_total_unspendables_bip30 = entry.total_unspendables_bip30;
392 m_total_unspendables_scripts = entry.total_unspendables_scripts;
393 m_total_unspendables_unclaimed_rewards = entry.total_unspendables_unclaimed_rewards;
394 }
395
396 return true;
397 }
398
399 bool CoinStatsIndex::CustomCommit(CDBBatch& batch)
400 {
401 // DB_MUHASH should always be committed in a batch together with DB_BEST_BLOCK
402 // to prevent an inconsistent state of the DB.
403 batch.Write(DB_MUHASH, m_muhash);
404 return true;
405 }
406
407 // Reverse a single block as part of a reorg
408 bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex)
409 {
410 CBlockUndo block_undo;
411 std::pair<uint256, DBVal> read_out;
412
413 const CAmount block_subsidy{GetBlockSubsidy(pindex->nHeight, Params().GetConsensus())};
414 m_total_subsidy -= block_subsidy;
415
416 // Ignore genesis block
417 if (pindex->nHeight > 0) {
418 if (!m_chainstate->m_blockman.ReadBlockUndo(block_undo, *pindex)) {
419 return false;
420 }
421
422 if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) {
423 return false;
424 }
425
426 uint256 expected_block_hash{pindex->pprev->GetBlockHash()};
427 if (read_out.first != expected_block_hash) {
428 LogPrintf("WARNING: previous block header belongs to unexpected block %s; expected %s\n",
429 read_out.first.ToString(), expected_block_hash.ToString());
430
431 if (!m_db->Read(DBHashKey(expected_block_hash), read_out)) {
432 LogError("%s: previous block header not found; expected %s\n",
433 __func__, expected_block_hash.ToString());
434 return false;
435 }
436 }
437 }
438
439 // Remove the new UTXOs that were created from the block
440 for (size_t i = 0; i < block.vtx.size(); ++i) {
441 const auto& tx{block.vtx.at(i)};
442
443 for (uint32_t j = 0; j < tx->vout.size(); ++j) {
444 const CTxOut& out{tx->vout[j]};
445 COutPoint outpoint{tx->GetHash(), j};
446 Coin coin{out, pindex->nHeight, tx->IsCoinBase()};
447
448 // Skip unspendable coins
449 if (coin.out.scriptPubKey.IsUnspendable()) {
450 m_total_unspendable_amount -= coin.out.nValue;
451 m_total_unspendables_scripts -= coin.out.nValue;
452 continue;
453 }
454
455 RemoveCoinHash(m_muhash, outpoint, coin);
456
457 if (tx->IsCoinBase()) {
458 m_total_coinbase_amount -= coin.out.nValue;
459 } else {
460 m_total_new_outputs_ex_coinbase_amount -= coin.out.nValue;
461 }
462
463 --m_transaction_output_count;
464 m_total_amount -= coin.out.nValue;
465 m_bogo_size -= GetBogoSize(coin.out.scriptPubKey);
466 }
467
468 // The coinbase tx has no undo data since no former output is spent
469 if (!tx->IsCoinBase()) {
470 const auto& tx_undo{block_undo.vtxundo.at(i - 1)};
471
472 for (size_t j = 0; j < tx_undo.vprevout.size(); ++j) {
473 Coin coin{tx_undo.vprevout[j]};
474 COutPoint outpoint{tx->vin[j].prevout.hash, tx->vin[j].prevout.n};
475
476 ApplyCoinHash(m_muhash, outpoint, coin);
477
478 m_total_prevout_spent_amount -= coin.out.nValue;
479
480 m_transaction_output_count++;
481 m_total_amount += coin.out.nValue;
482 m_bogo_size += GetBogoSize(coin.out.scriptPubKey);
483 }
484 }
485 }
486
487 const CAmount unclaimed_rewards{(m_total_new_outputs_ex_coinbase_amount + m_total_coinbase_amount + m_total_unspendable_amount) - (m_total_prevout_spent_amount + m_total_subsidy)};
488 m_total_unspendable_amount -= unclaimed_rewards;
489 m_total_unspendables_unclaimed_rewards -= unclaimed_rewards;
490
491 // Check that the rolled back internal values are consistent with the DB read out
492 uint256 out;
493 m_muhash.Finalize(out);
494 Assert(read_out.second.muhash == out);
495
496 Assert(m_transaction_output_count == read_out.second.transaction_output_count);
497 Assert(m_total_amount == read_out.second.total_amount);
498 Assert(m_bogo_size == read_out.second.bogo_size);
499 Assert(m_total_subsidy == read_out.second.total_subsidy);
500 Assert(m_total_unspendable_amount == read_out.second.total_unspendable_amount);
501 Assert(m_total_prevout_spent_amount == read_out.second.total_prevout_spent_amount);
502 Assert(m_total_new_outputs_ex_coinbase_amount == read_out.second.total_new_outputs_ex_coinbase_amount);
503 Assert(m_total_coinbase_amount == read_out.second.total_coinbase_amount);
504 Assert(m_total_unspendables_genesis_block == read_out.second.total_unspendables_genesis_block);
505 Assert(m_total_unspendables_bip30 == read_out.second.total_unspendables_bip30);
506 Assert(m_total_unspendables_scripts == read_out.second.total_unspendables_scripts);
507 Assert(m_total_unspendables_unclaimed_rewards == read_out.second.total_unspendables_unclaimed_rewards);
508
509 return true;
510 }
511