blockfilter_index_tests.cpp raw
1 // Copyright (c) 2017-present The Bitcoin Core developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4
5 #include <addresstype.h>
6 #include <blockfilter.h>
7 #include <chain.h>
8 #include <consensus/merkle.h>
9 #include <consensus/validation.h>
10 #include <index/base.h>
11 #include <index/blockfilterindex.h>
12 #include <interfaces/chain.h>
13 #include <interfaces/mining.h>
14 #include <key.h>
15 #include <node/blockstorage.h>
16 #include <pow.h>
17 #include <primitives/block.h>
18 #include <primitives/transaction.h>
19 #include <script/script.h>
20 #include <sync.h>
21 #include <test/util/blockfilter.h>
22 #include <test/util/common.h>
23 #include <test/util/setup_common.h>
24 #include <test/util/time.h>
25 #include <tinyformat.h>
26 #include <uint256.h>
27 #include <util/check.h>
28 #include <util/fs.h>
29 #include <validation.h>
30
31 #include <boost/test/unit_test.hpp>
32
33 #include <compare>
34 #include <cstddef>
35 #include <cstdint>
36 #include <functional>
37 #include <future>
38 #include <memory>
39 #include <span>
40 #include <string>
41 #include <thread>
42 #include <utility>
43 #include <vector>
44
45 using node::BlockManager;
46
47 BOOST_AUTO_TEST_SUITE(blockfilter_index_tests)
48
49 struct BuildChainTestingSetup : public TestChain100Setup {
50 CBlock CreateBlock(const CBlockIndex* prev, const std::vector<CMutableTransaction>& txns, const CScript& scriptPubKey);
51 bool BuildChain(const CBlockIndex* pindex, const CScript& coinbase_script_pub_key, size_t length, std::vector<std::shared_ptr<CBlock>>& chain);
52 };
53
54 static bool CheckFilterLookups(BlockFilterIndex& filter_index, const CBlockIndex* block_index,
55 uint256& last_header, const BlockManager& blockman)
56 {
57 BlockFilter expected_filter;
58 if (!ComputeFilter(filter_index.GetFilterType(), *block_index, expected_filter, blockman)) {
59 BOOST_ERROR("ComputeFilter failed on block " << block_index->nHeight);
60 return false;
61 }
62
63 BlockFilter filter;
64 uint256 filter_header;
65 std::vector<BlockFilter> filters;
66 std::vector<uint256> filter_hashes;
67
68 BOOST_CHECK(filter_index.LookupFilter(block_index, filter));
69 BOOST_CHECK(filter_index.LookupFilterHeader(block_index, filter_header));
70 BOOST_CHECK(filter_index.LookupFilterRange(block_index->nHeight, block_index, filters));
71 BOOST_CHECK(filter_index.LookupFilterHashRange(block_index->nHeight, block_index,
72 filter_hashes));
73
74 BOOST_CHECK_EQUAL(filters.size(), 1U);
75 BOOST_CHECK_EQUAL(filter_hashes.size(), 1U);
76
77 BOOST_CHECK_EQUAL(filter.GetHash(), expected_filter.GetHash());
78 BOOST_CHECK_EQUAL(filter_header, expected_filter.ComputeHeader(last_header));
79 BOOST_CHECK_EQUAL(filters[0].GetHash(), expected_filter.GetHash());
80 BOOST_CHECK_EQUAL(filter_hashes[0], expected_filter.GetHash());
81
82 filters.clear();
83 filter_hashes.clear();
84 last_header = filter_header;
85 return true;
86 }
87
88 CBlock BuildChainTestingSetup::CreateBlock(const CBlockIndex* prev,
89 const std::vector<CMutableTransaction>& txns,
90 const CScript& scriptPubKey)
91 {
92 auto mining{interfaces::MakeMining(m_node)};
93 auto block_template{mining->createNewBlock({
94 .coinbase_output_script = scriptPubKey,
95 }, /*cooldown=*/false)};
96 BOOST_REQUIRE(block_template);
97 CBlock block{block_template->getBlock()};
98 block.hashPrevBlock = prev->GetBlockHash();
99 block.nTime = prev->nTime + 1;
100
101 // Replace mempool-selected txns with just coinbase plus passed-in txns:
102 block.vtx.resize(1);
103 for (const CMutableTransaction& tx : txns) {
104 block.vtx.push_back(MakeTransactionRef(tx));
105 }
106 {
107 CMutableTransaction tx_coinbase{*block.vtx.at(0)};
108 tx_coinbase.nLockTime = static_cast<uint32_t>(prev->nHeight);
109 tx_coinbase.vin.at(0).scriptSig = CScript{} << prev->nHeight + 1;
110 block.vtx.at(0) = MakeTransactionRef(std::move(tx_coinbase));
111 block.hashMerkleRoot = BlockMerkleRoot(block);
112 }
113
114 while (!CheckProofOfWork(block.GetHash(), block.nBits, m_node.chainman->GetConsensus())) ++block.nNonce;
115
116 return block;
117 }
118
119 bool BuildChainTestingSetup::BuildChain(const CBlockIndex* pindex,
120 const CScript& coinbase_script_pub_key,
121 size_t length,
122 std::vector<std::shared_ptr<CBlock>>& chain)
123 {
124 std::vector<CMutableTransaction> no_txns;
125
126 chain.resize(length);
127 for (auto& block : chain) {
128 block = std::make_shared<CBlock>(CreateBlock(pindex, no_txns, coinbase_script_pub_key));
129
130 BlockValidationState state;
131 if (!Assert(m_node.chainman)->ProcessNewBlockHeaders({{*block}}, true, state, &pindex)) {
132 return false;
133 }
134 }
135
136 return true;
137 }
138
139 BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup)
140 {
141 BlockFilterIndex filter_index(interfaces::MakeChain(m_node), BlockFilterType::BASIC, 1_MiB, true);
142 BOOST_REQUIRE(filter_index.Init());
143
144 uint256 last_header;
145
146 // Filter should not be found in the index before it is started.
147 {
148 LOCK(cs_main);
149
150 BlockFilter filter;
151 uint256 filter_header;
152 std::vector<BlockFilter> filters;
153 std::vector<uint256> filter_hashes;
154
155 for (const CBlockIndex* block_index = m_node.chainman->ActiveChain().Genesis();
156 block_index != nullptr;
157 block_index = m_node.chainman->ActiveChain().Next(*block_index)) {
158 BOOST_CHECK(!filter_index.LookupFilter(block_index, filter));
159 BOOST_CHECK(!filter_index.LookupFilterHeader(block_index, filter_header));
160 BOOST_CHECK(!filter_index.LookupFilterRange(block_index->nHeight, block_index, filters));
161 BOOST_CHECK(!filter_index.LookupFilterHashRange(block_index->nHeight, block_index,
162 filter_hashes));
163 }
164 }
165
166 // BlockUntilSyncedToCurrentChain should return false before index is started.
167 BOOST_CHECK(!filter_index.BlockUntilSyncedToCurrentChain());
168
169 filter_index.Sync();
170
171 // Check that filter index has all blocks that were in the chain before it started.
172 {
173 LOCK(cs_main);
174 const CBlockIndex* block_index;
175 for (block_index = m_node.chainman->ActiveChain().Genesis();
176 block_index != nullptr;
177 block_index = m_node.chainman->ActiveChain().Next(*block_index)) {
178 CheckFilterLookups(filter_index, block_index, last_header, m_node.chainman->m_blockman);
179 }
180 }
181
182 // Create two forks.
183 const CBlockIndex* tip;
184 {
185 LOCK(cs_main);
186 tip = m_node.chainman->ActiveChain().Tip();
187 }
188 CKey coinbase_key_A = GenerateRandomKey();
189 CKey coinbase_key_B = GenerateRandomKey();
190 CScript coinbase_script_pub_key_A = GetScriptForDestination(PKHash(coinbase_key_A.GetPubKey()));
191 CScript coinbase_script_pub_key_B = GetScriptForDestination(PKHash(coinbase_key_B.GetPubKey()));
192 std::vector<std::shared_ptr<CBlock>> chainA, chainB;
193 BOOST_REQUIRE(BuildChain(tip, coinbase_script_pub_key_A, 10, chainA));
194 BOOST_REQUIRE(BuildChain(tip, coinbase_script_pub_key_B, 10, chainB));
195
196 // Check that new blocks on chain A get indexed.
197 uint256 chainA_last_header = last_header;
198 for (size_t i = 0; i < 2; i++) {
199 const auto& block = chainA[i];
200 BOOST_REQUIRE(Assert(m_node.chainman)->ProcessNewBlock(block, true, true, nullptr));
201 }
202 for (size_t i = 0; i < 2; i++) {
203 const auto& block = chainA[i];
204 const CBlockIndex* block_index;
205 {
206 LOCK(cs_main);
207 block_index = m_node.chainman->m_blockman.LookupBlockIndex(block->GetHash());
208 }
209
210 BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain());
211 CheckFilterLookups(filter_index, block_index, chainA_last_header, m_node.chainman->m_blockman);
212 }
213
214 // Reorg to chain B.
215 uint256 chainB_last_header = last_header;
216 for (size_t i = 0; i < 3; i++) {
217 const auto& block = chainB[i];
218 BOOST_REQUIRE(Assert(m_node.chainman)->ProcessNewBlock(block, true, true, nullptr));
219 }
220 for (size_t i = 0; i < 3; i++) {
221 const auto& block = chainB[i];
222 const CBlockIndex* block_index;
223 {
224 LOCK(cs_main);
225 block_index = m_node.chainman->m_blockman.LookupBlockIndex(block->GetHash());
226 }
227
228 BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain());
229 CheckFilterLookups(filter_index, block_index, chainB_last_header, m_node.chainman->m_blockman);
230 }
231
232 // Check that filters for stale blocks on A can be retrieved.
233 chainA_last_header = last_header;
234 for (size_t i = 0; i < 2; i++) {
235 const auto& block = chainA[i];
236 const CBlockIndex* block_index;
237 {
238 LOCK(cs_main);
239 block_index = m_node.chainman->m_blockman.LookupBlockIndex(block->GetHash());
240 }
241
242 BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain());
243 CheckFilterLookups(filter_index, block_index, chainA_last_header, m_node.chainman->m_blockman);
244 }
245
246 // Reorg back to chain A.
247 for (size_t i = 2; i < 4; i++) {
248 const auto& block = chainA[i];
249 BOOST_REQUIRE(Assert(m_node.chainman)->ProcessNewBlock(block, true, true, nullptr));
250 }
251
252 // Check that chain A and B blocks can be retrieved.
253 chainA_last_header = last_header;
254 chainB_last_header = last_header;
255 for (size_t i = 0; i < 3; i++) {
256 const CBlockIndex* block_index;
257
258 {
259 LOCK(cs_main);
260 block_index = m_node.chainman->m_blockman.LookupBlockIndex(chainA[i]->GetHash());
261 }
262 BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain());
263 CheckFilterLookups(filter_index, block_index, chainA_last_header, m_node.chainman->m_blockman);
264
265 {
266 LOCK(cs_main);
267 block_index = m_node.chainman->m_blockman.LookupBlockIndex(chainB[i]->GetHash());
268 }
269 BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain());
270 CheckFilterLookups(filter_index, block_index, chainB_last_header, m_node.chainman->m_blockman);
271 }
272
273 // Test lookups for a range of filters/hashes.
274 std::vector<BlockFilter> filters;
275 std::vector<uint256> filter_hashes;
276
277 {
278 LOCK(cs_main);
279 tip = m_node.chainman->ActiveChain().Tip();
280 }
281 BOOST_CHECK(filter_index.LookupFilterRange(0, tip, filters));
282 BOOST_CHECK(filter_index.LookupFilterHashRange(0, tip, filter_hashes));
283
284 assert(tip->nHeight >= 0);
285 BOOST_CHECK_EQUAL(filters.size(), tip->nHeight + 1U);
286 BOOST_CHECK_EQUAL(filter_hashes.size(), tip->nHeight + 1U);
287
288 filters.clear();
289 filter_hashes.clear();
290
291 filter_index.Interrupt();
292 filter_index.Stop();
293 }
294
295 BOOST_FIXTURE_TEST_CASE(blockfilter_index_init_destroy, BasicTestingSetup)
296 {
297 BlockFilterIndex* filter_index;
298
299 filter_index = GetBlockFilterIndex(BlockFilterType::BASIC);
300 BOOST_CHECK(filter_index == nullptr);
301
302 BOOST_CHECK(InitBlockFilterIndex([&]{ return interfaces::MakeChain(m_node); }, BlockFilterType::BASIC, 1_MiB, true, false));
303
304 filter_index = GetBlockFilterIndex(BlockFilterType::BASIC);
305 BOOST_CHECK(filter_index != nullptr);
306 BOOST_CHECK(filter_index->GetFilterType() == BlockFilterType::BASIC);
307
308 // Initialize returns false if index already exists.
309 BOOST_CHECK(!InitBlockFilterIndex([&]{ return interfaces::MakeChain(m_node); }, BlockFilterType::BASIC, 1_MiB, true, false));
310
311 int iter_count = 0;
312 ForEachBlockFilterIndex([&iter_count](BlockFilterIndex& _index) { iter_count++; });
313 BOOST_CHECK_EQUAL(iter_count, 1);
314
315 BOOST_CHECK(DestroyBlockFilterIndex(BlockFilterType::BASIC));
316
317 // Destroy returns false because index was already destroyed.
318 BOOST_CHECK(!DestroyBlockFilterIndex(BlockFilterType::BASIC));
319
320 filter_index = GetBlockFilterIndex(BlockFilterType::BASIC);
321 BOOST_CHECK(filter_index == nullptr);
322
323 // Reinitialize index.
324 BOOST_CHECK(InitBlockFilterIndex([&]{ return interfaces::MakeChain(m_node); }, BlockFilterType::BASIC, 1_MiB, true, false));
325
326 DestroyAllBlockFilterIndexes();
327
328 filter_index = GetBlockFilterIndex(BlockFilterType::BASIC);
329 BOOST_CHECK(filter_index == nullptr);
330 }
331
332 class IndexReorgCrash : public BaseIndex
333 {
334 private:
335 FakeNodeClock& m_clock;
336 std::unique_ptr<BaseIndex::DB> m_db;
337 std::shared_future<void> m_blocker;
338 int m_blocking_height;
339
340 public:
341 explicit IndexReorgCrash(std::unique_ptr<interfaces::Chain> chain, std::shared_future<void> blocker, int blocking_height, FakeNodeClock& clock)
342 : BaseIndex(std::move(chain), "test index", "testidx"), m_clock(clock), m_blocker(blocker), m_blocking_height(blocking_height)
343 {
344 const fs::path path = gArgs.GetDataDirNet() / "index";
345 fs::create_directories(path);
346 m_db = std::make_unique<BaseIndex::DB>(path / "db", /*n_cache_size=*/0, /*f_memory=*/true, /*f_wipe=*/false);
347 }
348
349 bool AllowPrune() const override { return false; }
350 BaseIndex::DB& GetDB() const override { return *m_db; }
351
352 bool CustomAppend(const interfaces::BlockInfo& block) override
353 {
354 // Simulate a delay so new blocks can get connected during the initial sync
355 if (block.height == m_blocking_height) m_blocker.wait();
356
357 // Move mock time forward so the best index gets updated only when we are not at the blocking height
358 if (block.height == m_blocking_height - 1 || block.height > m_blocking_height) {
359 m_clock += 31s;
360 }
361
362 return true;
363 }
364 };
365
366 BOOST_FIXTURE_TEST_CASE(index_reorg_crash, BuildChainTestingSetup)
367 {
368 std::promise<void> promise;
369 std::shared_future<void> blocker(promise.get_future());
370 int blocking_height = WITH_LOCK(cs_main, return m_node.chainman->ActiveChain().Tip()->nHeight);
371
372 IndexReorgCrash index{interfaces::MakeChain(m_node), blocker, blocking_height, m_clock};
373 BOOST_REQUIRE(index.Init());
374 BOOST_REQUIRE(index.StartBackgroundSync());
375
376 auto func_wait_until = [&](int height, std::chrono::milliseconds timeout) {
377 auto deadline = std::chrono::steady_clock::now() + timeout;
378 while (index.GetSummary().best_block_height < height) {
379 if (std::chrono::steady_clock::now() > deadline) {
380 BOOST_FAIL(strprintf("Timeout waiting for index height %d (current: %d)", height, index.GetSummary().best_block_height));
381 return;
382 }
383 std::this_thread::sleep_for(100ms);
384 }
385 };
386
387 // Wait until the index is one block before the fork point
388 func_wait_until(blocking_height - 1, /*timeout=*/5s);
389
390 // Create a fork to trigger the reorg
391 std::vector<std::shared_ptr<CBlock>> fork;
392 const CBlockIndex* prev_tip = WITH_LOCK(cs_main, return m_node.chainman->ActiveChain().Tip()->pprev);
393 BOOST_REQUIRE(BuildChain(prev_tip, GetScriptForDestination(PKHash(GenerateRandomKey().GetPubKey())), 3, fork));
394
395 for (const auto& block : fork) {
396 BOOST_REQUIRE(m_node.chainman->ProcessNewBlock(block, /*force_processing=*/true, /*min_pow_checked=*/true, nullptr));
397 }
398
399 // Unblock the index thread so it can process the reorg
400 promise.set_value();
401 // Wait for the index to reach the new tip
402 func_wait_until(blocking_height + 2, 5s);
403 index.Stop();
404 }
405
406 BOOST_AUTO_TEST_SUITE_END()
407