1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2022 The Limenka developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 6 #ifndef LIMENKA_COINS_H
7 #define LIMENKA_COINS_H
8 9 #include <compressor.h>
10 #include <core_memusage.h>
11 #include <memusage.h>
12 #include <primitives/transaction.h>
13 #include <serialize.h>
14 #include <support/allocators/pool.h>
15 #include <uint256.h>
16 #include <util/check.h>
17 #include <util/hasher.h>
18 19 #include <assert.h>
20 #include <stdint.h>
21 22 #include <functional>
23 #include <unordered_map>
24 25 /**
26 * A UTXO entry.
27 *
28 * Serialized format:
29 * - VARINT((coinbase ? 1 : 0) | (height << 1))
30 * - the non-spent CTxOut (via TxOutCompression)
31 */
32 class Coin
33 {
34 public:
35 //! unspent transaction output
36 CTxOut out;
37 38 //! whether containing transaction was a coinbase
39 unsigned int fCoinBase : 1;
40 41 //! at which height this containing transaction was included in the active block chain
42 uint32_t nHeight : 31;
43 44 //! construct a Coin from a CTxOut and height/coinbase information.
45 Coin(CTxOut&& outIn, int nHeightIn, bool fCoinBaseIn) : out(std::move(outIn)), fCoinBase(fCoinBaseIn), nHeight(nHeightIn) {}
46 Coin(const CTxOut& outIn, int nHeightIn, bool fCoinBaseIn) : out(outIn), fCoinBase(fCoinBaseIn), nHeight(nHeightIn) {}
47 48 void Clear() {
49 out.SetNull();
50 fCoinBase = false;
51 nHeight = 0;
52 }
53 54 //! empty constructor
55 Coin() : fCoinBase(false), nHeight(0) { }
56 57 bool IsCoinBase() const {
58 return fCoinBase;
59 }
60 61 template<typename Stream>
62 void Serialize(Stream &s) const {
63 assert(!IsSpent());
64 uint32_t code = nHeight * uint32_t{2} + fCoinBase;
65 ::Serialize(s, VARINT(code));
66 ::Serialize(s, Using<TxOutCompression>(out));
67 }
68 69 template<typename Stream>
70 void Unserialize(Stream &s) {
71 uint32_t code = 0;
72 ::Unserialize(s, VARINT(code));
73 nHeight = code >> 1;
74 fCoinBase = code & 1;
75 ::Unserialize(s, Using<TxOutCompression>(out));
76 }
77 78 /** Either this coin never existed (see e.g. coinEmpty in coins.cpp), or it
79 * did exist and has been spent.
80 */
81 bool IsSpent() const {
82 return out.IsNull();
83 }
84 85 size_t DynamicMemoryUsage() const {
86 return memusage::DynamicUsage(out.scriptPubKey);
87 }
88 };
89 90 struct CCoinsCacheEntry;
91 using CoinsCachePair = std::pair<const COutPoint, CCoinsCacheEntry>;
92 93 /**
94 * A Coin in one level of the coins database caching hierarchy.
95 *
96 * A coin can either be:
97 * - unspent or spent (in which case the Coin object will be nulled out - see Coin.Clear())
98 * - DIRTY or not DIRTY
99 * - FRESH or not FRESH
100 *
101 * Out of these 2^3 = 8 states, only some combinations are valid:
102 * - unspent, FRESH, DIRTY (e.g. a new coin created in the cache)
103 * - unspent, not FRESH, DIRTY (e.g. a coin changed in the cache during a reorg)
104 * - unspent, not FRESH, not DIRTY (e.g. an unspent coin fetched from the parent cache)
105 * - spent, FRESH, not DIRTY (e.g. a spent coin fetched from the parent cache)
106 * - spent, not FRESH, DIRTY (e.g. a coin is spent and spentness needs to be flushed to the parent)
107 */
108 struct CCoinsCacheEntry
109 {
110 private:
111 /**
112 * These are used to create a doubly linked list of flagged entries.
113 * They are set in SetDirty, SetFresh, and unset in SetClean.
114 * A flagged entry is any entry that is either DIRTY, FRESH, or both.
115 *
116 * DIRTY entries are tracked so that only modified entries can be passed to
117 * the parent cache for batch writing. This is a performance optimization
118 * compared to giving all entries in the cache to the parent and having the
119 * parent scan for only modified entries.
120 *
121 * FRESH-but-not-DIRTY coins can not occur in practice, since that would
122 * mean a spent coin exists in the parent CCoinsView and not in the child
123 * CCoinsViewCache. Nevertheless, if a spent coin is retrieved from the
124 * parent cache, the FRESH-but-not-DIRTY coin will be tracked by the linked
125 * list and deleted when Sync or Flush is called on the CCoinsViewCache.
126 */
127 CoinsCachePair* m_prev{nullptr};
128 CoinsCachePair* m_next{nullptr};
129 uint8_t m_flags{0};
130 131 //! Adding a flag requires a reference to the sentinel of the flagged pair linked list.
132 static void AddFlags(uint8_t flags, CoinsCachePair& pair, CoinsCachePair& sentinel) noexcept
133 {
134 Assume(flags & (DIRTY | FRESH));
135 if (!pair.second.m_flags) {
136 Assume(!pair.second.m_prev && !pair.second.m_next);
137 pair.second.m_prev = sentinel.second.m_prev;
138 pair.second.m_next = &sentinel;
139 sentinel.second.m_prev = &pair;
140 pair.second.m_prev->second.m_next = &pair;
141 }
142 Assume(pair.second.m_prev && pair.second.m_next);
143 pair.second.m_flags |= flags;
144 }
145 146 public:
147 Coin coin; // The actual cached data.
148 149 enum Flags {
150 /**
151 * DIRTY means the CCoinsCacheEntry is potentially different from the
152 * version in the parent cache. Failure to mark a coin as DIRTY when
153 * it is potentially different from the parent cache will cause a
154 * consensus failure, since the coin's state won't get written to the
155 * parent when the cache is flushed.
156 */
157 DIRTY = (1 << 0),
158 /**
159 * FRESH means the parent cache does not have this coin or that it is a
160 * spent coin in the parent cache. If a FRESH coin in the cache is
161 * later spent, it can be deleted entirely and doesn't ever need to be
162 * flushed to the parent. This is a performance optimization. Marking a
163 * coin as FRESH when it exists unspent in the parent cache will cause a
164 * consensus failure, since it might not be deleted from the parent
165 * when this cache is flushed.
166 */
167 FRESH = (1 << 1),
168 };
169 170 CCoinsCacheEntry() noexcept = default;
171 explicit CCoinsCacheEntry(Coin&& coin_) noexcept : coin(std::move(coin_)) {}
172 ~CCoinsCacheEntry()
173 {
174 SetClean();
175 }
176 177 static void SetDirty(CoinsCachePair& pair, CoinsCachePair& sentinel) noexcept { AddFlags(DIRTY, pair, sentinel); }
178 static void SetFresh(CoinsCachePair& pair, CoinsCachePair& sentinel) noexcept { AddFlags(FRESH, pair, sentinel); }
179 180 void SetClean() noexcept
181 {
182 if (!m_flags) return;
183 m_next->second.m_prev = m_prev;
184 m_prev->second.m_next = m_next;
185 m_flags = 0;
186 m_prev = m_next = nullptr;
187 }
188 bool IsDirty() const noexcept { return m_flags & DIRTY; }
189 bool IsFresh() const noexcept { return m_flags & FRESH; }
190 191 //! Only call Next when this entry is DIRTY, FRESH, or both
192 CoinsCachePair* Next() const noexcept
193 {
194 Assume(m_flags);
195 return m_next;
196 }
197 198 //! Only call Prev when this entry is DIRTY, FRESH, or both
199 CoinsCachePair* Prev() const noexcept
200 {
201 Assume(m_flags);
202 return m_prev;
203 }
204 205 //! Only use this for initializing the linked list sentinel
206 void SelfRef(CoinsCachePair& pair) noexcept
207 {
208 Assume(&pair.second == this);
209 m_prev = &pair;
210 m_next = &pair;
211 // Set sentinel to DIRTY so we can call Next on it
212 m_flags = DIRTY;
213 }
214 };
215 216 /**
217 * PoolAllocator's MAX_BLOCK_SIZE_BYTES parameter here uses sizeof the data, and adds the size
218 * of 4 pointers. We do not know the exact node size used in the std::unordered_node implementation
219 * because it is implementation defined. Most implementations have an overhead of 1 or 2 pointers,
220 * so nodes can be connected in a linked list, and in some cases the hash value is stored as well.
221 * Using an additional sizeof(void*)*4 for MAX_BLOCK_SIZE_BYTES should thus be sufficient so that
222 * all implementations can allocate the nodes from the PoolAllocator.
223 */
224 using CCoinsMap = std::unordered_map<COutPoint,
225 CCoinsCacheEntry,
226 SaltedOutpointHasher,
227 std::equal_to<COutPoint>,
228 PoolAllocator<CoinsCachePair,
229 sizeof(CoinsCachePair) + sizeof(void*) * 4>>;
230 231 using CCoinsMapMemoryResource = CCoinsMap::allocator_type::ResourceType;
232 233 /** Cursor for iterating over CoinsView state */
234 class CCoinsViewCursor
235 {
236 public:
237 CCoinsViewCursor(const uint256 &hashBlockIn): hashBlock(hashBlockIn) {}
238 virtual ~CCoinsViewCursor() = default;
239 240 virtual bool GetKey(COutPoint &key) const = 0;
241 virtual bool GetValue(Coin &coin) const = 0;
242 243 virtual bool Valid() const = 0;
244 virtual void Next() = 0;
245 246 //! Get best block at the time this cursor was created
247 const uint256 &GetBestBlock() const { return hashBlock; }
248 private:
249 uint256 hashBlock;
250 };
251 252 /**
253 * Cursor for iterating over the linked list of flagged entries in CCoinsViewCache.
254 *
255 * This is a helper struct to encapsulate the diverging logic between a non-erasing
256 * CCoinsViewCache::Sync and an erasing CCoinsViewCache::Flush. This allows the receiver
257 * of CCoinsView::BatchWrite to iterate through the flagged entries without knowing
258 * the caller's intent.
259 *
260 * However, the receiver can still call CoinsViewCacheCursor::WillErase to see if the
261 * caller will erase the entry after BatchWrite returns. If so, the receiver can
262 * perform optimizations such as moving the coin out of the CCoinsCachEntry instead
263 * of copying it.
264 */
265 struct CoinsViewCacheCursor
266 {
267 //! If will_erase is not set, iterating through the cursor will erase spent coins from the map,
268 //! and other coins will be unflagged (removing them from the linked list).
269 //! If will_erase is set, the underlying map and linked list will not be modified,
270 //! as the caller is expected to wipe the entire map anyway.
271 //! This is an optimization compared to erasing all entries as the cursor iterates them when will_erase is set.
272 //! Calling CCoinsMap::clear() afterwards is faster because a CoinsCachePair cannot be coerced back into a
273 //! CCoinsMap::iterator to be erased, and must therefore be looked up again by key in the CCoinsMap before being erased.
274 CoinsViewCacheCursor(size_t& usage LIFETIMEBOUND,
275 CoinsCachePair& sentinel LIFETIMEBOUND,
276 CCoinsMap& map LIFETIMEBOUND,
277 bool will_erase) noexcept
278 : m_usage(usage), m_sentinel(sentinel), m_map(map), m_will_erase(will_erase) {}
279 280 inline CoinsCachePair* Begin() const noexcept { return m_sentinel.second.Next(); }
281 inline CoinsCachePair* End() const noexcept { return &m_sentinel; }
282 283 //! Return the next entry after current, possibly erasing current
284 inline CoinsCachePair* NextAndMaybeErase(CoinsCachePair& current) noexcept
285 {
286 const auto next_entry{current.second.Next()};
287 // If we are not going to erase the cache, we must still erase spent entries.
288 // Otherwise, clear the state of the entry.
289 if (!m_will_erase) {
290 if (current.second.coin.IsSpent()) {
291 m_usage -= current.second.coin.DynamicMemoryUsage();
292 m_map.erase(current.first);
293 } else {
294 current.second.SetClean();
295 }
296 }
297 return next_entry;
298 }
299 300 inline bool WillErase(CoinsCachePair& current) const noexcept { return m_will_erase || current.second.coin.IsSpent(); }
301 private:
302 size_t& m_usage;
303 CoinsCachePair& m_sentinel;
304 CCoinsMap& m_map;
305 bool m_will_erase;
306 };
307 308 /** Abstract view on the open txout dataset. */
309 class CCoinsView
310 {
311 public:
312 //! Retrieve the Coin (unspent transaction output) for a given outpoint.
313 virtual std::optional<Coin> GetCoin(const COutPoint& outpoint) const;
314 315 //! Just check whether a given outpoint is unspent.
316 virtual bool HaveCoin(const COutPoint &outpoint) const;
317 318 //! Retrieve the block hash whose state this CCoinsView currently represents
319 virtual uint256 GetBestBlock() const;
320 321 //! Retrieve the range of blocks that may have been only partially written.
322 //! If the database is in a consistent state, the result is the empty vector.
323 //! Otherwise, a two-element vector is returned consisting of the new and
324 //! the old block hash, in that order.
325 virtual std::vector<uint256> GetHeadBlocks() const;
326 327 //! Do a bulk modification (multiple Coin changes + BestBlock change).
328 //! The passed cursor is used to iterate through the coins.
329 virtual bool BatchWrite(CoinsViewCacheCursor& cursor, const uint256& hashBlock);
330 331 //! Get a cursor to iterate over the whole state
332 virtual std::unique_ptr<CCoinsViewCursor> Cursor() const;
333 334 //! As we use CCoinsViews polymorphically, have a virtual destructor
335 virtual ~CCoinsView() = default;
336 337 //! Estimate database size (0 if not implemented)
338 virtual size_t EstimateSize() const { return 0; }
339 };
340 341 342 /** CCoinsView backed by another CCoinsView */
343 class CCoinsViewBacked : public CCoinsView
344 {
345 protected:
346 CCoinsView *base;
347 348 public:
349 CCoinsViewBacked(CCoinsView *viewIn);
350 std::optional<Coin> GetCoin(const COutPoint& outpoint) const override;
351 bool HaveCoin(const COutPoint &outpoint) const override;
352 uint256 GetBestBlock() const override;
353 std::vector<uint256> GetHeadBlocks() const override;
354 void SetBackend(CCoinsView &viewIn);
355 bool BatchWrite(CoinsViewCacheCursor& cursor, const uint256 &hashBlock) override;
356 std::unique_ptr<CCoinsViewCursor> Cursor() const override;
357 size_t EstimateSize() const override;
358 };
359 360 361 /** CCoinsView that adds a memory cache for transactions to another CCoinsView */
362 class CCoinsViewCache : public CCoinsViewBacked
363 {
364 private:
365 const bool m_deterministic;
366 367 protected:
368 /**
369 * Make mutable so that we can "fill the cache" even from Get-methods
370 * declared as "const".
371 */
372 mutable uint256 hashBlock;
373 mutable CCoinsMapMemoryResource m_cache_coins_memory_resource{};
374 /* The starting sentinel of the flagged entry circular doubly linked list. */
375 mutable CoinsCachePair m_sentinel;
376 mutable CCoinsMap cacheCoins;
377 378 /* Cached dynamic memory usage for the inner Coin objects. */
379 mutable size_t cachedCoinsUsage{0};
380 381 public:
382 CCoinsViewCache(CCoinsView *baseIn, bool deterministic = false);
383 384 /**
385 * By deleting the copy constructor, we prevent accidentally using it when one intends to create a cache on top of a base cache.
386 */
387 CCoinsViewCache(const CCoinsViewCache &) = delete;
388 389 // Standard CCoinsView methods
390 std::optional<Coin> GetCoin(const COutPoint& outpoint) const override;
391 bool HaveCoin(const COutPoint &outpoint) const override;
392 uint256 GetBestBlock() const override;
393 void SetBestBlock(const uint256 &hashBlock);
394 bool BatchWrite(CoinsViewCacheCursor& cursor, const uint256 &hashBlock) override;
395 std::unique_ptr<CCoinsViewCursor> Cursor() const override {
396 throw std::logic_error("CCoinsViewCache cursor iteration not supported.");
397 }
398 399 /**
400 * Check if we have the given utxo already loaded in this cache.
401 * The semantics are the same as HaveCoin(), but no calls to
402 * the backing CCoinsView are made.
403 */
404 bool HaveCoinInCache(const COutPoint &outpoint) const;
405 406 /**
407 * Return a reference to Coin in the cache, or coinEmpty if not found. This is
408 * more efficient than GetCoin.
409 *
410 * Generally, do not hold the reference returned for more than a short scope.
411 * While the current implementation allows for modifications to the contents
412 * of the cache while holding the reference, this behavior should not be relied
413 * on! To be safe, best to not hold the returned reference through any other
414 * calls to this cache.
415 */
416 const Coin& AccessCoin(const COutPoint &output) const;
417 418 /**
419 * Add a coin. Set possible_overwrite to true if an unspent version may
420 * already exist in the cache.
421 */
422 void AddCoin(const COutPoint& outpoint, Coin&& coin, bool possible_overwrite);
423 424 /**
425 * Emplace a coin into cacheCoins without performing any checks, marking
426 * the emplaced coin as dirty.
427 *
428 * NOT FOR GENERAL USE. Used only when loading coins from a UTXO snapshot.
429 * @sa ChainstateManager::PopulateAndValidateSnapshot()
430 */
431 void EmplaceCoinInternalDANGER(COutPoint&& outpoint, Coin&& coin);
432 433 /**
434 * Spend a coin. Pass moveto in order to get the deleted data.
435 * If no unspent output exists for the passed outpoint, this call
436 * has no effect.
437 */
438 bool SpendCoin(const COutPoint &outpoint, Coin* moveto = nullptr);
439 440 /**
441 * Push the modifications applied to this cache to its base and wipe local state.
442 * Failure to call this method or Sync() before destruction will cause the changes
443 * to be forgotten.
444 * If false is returned, the state of this cache (and its backing view) will be undefined.
445 */
446 bool Flush();
447 448 /**
449 * Push the modifications applied to this cache to its base while retaining
450 * the contents of this cache (except for spent coins, which we erase).
451 * Failure to call this method or Flush() before destruction will cause the changes
452 * to be forgotten.
453 * If false is returned, the state of this cache (and its backing view) will be undefined.
454 */
455 bool Sync();
456 457 /**
458 * Removes the UTXO with the given outpoint from the cache, if it is
459 * not modified.
460 */
461 void Uncache(const COutPoint &outpoint);
462 463 //! Calculate the size of the cache (in number of transaction outputs)
464 unsigned int GetCacheSize() const;
465 466 //! Calculate the size of the cache (in bytes)
467 size_t DynamicMemoryUsage() const;
468 469 //! Check whether all prevouts of the transaction are present in the UTXO set represented by this view
470 bool HaveInputs(const CTransaction& tx) const;
471 472 //! Force a reallocation of the cache map. This is required when downsizing
473 //! the cache because the map's allocator may be hanging onto a lot of
474 //! memory despite having called .clear().
475 //!
476 //! See: https://stackoverflow.com/questions/42114044/how-to-release-unordered-map-memory
477 void ReallocateCache();
478 479 //! Run an internal sanity check on the cache data structure. */
480 void SanityCheck() const;
481 482 private:
483 /**
484 * @note this is marked const, but may actually append to `cacheCoins`, increasing
485 * memory usage.
486 */
487 CCoinsMap::iterator FetchCoin(const COutPoint &outpoint) const;
488 };
489 490 //! Utility function to add all of a transaction's outputs to a cache.
491 //! When check is false, this assumes that overwrites are only possible for coinbase transactions.
492 //! When check is true, the underlying view may be queried to determine whether an addition is
493 //! an overwrite.
494 // TODO: pass in a boolean to limit these possible overwrites to known
495 // (pre-BIP34) cases.
496 void AddCoins(CCoinsViewCache& cache, const CTransaction& tx, int nHeight, bool check = false);
497 498 //! Utility function to find any unspent output with a given txid.
499 //! This function can be quite expensive because in the event of a transaction
500 //! which is not found in the cache, it can cause up to MAX_OUTPUTS_PER_BLOCK
501 //! lookups to database, so it should be used with care.
502 const Coin& AccessByTxid(const CCoinsViewCache& cache, const Txid& txid);
503 504 /**
505 * This is a minimally invasive approach to shutdown on LevelDB read errors from the
506 * chainstate, while keeping user interface out of the common library, which is shared
507 * between limenkad, and limenka-qt and non-server tools.
508 *
509 * Writes do not need similar protection, as failure to write is handled by the caller.
510 */
511 class CCoinsViewErrorCatcher final : public CCoinsViewBacked
512 {
513 public:
514 explicit CCoinsViewErrorCatcher(CCoinsView* view) : CCoinsViewBacked(view) {}
515 516 void AddReadErrCallback(std::function<void()> f) {
517 m_err_callbacks.emplace_back(std::move(f));
518 }
519 520 std::optional<Coin> GetCoin(const COutPoint& outpoint) const override;
521 bool HaveCoin(const COutPoint &outpoint) const override;
522 523 private:
524 /** A list of callbacks to execute upon leveldb read error. */
525 std::vector<std::function<void()>> m_err_callbacks;
526 527 };
528 529 #endif // LIMENKA_COINS_H
530