mempool_persist.cpp raw
1 // Copyright (c) 2022 The Limenka developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4
5 #include <node/mempool_persist.h>
6
7 #include <clientversion.h>
8 #include <consensus/amount.h>
9 #include <logging.h>
10 #include <primitives/transaction.h>
11 #include <random.h>
12 #include <serialize.h>
13 #include <streams.h>
14 #include <sync.h>
15 #include <txmempool.h>
16 #include <uint256.h>
17 #include <util/fs.h>
18 #include <util/fs_helpers.h>
19 #include <util/obfuscation.h>
20 #include <util/serfloat.h>
21 #include <util/signalinterrupt.h>
22 #include <util/syserror.h>
23 #include <util/time.h>
24 #include <validation.h>
25
26 #include <cstdint>
27 #include <cstdio>
28 #include <exception>
29 #include <functional>
30 #include <map>
31 #include <memory>
32 #include <set>
33 #include <stdexcept>
34 #include <utility>
35 #include <vector>
36
37 using fsbridge::FopenFn;
38
39 namespace node {
40
41 static const uint64_t MEMPOOL_DUMP_VERSION_NO_XOR_KEY{1};
42 static const uint64_t MEMPOOL_DUMP_VERSION{2};
43 static constexpr uint64_t MEMPOOL_KNOTS_DUMP_VERSION = 0;
44
45 bool LoadMempoolKnots(CTxMemPool& pool, const fs::path& knots_filepath, FopenFn mockable_fopen_function)
46 {
47 AutoFile file{mockable_fopen_function(knots_filepath, "rb")};
48 if (file.IsNull()) {
49 // Typically missing if there's nothing to save
50 return false;
51 }
52
53 try {
54 uint64_t version;
55 file >> version;
56 if (version != MEMPOOL_KNOTS_DUMP_VERSION) {
57 return false;
58 }
59
60 const unsigned int priority_deltas_count = ReadCompactSize(file);
61 uint256 txid;
62 uint64_t encoded_priority;
63 for (unsigned int i = 0; i < priority_deltas_count; ++i) {
64 Unserialize(file, txid);
65 Unserialize(file, encoded_priority);
66 const double priority = DecodeDouble(encoded_priority);
67 pool.PrioritiseTransaction(txid, priority, 0);
68 }
69 } catch (const std::exception& e) {
70 LogInfo("Failed to deserialize mempool-knots data on file: %s. Continuing anyway.\n", e.what());
71 return false;
72 }
73
74 return true;
75 }
76
77 bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active_chainstate, ImportMempoolOptions&& opts)
78 {
79 if (load_path.empty()) return false;
80
81 AutoFile file{opts.mockable_fopen_function(load_path, "rb")};
82 if (file.IsNull()) {
83 LogInfo("Failed to open mempool file. Continuing anyway.\n");
84 return false;
85 }
86
87 int64_t count = 0;
88 int64_t expired = 0;
89 int64_t failed = 0;
90 int64_t already_there = 0;
91 int64_t unbroadcast = 0;
92 const auto now{NodeClock::now()};
93
94 try {
95 uint64_t version;
96 file >> version;
97 Obfuscation xor_key{};
98 if (version == MEMPOOL_DUMP_VERSION_NO_XOR_KEY) {
99 // Leave XOR-key empty
100 } else if (version == MEMPOOL_DUMP_VERSION) {
101 file >> xor_key;
102 } else {
103 return false;
104 }
105 file.SetXor(xor_key);
106 uint64_t total_txns_to_load;
107 file >> total_txns_to_load;
108 static constexpr uint64_t MAX_MEMPOOL_LOAD_TXNS = 500000;
109 if (total_txns_to_load > MAX_MEMPOOL_LOAD_TXNS) {
110 LogWarning("Mempool dump file specifies %u transactions (max %u), truncating\n", total_txns_to_load, MAX_MEMPOOL_LOAD_TXNS);
111 total_txns_to_load = MAX_MEMPOOL_LOAD_TXNS;
112 }
113 uint64_t txns_tried = 0;
114 LogInfo("Loading %u mempool transactions from file...\n", total_txns_to_load);
115 int next_tenth_to_report = 0;
116 while (txns_tried < total_txns_to_load) {
117 const int percentage_done(100.0 * txns_tried / total_txns_to_load);
118 if (next_tenth_to_report < percentage_done / 10) {
119 LogInfo("Progress loading mempool transactions from file: %d%% (tried %u, %u remaining)\n",
120 percentage_done, txns_tried, total_txns_to_load - txns_tried);
121 next_tenth_to_report = percentage_done / 10;
122 }
123 ++txns_tried;
124
125 CTransactionRef tx;
126 int64_t nTime;
127 int64_t nFeeDelta;
128 file >> TX_WITH_WITNESS(tx);
129 file >> nTime;
130 file >> nFeeDelta;
131
132 if (opts.use_current_time) {
133 nTime = TicksSinceEpoch<std::chrono::seconds>(now);
134 }
135
136 CAmount amountdelta = nFeeDelta;
137 if (amountdelta && std::abs(amountdelta) > tx->GetValueOut()) {
138 amountdelta = 0;
139 }
140 if (amountdelta && opts.apply_fee_delta_priority) {
141 pool.PrioritiseTransaction(tx->GetHash(), amountdelta);
142 }
143 if (nTime > TicksSinceEpoch<std::chrono::seconds>(now - pool.m_opts.expiry)) {
144 LOCK(cs_main);
145 const auto& accepted = AcceptToMemoryPool(active_chainstate, tx, nTime, empty_ignore_rejects, /*test_accept=*/false);
146 if (accepted.m_result_type == MempoolAcceptResult::ResultType::VALID) {
147 ++count;
148 } else {
149 // mempool may contain the transaction already, e.g. from
150 // wallet(s) having loaded it while we were processing
151 // mempool transactions; consider these as valid, instead of
152 // failed, but mark them as 'already there'
153 if (pool.exists(GenTxid::Txid(tx->GetHash()))) {
154 ++already_there;
155 } else {
156 ++failed;
157 }
158 }
159 } else {
160 ++expired;
161 }
162 if (active_chainstate.m_chainman.m_interrupt)
163 return false;
164 }
165 constexpr size_t MAX_MAPDELTAS = 1000000;
166 size_t mapDeltasSize = ReadCompactSize(file);
167 if (mapDeltasSize > MAX_MAPDELTAS) return false;
168 std::map<uint256, CAmount> mapDeltas;
169 for (size_t i = 0; i < mapDeltasSize; ++i) {
170 uint256 key;
171 CAmount val;
172 file >> key >> val;
173 mapDeltas[key] = val;
174 }
175
176 if (opts.apply_fee_delta_priority) {
177 for (const auto& i : mapDeltas) {
178 pool.PrioritiseTransaction(i.first, i.second);
179 }
180 }
181
182 std::set<uint256> unbroadcast_txids;
183 file >> unbroadcast_txids;
184 if (opts.apply_unbroadcast_set) {
185 unbroadcast = unbroadcast_txids.size();
186 for (const auto& txid : unbroadcast_txids) {
187 // Ensure transactions were accepted to mempool then add to
188 // unbroadcast set.
189 if (pool.get(txid) != nullptr) pool.AddUnbroadcastTx(txid);
190 }
191 }
192 } catch (const std::exception& e) {
193 LogInfo("Failed to deserialize mempool data on file: %s. Continuing anyway.\n", e.what());
194 return false;
195 }
196
197 if (opts.load_knots_data) {
198 auto knots_filepath = load_path;
199 knots_filepath.replace_filename("mempool-knots.dat");
200 LoadMempoolKnots(pool, knots_filepath, opts.mockable_fopen_function);
201 }
202
203 LogInfo("Imported mempool transactions from file: %i succeeded, %i failed, %i expired, %i already there, %i waiting for initial broadcast\n", count, failed, expired, already_there, unbroadcast);
204 return true;
205 }
206
207 bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mockable_fopen_function, bool skip_file_commit)
208 {
209 auto start = SteadyClock::now();
210
211 std::map<uint256, CAmount> mapDeltas;
212 std::map<uint256, double> priority_deltas;
213 std::vector<TxMempoolInfo> vinfo;
214 std::set<uint256> unbroadcast_txids;
215
216 static Mutex dump_mutex;
217 LOCK(dump_mutex);
218
219 {
220 LOCK(pool.cs);
221 for (const auto &i : pool.mapDeltas) {
222 if (i.second.first) { // priority delta
223 priority_deltas[i.first] = i.second.first;
224 }
225 if (i.second.second) { // fee delta
226 mapDeltas[i.first] = i.second.second;
227 }
228 }
229 vinfo = pool.infoAll();
230 unbroadcast_txids = pool.GetUnbroadcastTxs();
231 }
232
233 auto mid = SteadyClock::now();
234
235 AutoFile file{mockable_fopen_function(dump_path + ".new", "wb")};
236 if (file.IsNull()) {
237 return false;
238 }
239
240 try {
241 const uint64_t version{pool.m_opts.persist_v1_dat ? MEMPOOL_DUMP_VERSION_NO_XOR_KEY : MEMPOOL_DUMP_VERSION};
242 file << version;
243
244 Obfuscation xor_key{};
245 if (!pool.m_opts.persist_v1_dat) {
246 xor_key = Obfuscation{FastRandomContext{}.randbytes<Obfuscation::KEY_SIZE>()};
247 file << xor_key;
248 }
249 file.SetXor(xor_key);
250
251 uint64_t mempool_transactions_to_write(vinfo.size());
252 file << mempool_transactions_to_write;
253 LogInfo("Writing %u mempool transactions to file...\n", mempool_transactions_to_write);
254 for (const auto& i : vinfo) {
255 file << TX_WITH_WITNESS(*(i.tx));
256 file << int64_t{count_seconds(i.m_time)};
257 file << int64_t{i.nFeeDelta};
258 mapDeltas.erase(i.tx->GetHash());
259 }
260
261 file << mapDeltas;
262
263 LogInfo("Writing %d unbroadcast transactions to file.\n", unbroadcast_txids.size());
264 file << unbroadcast_txids;
265
266 if (!skip_file_commit && !file.Commit()) {
267 (void)file.fclose();
268 throw std::runtime_error("Commit failed");
269 }
270 if (file.fclose() != 0) {
271 const fs::path file_fspath{dump_path + ".new"};
272 throw std::runtime_error(
273 strprintf("Error closing %s: %s", fs::PathToString(file_fspath), SysErrorString(errno)));
274 }
275
276 auto knots_filepath = dump_path;
277 knots_filepath.replace_filename("mempool-knots.dat");
278 LogInfo("Writing %u mempool prioritizations to file...\n", priority_deltas.size());
279 if (priority_deltas.size()) {
280 auto knots_tmppath = knots_filepath;
281 knots_tmppath += ".new";
282
283 AutoFile file{mockable_fopen_function(knots_tmppath, "wb")};
284 if (file.IsNull()) return false;
285
286 uint64_t version = MEMPOOL_KNOTS_DUMP_VERSION;
287 file << version;
288
289 WriteCompactSize(file, priority_deltas.size());
290 for (const auto& [txid, priority] : priority_deltas) {
291 Serialize(file, txid);
292 const uint64_t encoded_priority = EncodeDouble(priority);
293 Serialize(file, encoded_priority);
294 }
295
296 if (!file.Commit()) throw std::runtime_error("Commit failed");
297 if (file.fclose() != 0) {
298 throw std::runtime_error(
299 strprintf("Error closing %s: %s", fs::PathToString(knots_tmppath), SysErrorString(errno)));
300 }
301 if (!RenameOver(knots_tmppath, knots_filepath)) {
302 throw std::runtime_error("Rename failed (mempool-knots.dat)");
303 }
304 } else {
305 if (!fs::remove(knots_filepath)) {
306 LogWarning("Failed to remove stale mempool-knots.dat\n");
307 }
308 }
309
310 if (!RenameOver(dump_path + ".new", dump_path)) {
311 throw std::runtime_error("Rename failed");
312 }
313 auto last = SteadyClock::now();
314
315 LogInfo("Dumped mempool: %.3fs to copy, %.3fs to dump, %d bytes dumped to file\n",
316 Ticks<SecondsDouble>(mid - start),
317 Ticks<SecondsDouble>(last - mid),
318 (priority_deltas.empty() ? 0 : fs::file_size(knots_filepath)) +
319 fs::file_size(dump_path));
320 } catch (const std::exception& e) {
321 LogInfo("Failed to dump mempool: %s. Continuing anyway.\n", e.what());
322 (void)file.fclose();
323 return false;
324 }
325 return true;
326 }
327
328 } // namespace node
329