fees.cpp raw

   1  // Copyright (c) 2009-2010 Satoshi Nakamoto
   2  // Copyright (c) 2009-2022 The Limenka developers
   3  // Distributed under the MIT software license, see the accompanying
   4  // file COPYING or http://www.opensource.org/licenses/mit-license.php.
   5  
   6  #include <policy/fees.h>
   7  
   8  #include <common/system.h>
   9  #include <consensus/amount.h>
  10  #include <kernel/mempool_entry.h>
  11  #include <logging.h>
  12  #include <policy/feerate.h>
  13  #include <primitives/transaction.h>
  14  #include <random.h>
  15  #include <serialize.h>
  16  #include <streams.h>
  17  #include <sync.h>
  18  #include <tinyformat.h>
  19  #include <uint256.h>
  20  #include <util/fs.h>
  21  #include <util/serfloat.h>
  22  #include <util/syserror.h>
  23  #include <util/time.h>
  24  
  25  #include <algorithm>
  26  #include <cassert>
  27  #include <chrono>
  28  #include <cmath>
  29  #include <cstddef>
  30  #include <cstdint>
  31  #include <exception>
  32  #include <stdexcept>
  33  #include <utility>
  34  
  35  // The current format written, and the version required to read. Must be
  36  // increased to at least 289900+1 on the next breaking change.
  37  constexpr int CURRENT_FEES_FILE_VERSION{149900};
  38  
  39  static constexpr double INF_FEERATE = 1e99;
  40  
  41  std::string StringForFeeEstimateHorizon(FeeEstimateHorizon horizon)
  42  {
  43      switch (horizon) {
  44      case FeeEstimateHorizon::SHORT_HALFLIFE: return "short";
  45      case FeeEstimateHorizon::MED_HALFLIFE: return "medium";
  46      case FeeEstimateHorizon::LONG_HALFLIFE: return "long";
  47      } // no default case, so the compiler can warn about missing cases
  48      assert(false);
  49  }
  50  
  51  namespace {
  52  
  53  struct EncodedDoubleFormatter
  54  {
  55      template<typename Stream> void Ser(Stream &s, double v)
  56      {
  57          s << EncodeDouble(v);
  58      }
  59  
  60      template<typename Stream> void Unser(Stream& s, double& v)
  61      {
  62          uint64_t encoded;
  63          s >> encoded;
  64          v = DecodeDouble(encoded);
  65      }
  66  };
  67  
  68  } // namespace
  69  
  70  /**
  71   * We will instantiate an instance of this class to track transactions that were
  72   * included in a block. We will lump transactions into a bucket according to their
  73   * approximate feerate and then track how long it took for those txs to be included in a block
  74   *
  75   * The tracking of unconfirmed (mempool) transactions is completely independent of the
  76   * historical tracking of transactions that have been confirmed in a block.
  77   */
  78  class TxConfirmStats
  79  {
  80  private:
  81      //Define the buckets we will group transactions into
  82      const std::vector<double>& buckets;              // The upper-bound of the range for the bucket (inclusive)
  83      const std::map<double, unsigned int>& bucketMap; // Map of bucket upper-bound to index into all vectors by bucket
  84  
  85      // For each bucket X:
  86      // Count the total # of txs in each bucket
  87      // Track the historical moving average of this total over blocks
  88      std::vector<double> txCtAvg;
  89  
  90      // Count the total # of txs confirmed within Y blocks in each bucket
  91      // Track the historical moving average of these totals over blocks
  92      std::vector<std::vector<double>> confAvg; // confAvg[Y][X]
  93  
  94      // Track moving avg of txs which have been evicted from the mempool
  95      // after failing to be confirmed within Y blocks
  96      std::vector<std::vector<double>> failAvg; // failAvg[Y][X]
  97  
  98      // Sum the total feerate of all tx's in each bucket
  99      // Track the historical moving average of this total over blocks
 100      std::vector<double> m_feerate_avg;
 101  
 102      // Combine the conf counts with tx counts to calculate the confirmation % for each Y,X
 103      // Combine the total value with the tx counts to calculate the avg feerate per bucket
 104  
 105      double decay;
 106  
 107      // Resolution (# of blocks) with which confirmations are tracked
 108      unsigned int scale;
 109  
 110      // Mempool counts of outstanding transactions
 111      // For each bucket X, track the number of transactions in the mempool
 112      // that are unconfirmed for each possible confirmation value Y
 113      std::vector<std::vector<int> > unconfTxs;  //unconfTxs[Y][X]
 114      // transactions still unconfirmed after GetMaxConfirms for each bucket
 115      std::vector<int> oldUnconfTxs;
 116  
 117      void resizeInMemoryCounters(size_t newbuckets);
 118  
 119  public:
 120      /**
 121       * Create new TxConfirmStats. This is called by BlockPolicyEstimator's
 122       * constructor with default values.
 123       * @param defaultBuckets contains the upper limits for the bucket boundaries
 124       * @param maxPeriods max number of periods to track
 125       * @param decay how much to decay the historical moving average per block
 126       */
 127      TxConfirmStats(const std::vector<double>& defaultBuckets, const std::map<double, unsigned int>& defaultBucketMap,
 128                     unsigned int maxPeriods, double decay, unsigned int scale);
 129  
 130      /** Roll the circular buffer for unconfirmed txs*/
 131      void ClearCurrent(unsigned int nBlockHeight);
 132  
 133      /**
 134       * Record a new transaction data point in the current block stats
 135       * @param blocksToConfirm the number of blocks it took this transaction to confirm
 136       * @param val the feerate of the transaction
 137       * @warning blocksToConfirm is 1-based and has to be >= 1
 138       */
 139      void Record(int blocksToConfirm, double val);
 140  
 141      /** Record a new transaction entering the mempool*/
 142      unsigned int NewTx(unsigned int nBlockHeight, double val);
 143  
 144      /** Remove a transaction from mempool tracking stats*/
 145      void removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight,
 146                    unsigned int bucketIndex, bool inBlock);
 147  
 148      /** Update our estimates by decaying our historical moving average and updating
 149          with the data gathered from the current block */
 150      void UpdateMovingAverages();
 151  
 152      /**
 153       * Calculate a feerate estimate.  Find the lowest value bucket (or range of buckets
 154       * to make sure we have enough data points) whose transactions still have sufficient likelihood
 155       * of being confirmed within the target number of confirmations
 156       * @param confTarget target number of confirmations
 157       * @param sufficientTxVal required average number of transactions per block in a bucket range
 158       * @param minSuccess the success probability we require
 159       * @param nBlockHeight the current block height
 160       */
 161      double EstimateMedianVal(int confTarget, double sufficientTxVal,
 162                               double minSuccess, unsigned int nBlockHeight,
 163                               EstimationResult *result = nullptr) const;
 164  
 165      /** Return the max number of confirms we're tracking */
 166      unsigned int GetMaxConfirms() const { return scale * confAvg.size(); }
 167  
 168      /** Write state of estimation data to a file*/
 169      void Write(AutoFile& fileout) const;
 170  
 171      /**
 172       * Read saved state of estimation data from a file and replace all internal data structures and
 173       * variables with this state.
 174       */
 175      void Read(AutoFile& filein, size_t numBuckets);
 176  };
 177  
 178  
 179  TxConfirmStats::TxConfirmStats(const std::vector<double>& defaultBuckets,
 180                                  const std::map<double, unsigned int>& defaultBucketMap,
 181                                 unsigned int maxPeriods, double _decay, unsigned int _scale)
 182      : buckets(defaultBuckets), bucketMap(defaultBucketMap), decay(_decay), scale(_scale)
 183  {
 184      assert(_scale != 0 && "_scale must be non-zero");
 185      confAvg.resize(maxPeriods);
 186      failAvg.resize(maxPeriods);
 187      for (unsigned int i = 0; i < maxPeriods; i++) {
 188          confAvg[i].resize(buckets.size());
 189          failAvg[i].resize(buckets.size());
 190      }
 191  
 192      txCtAvg.resize(buckets.size());
 193      m_feerate_avg.resize(buckets.size());
 194  
 195      resizeInMemoryCounters(buckets.size());
 196  }
 197  
 198  void TxConfirmStats::resizeInMemoryCounters(size_t newbuckets) {
 199      // newbuckets must be passed in because the buckets referred to during Read have not been updated yet.
 200      unconfTxs.resize(GetMaxConfirms());
 201      for (unsigned int i = 0; i < unconfTxs.size(); i++) {
 202          unconfTxs[i].resize(newbuckets);
 203      }
 204      oldUnconfTxs.resize(newbuckets);
 205  }
 206  
 207  // Roll the unconfirmed txs circular buffer
 208  void TxConfirmStats::ClearCurrent(unsigned int nBlockHeight)
 209  {
 210      for (unsigned int j = 0; j < buckets.size(); j++) {
 211          oldUnconfTxs[j] += unconfTxs[nBlockHeight % unconfTxs.size()][j];
 212          unconfTxs[nBlockHeight%unconfTxs.size()][j] = 0;
 213      }
 214  }
 215  
 216  
 217  void TxConfirmStats::Record(int blocksToConfirm, double feerate)
 218  {
 219      // blocksToConfirm is 1-based
 220      if (blocksToConfirm < 1)
 221          return;
 222      int periodsToConfirm = (blocksToConfirm + scale - 1) / scale;
 223      unsigned int bucketindex = bucketMap.lower_bound(feerate)->second;
 224      for (size_t i = periodsToConfirm; i <= confAvg.size(); i++) {
 225          confAvg[i - 1][bucketindex]++;
 226      }
 227      txCtAvg[bucketindex]++;
 228      m_feerate_avg[bucketindex] += feerate;
 229  }
 230  
 231  void TxConfirmStats::UpdateMovingAverages()
 232  {
 233      assert(confAvg.size() == failAvg.size());
 234      for (unsigned int j = 0; j < buckets.size(); j++) {
 235          for (unsigned int i = 0; i < confAvg.size(); i++) {
 236              confAvg[i][j] *= decay;
 237              failAvg[i][j] *= decay;
 238          }
 239          m_feerate_avg[j] *= decay;
 240          txCtAvg[j] *= decay;
 241      }
 242  }
 243  
 244  // returns -1 on error conditions
 245  double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal,
 246                                           double successBreakPoint, unsigned int nBlockHeight,
 247                                           EstimationResult *result) const
 248  {
 249      // Counters for a bucket (or range of buckets)
 250      double nConf = 0; // Number of tx's confirmed within the confTarget
 251      double totalNum = 0; // Total number of tx's that were ever confirmed
 252      int extraNum = 0;  // Number of tx's still in mempool for confTarget or longer
 253      double failNum = 0; // Number of tx's that were never confirmed but removed from the mempool after confTarget
 254      const int periodTarget = (confTarget + scale - 1) / scale;
 255      const int maxbucketindex = buckets.size() - 1;
 256  
 257      // We'll combine buckets until we have enough samples.
 258      // The near and far variables will define the range we've combined
 259      // The best variables are the last range we saw which still had a high
 260      // enough confirmation rate to count as success.
 261      // The cur variables are the current range we're counting.
 262      unsigned int curNearBucket = maxbucketindex;
 263      unsigned int bestNearBucket = maxbucketindex;
 264      unsigned int curFarBucket = maxbucketindex;
 265      unsigned int bestFarBucket = maxbucketindex;
 266  
 267      // We'll always group buckets into sets that meet sufficientTxVal --
 268      // this ensures that we're using consistent groups between different
 269      // confirmation targets.
 270      double partialNum = 0;
 271  
 272      bool foundAnswer = false;
 273      unsigned int bins = unconfTxs.size();
 274      bool newBucketRange = true;
 275      bool passing = true;
 276      EstimatorBucket passBucket;
 277      EstimatorBucket failBucket;
 278  
 279      // Start counting from highest feerate transactions
 280      for (int bucket = maxbucketindex; bucket >= 0; --bucket) {
 281          if (newBucketRange) {
 282              curNearBucket = bucket;
 283              newBucketRange = false;
 284          }
 285          curFarBucket = bucket;
 286          nConf += confAvg[periodTarget - 1][bucket];
 287          partialNum += txCtAvg[bucket];
 288          totalNum += txCtAvg[bucket];
 289          failNum += failAvg[periodTarget - 1][bucket];
 290          for (unsigned int confct = confTarget; confct < GetMaxConfirms(); confct++)
 291              extraNum += unconfTxs[(nBlockHeight - confct) % bins][bucket];
 292          extraNum += oldUnconfTxs[bucket];
 293          // If we have enough transaction data points in this range of buckets,
 294          // we can test for success
 295          // (Only count the confirmed data points, so that each confirmation count
 296          // will be looking at the same amount of data and same bucket breaks)
 297  
 298          if (partialNum < sufficientTxVal / (1 - decay)) {
 299              // the buckets we've added in this round aren't sufficient
 300              // so keep adding
 301              continue;
 302          } else {
 303              partialNum = 0; // reset for the next range we'll add
 304  
 305              double curPct = nConf / (totalNum + failNum + extraNum);
 306  
 307              // Check to see if we are no longer getting confirmed at the success rate
 308              if (curPct < successBreakPoint) {
 309                  if (passing == true) {
 310                      // First time we hit a failure record the failed bucket
 311                      unsigned int failMinBucket = std::min(curNearBucket, curFarBucket);
 312                      unsigned int failMaxBucket = std::max(curNearBucket, curFarBucket);
 313                      failBucket.start = failMinBucket ? buckets[failMinBucket - 1] : 0;
 314                      failBucket.end = buckets[failMaxBucket];
 315                      failBucket.withinTarget = nConf;
 316                      failBucket.totalConfirmed = totalNum;
 317                      failBucket.inMempool = extraNum;
 318                      failBucket.leftMempool = failNum;
 319                      passing = false;
 320                  }
 321                  continue;
 322              }
 323              // Otherwise update the cumulative stats, and the bucket variables
 324              // and reset the counters
 325              else {
 326                  failBucket = EstimatorBucket(); // Reset any failed bucket, currently passing
 327                  foundAnswer = true;
 328                  passing = true;
 329                  passBucket.withinTarget = nConf;
 330                  nConf = 0;
 331                  passBucket.totalConfirmed = totalNum;
 332                  totalNum = 0;
 333                  passBucket.inMempool = extraNum;
 334                  passBucket.leftMempool = failNum;
 335                  failNum = 0;
 336                  extraNum = 0;
 337                  bestNearBucket = curNearBucket;
 338                  bestFarBucket = curFarBucket;
 339                  newBucketRange = true;
 340              }
 341          }
 342      }
 343  
 344      double median = -1;
 345      double txSum = 0;
 346  
 347      // Calculate the "average" feerate of the best bucket range that met success conditions
 348      // Find the bucket with the median transaction and then report the average feerate from that bucket
 349      // This is a compromise between finding the median which we can't since we don't save all tx's
 350      // and reporting the average which is less accurate
 351      unsigned int minBucket = std::min(bestNearBucket, bestFarBucket);
 352      unsigned int maxBucket = std::max(bestNearBucket, bestFarBucket);
 353      for (unsigned int j = minBucket; j <= maxBucket; j++) {
 354          txSum += txCtAvg[j];
 355      }
 356      if (foundAnswer && txSum != 0) {
 357          txSum = txSum / 2;
 358          for (unsigned int j = minBucket; j <= maxBucket; j++) {
 359              if (txCtAvg[j] < txSum)
 360                  txSum -= txCtAvg[j];
 361              else { // we're in the right bucket
 362                  median = m_feerate_avg[j] / txCtAvg[j];
 363                  break;
 364              }
 365          }
 366  
 367          passBucket.start = minBucket ? buckets[minBucket-1] : 0;
 368          passBucket.end = buckets[maxBucket];
 369      }
 370  
 371      // If we were passing until we reached last few buckets with insufficient data, then report those as failed
 372      if (passing && !newBucketRange) {
 373          unsigned int failMinBucket = std::min(curNearBucket, curFarBucket);
 374          unsigned int failMaxBucket = std::max(curNearBucket, curFarBucket);
 375          failBucket.start = failMinBucket ? buckets[failMinBucket - 1] : 0;
 376          failBucket.end = buckets[failMaxBucket];
 377          failBucket.withinTarget = nConf;
 378          failBucket.totalConfirmed = totalNum;
 379          failBucket.inMempool = extraNum;
 380          failBucket.leftMempool = failNum;
 381      }
 382  
 383      float passed_within_target_perc = 0.0;
 384      float failed_within_target_perc = 0.0;
 385      if ((passBucket.totalConfirmed + passBucket.inMempool + passBucket.leftMempool)) {
 386          passed_within_target_perc = 100 * passBucket.withinTarget / (passBucket.totalConfirmed + passBucket.inMempool + passBucket.leftMempool);
 387      }
 388      if ((failBucket.totalConfirmed + failBucket.inMempool + failBucket.leftMempool)) {
 389          failed_within_target_perc = 100 * failBucket.withinTarget / (failBucket.totalConfirmed + failBucket.inMempool + failBucket.leftMempool);
 390      }
 391  
 392      LogDebug(BCLog::ESTIMATEFEE, "FeeEst: %d > %.0f%% decay %.5f: feerate: %g from (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out) Fail: (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out)\n",
 393               confTarget, 100.0 * successBreakPoint, decay,
 394               median, passBucket.start, passBucket.end,
 395               passed_within_target_perc,
 396               passBucket.withinTarget, passBucket.totalConfirmed, passBucket.inMempool, passBucket.leftMempool,
 397               failBucket.start, failBucket.end,
 398               failed_within_target_perc,
 399               failBucket.withinTarget, failBucket.totalConfirmed, failBucket.inMempool, failBucket.leftMempool);
 400  
 401  
 402      if (result) {
 403          result->pass = passBucket;
 404          result->fail = failBucket;
 405          result->decay = decay;
 406          result->scale = scale;
 407      }
 408      return median;
 409  }
 410  
 411  void TxConfirmStats::Write(AutoFile& fileout) const
 412  {
 413      fileout << Using<EncodedDoubleFormatter>(decay);
 414      fileout << scale;
 415      fileout << Using<VectorFormatter<EncodedDoubleFormatter>>(m_feerate_avg);
 416      fileout << Using<VectorFormatter<EncodedDoubleFormatter>>(txCtAvg);
 417      fileout << Using<VectorFormatter<VectorFormatter<EncodedDoubleFormatter>>>(confAvg);
 418      fileout << Using<VectorFormatter<VectorFormatter<EncodedDoubleFormatter>>>(failAvg);
 419  }
 420  
 421  void TxConfirmStats::Read(AutoFile& filein, size_t numBuckets)
 422  {
 423      // Read data file and do some very basic sanity checking
 424      // buckets and bucketMap are not updated yet, so don't access them
 425      // If there is a read failure, we'll just discard this entire object anyway
 426  
 427      // The current version will store the decay with each individual TxConfirmStats and also keep a scale factor
 428      filein >> Using<EncodedDoubleFormatter>(decay);
 429      if (decay <= 0 || decay >= 1) {
 430          throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)");
 431      }
 432      filein >> scale;
 433      if (scale == 0) {
 434          throw std::runtime_error("Corrupt estimates file. Scale must be non-zero");
 435      }
 436  
 437      filein >> Using<VectorFormatter<EncodedDoubleFormatter>>(m_feerate_avg);
 438      if (m_feerate_avg.size() != numBuckets) {
 439          throw std::runtime_error("Corrupt estimates file. Mismatch in feerate average bucket count");
 440      }
 441      filein >> Using<VectorFormatter<EncodedDoubleFormatter>>(txCtAvg);
 442      if (txCtAvg.size() != numBuckets) {
 443          throw std::runtime_error("Corrupt estimates file. Mismatch in tx count bucket count");
 444      }
 445      filein >> Using<VectorFormatter<VectorFormatter<EncodedDoubleFormatter>>>(confAvg);
 446      const size_t maxPeriods = confAvg.size();
 447  
 448      if (maxPeriods == 0 || scale > (6 * 24 * 7) / maxPeriods) { // one week
 449          throw std::runtime_error("Corrupt estimates file.  Must maintain estimates for between 1 and 1008 (one week) confirms");
 450      }
 451      for (unsigned int i = 0; i < maxPeriods; i++) {
 452          if (confAvg[i].size() != numBuckets) {
 453              throw std::runtime_error("Corrupt estimates file. Mismatch in feerate conf average bucket count");
 454          }
 455      }
 456  
 457      filein >> Using<VectorFormatter<VectorFormatter<EncodedDoubleFormatter>>>(failAvg);
 458      if (maxPeriods != failAvg.size()) {
 459          throw std::runtime_error("Corrupt estimates file. Mismatch in confirms tracked for failures");
 460      }
 461      for (unsigned int i = 0; i < maxPeriods; i++) {
 462          if (failAvg[i].size() != numBuckets) {
 463              throw std::runtime_error("Corrupt estimates file. Mismatch in one of failure average bucket counts");
 464          }
 465      }
 466  
 467      // Resize the current block variables which aren't stored in the data file
 468      // to match the number of confirms and buckets
 469      resizeInMemoryCounters(numBuckets);
 470  
 471      const size_t maxConfirms = scale * maxPeriods;
 472      LogDebug(BCLog::ESTIMATEFEE, "Reading estimates: %u buckets counting confirms up to %u blocks\n",
 473               numBuckets, maxConfirms);
 474  }
 475  
 476  unsigned int TxConfirmStats::NewTx(unsigned int nBlockHeight, double val)
 477  {
 478      unsigned int bucketindex = bucketMap.lower_bound(val)->second;
 479      unsigned int blockIndex = nBlockHeight % unconfTxs.size();
 480      unconfTxs[blockIndex][bucketindex]++;
 481      return bucketindex;
 482  }
 483  
 484  void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, unsigned int bucketindex, bool inBlock)
 485  {
 486      //nBestSeenHeight is not updated yet for the new block
 487      int blocksAgo = nBestSeenHeight - entryHeight;
 488      if (nBestSeenHeight == 0)  // the BlockPolicyEstimator hasn't seen any blocks yet
 489          blocksAgo = 0;
 490      if (blocksAgo < 0) {
 491          LogDebug(BCLog::ESTIMATEFEE, "Blockpolicy error, blocks ago is negative for mempool tx\n");
 492          return;  //This can't happen because we call this with our best seen height, no entries can have higher
 493      }
 494  
 495      if (blocksAgo >= (int)unconfTxs.size()) {
 496          if (oldUnconfTxs[bucketindex] > 0) {
 497              oldUnconfTxs[bucketindex]--;
 498          } else {
 499              LogDebug(BCLog::ESTIMATEFEE, "Blockpolicy error, mempool tx removed from >25 blocks,bucketIndex=%u already\n",
 500                       bucketindex);
 501          }
 502      }
 503      else {
 504          unsigned int blockIndex = entryHeight % unconfTxs.size();
 505          if (unconfTxs[blockIndex][bucketindex] > 0) {
 506              unconfTxs[blockIndex][bucketindex]--;
 507          } else {
 508              LogDebug(BCLog::ESTIMATEFEE, "Blockpolicy error, mempool tx removed from blockIndex=%u,bucketIndex=%u already\n",
 509                       blockIndex, bucketindex);
 510          }
 511      }
 512      if (!inBlock && (unsigned int)blocksAgo >= scale) { // Only counts as a failure if not confirmed for entire period
 513          assert(scale != 0);
 514          unsigned int periodsAgo = blocksAgo / scale;
 515          for (size_t i = 0; i < periodsAgo && i < failAvg.size(); i++) {
 516              failAvg[i][bucketindex]++;
 517          }
 518      }
 519  }
 520  
 521  bool CBlockPolicyEstimator::removeTx(uint256 hash)
 522  {
 523      LOCK(m_cs_fee_estimator);
 524      return _removeTx(hash, /*inBlock=*/false);
 525  }
 526  
 527  bool CBlockPolicyEstimator::_removeTx(const uint256& hash, bool inBlock)
 528  {
 529      AssertLockHeld(m_cs_fee_estimator);
 530      std::map<uint256, TxStatsInfo>::iterator pos = mapMemPoolTxs.find(hash);
 531      if (pos != mapMemPoolTxs.end()) {
 532          feeStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock);
 533          shortStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock);
 534          longStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock);
 535          mapMemPoolTxs.erase(hash);
 536          return true;
 537      } else {
 538          return false;
 539      }
 540  }
 541  
 542  CBlockPolicyEstimator::CBlockPolicyEstimator(const fs::path& estimation_filepath, const bool read_stale_estimates)
 543      : m_estimation_filepath{estimation_filepath}
 544  {
 545      static_assert(MIN_BUCKET_FEERATE > 0, "Min feerate must be nonzero");
 546      size_t bucketIndex = 0;
 547  
 548      for (double bucketBoundary = MIN_BUCKET_FEERATE; bucketBoundary <= MAX_BUCKET_FEERATE; bucketBoundary *= FEE_SPACING, bucketIndex++) {
 549          buckets.push_back(bucketBoundary);
 550          bucketMap[bucketBoundary] = bucketIndex;
 551      }
 552      buckets.push_back(INF_FEERATE);
 553      bucketMap[INF_FEERATE] = bucketIndex;
 554      assert(bucketMap.size() == buckets.size());
 555  
 556      feeStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE));
 557      shortStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE));
 558      longStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE));
 559  
 560      AutoFile est_file{fsbridge::fopen(m_estimation_filepath, "rb")};
 561  
 562      if (est_file.IsNull()) {
 563          LogPrintf("%s is not found. Continue anyway.\n", fs::PathToString(m_estimation_filepath));
 564          return;
 565      }
 566  
 567      std::chrono::hours file_age = GetFeeEstimatorFileAge();
 568      if (file_age > MAX_FILE_AGE && !read_stale_estimates) {
 569          LogWarning("Fee estimation file %s too old (age=%lld > %lld hours) and will not be used to avoid serving stale estimates.", fs::PathToString(m_estimation_filepath), Ticks<std::chrono::hours>(file_age), Ticks<std::chrono::hours>(MAX_FILE_AGE));
 570          return;
 571      }
 572  
 573      if (!Read(est_file)) {
 574          LogWarning("Failed to read fee estimates from %s. Continue anyway.", fs::PathToString(m_estimation_filepath));
 575      }
 576  }
 577  
 578  CBlockPolicyEstimator::~CBlockPolicyEstimator() = default;
 579  
 580  void CBlockPolicyEstimator::TransactionAddedToMempool(const NewMempoolTransactionInfo& tx, uint64_t /*unused*/)
 581  {
 582      processTransaction(tx);
 583  }
 584  
 585  void CBlockPolicyEstimator::TransactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason /*unused*/, uint64_t /*unused*/)
 586  {
 587      removeTx(tx->GetHash());
 588  }
 589  
 590  void CBlockPolicyEstimator::MempoolTransactionsRemovedForBlock(const std::vector<RemovedMempoolTransactionInfo>& txs_removed_for_block, unsigned int nBlockHeight)
 591  {
 592      processBlock(txs_removed_for_block, nBlockHeight);
 593  }
 594  
 595  void CBlockPolicyEstimator::processTransaction(const NewMempoolTransactionInfo& tx)
 596  {
 597      LOCK(m_cs_fee_estimator);
 598      const unsigned int txHeight = tx.info.txHeight;
 599      const auto& hash = tx.info.m_tx->GetHash();
 600      if (mapMemPoolTxs.count(hash)) {
 601          LogDebug(BCLog::ESTIMATEFEE, "Blockpolicy error mempool tx %s already being tracked\n",
 602                   hash.ToString());
 603          return;
 604      }
 605  
 606      if (txHeight != nBestSeenHeight) {
 607          // Ignore side chains and re-orgs; assuming they are random they don't
 608          // affect the estimate.  We'll potentially double count transactions in 1-block reorgs.
 609          // Ignore txs if BlockPolicyEstimator is not in sync with ActiveChain().Tip().
 610          // It will be synced next time a block is processed.
 611          return;
 612      }
 613      // This transaction should only count for fee estimation if:
 614      // - it's not being re-added during a reorg which bypasses typical mempool fee limits
 615      // - the node is not behind
 616      // - the transaction is not dependent on any other transactions in the mempool
 617      // - it's not part of a package.
 618      const bool validForFeeEstimation = tx.m_ignore_rejects.empty() && !tx.m_submitted_in_package && tx.m_chainstate_is_current && tx.m_has_no_mempool_parents;
 619  
 620      // Only want to be updating estimates when our blockchain is synced,
 621      // otherwise we'll miscalculate how many blocks its taking to get included.
 622      if (!validForFeeEstimation) {
 623          untrackedTxs++;
 624          return;
 625      }
 626      trackedTxs++;
 627  
 628      // Feerates are stored and reported as BTC-per-kb:
 629      const CFeeRate feeRate(tx.info.m_fee, tx.info.m_virtual_transaction_size);
 630  
 631      mapMemPoolTxs[hash].blockHeight = txHeight;
 632      unsigned int bucketIndex = feeStats->NewTx(txHeight, static_cast<double>(feeRate.GetFeePerK()));
 633      mapMemPoolTxs[hash].bucketIndex = bucketIndex;
 634      unsigned int bucketIndex2 = shortStats->NewTx(txHeight, static_cast<double>(feeRate.GetFeePerK()));
 635      assert(bucketIndex == bucketIndex2);
 636      unsigned int bucketIndex3 = longStats->NewTx(txHeight, static_cast<double>(feeRate.GetFeePerK()));
 637      assert(bucketIndex == bucketIndex3);
 638  }
 639  
 640  bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const RemovedMempoolTransactionInfo& tx)
 641  {
 642      AssertLockHeld(m_cs_fee_estimator);
 643      if (!_removeTx(tx.info.m_tx->GetHash(), true)) {
 644          // This transaction wasn't being tracked for fee estimation
 645          return false;
 646      }
 647  
 648      // How many blocks did it take for miners to include this transaction?
 649      // blocksToConfirm is 1-based, so a transaction included in the earliest
 650      // possible block has confirmation count of 1
 651      int blocksToConfirm = nBlockHeight - tx.info.txHeight;
 652      if (blocksToConfirm <= 0) {
 653          // This can't happen because we don't process transactions from a block with a height
 654          // lower than our greatest seen height
 655          LogDebug(BCLog::ESTIMATEFEE, "Blockpolicy error Transaction had negative blocksToConfirm\n");
 656          return false;
 657      }
 658  
 659      // Feerates are stored and reported as BTC-per-kb:
 660      CFeeRate feeRate(tx.info.m_fee, tx.info.m_virtual_transaction_size);
 661  
 662      feeStats->Record(blocksToConfirm, static_cast<double>(feeRate.GetFeePerK()));
 663      shortStats->Record(blocksToConfirm, static_cast<double>(feeRate.GetFeePerK()));
 664      longStats->Record(blocksToConfirm, static_cast<double>(feeRate.GetFeePerK()));
 665      return true;
 666  }
 667  
 668  void CBlockPolicyEstimator::processBlock(const std::vector<RemovedMempoolTransactionInfo>& txs_removed_for_block,
 669                                           unsigned int nBlockHeight)
 670  {
 671      LOCK(m_cs_fee_estimator);
 672      if (nBlockHeight <= nBestSeenHeight) {
 673          // Ignore side chains and re-orgs; assuming they are random
 674          // they don't affect the estimate.
 675          // And if an attacker can re-org the chain at will, then
 676          // you've got much bigger problems than "attacker can influence
 677          // transaction fees."
 678          return;
 679      }
 680  
 681      // Must update nBestSeenHeight in sync with ClearCurrent so that
 682      // calls to removeTx (via processBlockTx) correctly calculate age
 683      // of unconfirmed txs to remove from tracking.
 684      nBestSeenHeight = nBlockHeight;
 685  
 686      // Update unconfirmed circular buffer
 687      feeStats->ClearCurrent(nBlockHeight);
 688      shortStats->ClearCurrent(nBlockHeight);
 689      longStats->ClearCurrent(nBlockHeight);
 690  
 691      // Decay all exponential averages
 692      feeStats->UpdateMovingAverages();
 693      shortStats->UpdateMovingAverages();
 694      longStats->UpdateMovingAverages();
 695  
 696      unsigned int countedTxs = 0;
 697      // Update averages with data points from current block
 698      for (const auto& tx : txs_removed_for_block) {
 699          if (processBlockTx(nBlockHeight, tx))
 700              countedTxs++;
 701      }
 702  
 703      if (firstRecordedHeight == 0 && countedTxs > 0) {
 704          firstRecordedHeight = nBestSeenHeight;
 705          LogDebug(BCLog::ESTIMATEFEE, "Blockpolicy first recorded height %u\n", firstRecordedHeight);
 706      }
 707  
 708  
 709      LogDebug(BCLog::ESTIMATEFEE, "Blockpolicy estimates updated by %u of %u block txs, since last block %u of %u tracked, mempool map size %u, max target %u from %s\n",
 710               countedTxs, txs_removed_for_block.size(), trackedTxs, trackedTxs + untrackedTxs, mapMemPoolTxs.size(),
 711               MaxUsableEstimate(), HistoricalBlockSpan() > BlockSpan() ? "historical" : "current");
 712  
 713      trackedTxs = 0;
 714      untrackedTxs = 0;
 715  }
 716  
 717  CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) const
 718  {
 719      // It's not possible to get reasonable estimates for confTarget of 1
 720      if (confTarget <= 1)
 721          return CFeeRate(0);
 722  
 723      return estimateRawFee(confTarget, DOUBLE_SUCCESS_PCT, FeeEstimateHorizon::MED_HALFLIFE);
 724  }
 725  
 726  CFeeRate CBlockPolicyEstimator::estimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult* result) const
 727  {
 728      TxConfirmStats* stats = nullptr;
 729      double sufficientTxs = SUFFICIENT_FEETXS;
 730      switch (horizon) {
 731      case FeeEstimateHorizon::SHORT_HALFLIFE: {
 732          stats = shortStats.get();
 733          sufficientTxs = SUFFICIENT_TXS_SHORT;
 734          break;
 735      }
 736      case FeeEstimateHorizon::MED_HALFLIFE: {
 737          stats = feeStats.get();
 738          break;
 739      }
 740      case FeeEstimateHorizon::LONG_HALFLIFE: {
 741          stats = longStats.get();
 742          break;
 743      }
 744      } // no default case, so the compiler can warn about missing cases
 745      assert(stats);
 746  
 747      LOCK(m_cs_fee_estimator);
 748      // Return failure if trying to analyze a target we're not tracking
 749      if (confTarget <= 0 || (unsigned int)confTarget > stats->GetMaxConfirms())
 750          return CFeeRate(0);
 751      if (successThreshold > 1)
 752          return CFeeRate(0);
 753  
 754      double median = stats->EstimateMedianVal(confTarget, sufficientTxs, successThreshold, nBestSeenHeight, result);
 755  
 756      if (median < 0)
 757          return CFeeRate(0);
 758  
 759      return CFeeRate(llround(median));
 760  }
 761  
 762  unsigned int CBlockPolicyEstimator::HighestTargetTracked(FeeEstimateHorizon horizon) const
 763  {
 764      LOCK(m_cs_fee_estimator);
 765      switch (horizon) {
 766      case FeeEstimateHorizon::SHORT_HALFLIFE: {
 767          return shortStats->GetMaxConfirms();
 768      }
 769      case FeeEstimateHorizon::MED_HALFLIFE: {
 770          return feeStats->GetMaxConfirms();
 771      }
 772      case FeeEstimateHorizon::LONG_HALFLIFE: {
 773          return longStats->GetMaxConfirms();
 774      }
 775      } // no default case, so the compiler can warn about missing cases
 776      assert(false);
 777  }
 778  
 779  unsigned int CBlockPolicyEstimator::BlockSpan() const
 780  {
 781      if (firstRecordedHeight == 0) return 0;
 782      assert(nBestSeenHeight >= firstRecordedHeight);
 783  
 784      return nBestSeenHeight - firstRecordedHeight;
 785  }
 786  
 787  unsigned int CBlockPolicyEstimator::HistoricalBlockSpan() const
 788  {
 789      if (historicalFirst == 0) return 0;
 790      assert(historicalBest >= historicalFirst);
 791  
 792      if (nBestSeenHeight - historicalBest > OLDEST_ESTIMATE_HISTORY) return 0;
 793  
 794      return historicalBest - historicalFirst;
 795  }
 796  
 797  unsigned int CBlockPolicyEstimator::MaxUsableEstimate() const
 798  {
 799      // Block spans are divided by 2 to make sure there are enough potential failing data points for the estimate
 800      return std::min(longStats->GetMaxConfirms(), std::max(BlockSpan(), HistoricalBlockSpan()) / 2);
 801  }
 802  
 803  /** Return a fee estimate at the required successThreshold from the shortest
 804   * time horizon which tracks confirmations up to the desired target.  If
 805   * checkShorterHorizon is requested, also allow short time horizon estimates
 806   * for a lower target to reduce the given answer */
 807  double CBlockPolicyEstimator::estimateCombinedFee(unsigned int confTarget, double successThreshold, bool checkShorterHorizon, EstimationResult *result) const
 808  {
 809      double estimate = -1;
 810      if (confTarget >= 1 && confTarget <= longStats->GetMaxConfirms()) {
 811          // Find estimate from shortest time horizon possible
 812          if (confTarget <= shortStats->GetMaxConfirms()) { // short horizon
 813              estimate = shortStats->EstimateMedianVal(confTarget, SUFFICIENT_TXS_SHORT, successThreshold, nBestSeenHeight, result);
 814          }
 815          else if (confTarget <= feeStats->GetMaxConfirms()) { // medium horizon
 816              estimate = feeStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, successThreshold, nBestSeenHeight, result);
 817          }
 818          else { // long horizon
 819              estimate = longStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, successThreshold, nBestSeenHeight, result);
 820          }
 821          if (checkShorterHorizon) {
 822              EstimationResult tempResult;
 823              // If a lower confTarget from a more recent horizon returns a lower answer use it.
 824              if (confTarget > feeStats->GetMaxConfirms()) {
 825                  double medMax = feeStats->EstimateMedianVal(feeStats->GetMaxConfirms(), SUFFICIENT_FEETXS, successThreshold, nBestSeenHeight, &tempResult);
 826                  if (medMax > 0 && (estimate == -1 || medMax < estimate)) {
 827                      estimate = medMax;
 828                      if (result) *result = tempResult;
 829                  }
 830              }
 831              if (confTarget > shortStats->GetMaxConfirms()) {
 832                  double shortMax = shortStats->EstimateMedianVal(shortStats->GetMaxConfirms(), SUFFICIENT_TXS_SHORT, successThreshold, nBestSeenHeight, &tempResult);
 833                  if (shortMax > 0 && (estimate == -1 || shortMax < estimate)) {
 834                      estimate = shortMax;
 835                      if (result) *result = tempResult;
 836                  }
 837              }
 838          }
 839      }
 840      return estimate;
 841  }
 842  
 843  /** Ensure that for a conservative estimate, the DOUBLE_SUCCESS_PCT is also met
 844   * at 2 * target for any longer time horizons.
 845   */
 846  double CBlockPolicyEstimator::estimateConservativeFee(unsigned int doubleTarget, EstimationResult *result) const
 847  {
 848      double estimate = -1;
 849      EstimationResult tempResult;
 850      if (doubleTarget <= shortStats->GetMaxConfirms()) {
 851          estimate = feeStats->EstimateMedianVal(doubleTarget, SUFFICIENT_FEETXS, DOUBLE_SUCCESS_PCT, nBestSeenHeight, result);
 852      }
 853      if (doubleTarget <= feeStats->GetMaxConfirms()) {
 854          double longEstimate = longStats->EstimateMedianVal(doubleTarget, SUFFICIENT_FEETXS, DOUBLE_SUCCESS_PCT, nBestSeenHeight, &tempResult);
 855          if (longEstimate > estimate) {
 856              estimate = longEstimate;
 857              if (result) *result = tempResult;
 858          }
 859      }
 860      return estimate;
 861  }
 862  
 863  /** estimateSmartFee returns the max of the feerates calculated with a 60%
 864   * threshold required at target / 2, an 85% threshold required at target and a
 865   * 95% threshold required at 2 * target.  Each calculation is performed at the
 866   * shortest time horizon which tracks the required target.  Conservative
 867   * estimates, however, required the 95% threshold at 2 * target be met for any
 868   * longer time horizons also.
 869   */
 870  CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, FeeCalculation *feeCalc, bool conservative) const
 871  {
 872      LOCK(m_cs_fee_estimator);
 873  
 874      if (feeCalc) {
 875          feeCalc->desiredTarget = confTarget;
 876          feeCalc->returnedTarget = confTarget;
 877      }
 878  
 879      double median = -1;
 880      EstimationResult tempResult;
 881  
 882      // Return failure if trying to analyze a target we're not tracking
 883      if (confTarget <= 0 || (unsigned int)confTarget > longStats->GetMaxConfirms()) {
 884          return CFeeRate(0);  // error condition
 885      }
 886  
 887      // It's not possible to get reasonable estimates for confTarget of 1
 888      if (confTarget == 1) confTarget = 2;
 889  
 890      unsigned int maxUsableEstimate = MaxUsableEstimate();
 891      if ((unsigned int)confTarget > maxUsableEstimate) {
 892          confTarget = maxUsableEstimate;
 893      }
 894      if (feeCalc) feeCalc->returnedTarget = confTarget;
 895  
 896      if (confTarget <= 1) return CFeeRate(0); // error condition
 897  
 898      assert(confTarget > 0); //estimateCombinedFee and estimateConservativeFee take unsigned ints
 899      /** true is passed to estimateCombined fee for target/2 and target so
 900       * that we check the max confirms for shorter time horizons as well.
 901       * This is necessary to preserve monotonically increasing estimates.
 902       * For non-conservative estimates we do the same thing for 2*target, but
 903       * for conservative estimates we want to skip these shorter horizons
 904       * checks for 2*target because we are taking the max over all time
 905       * horizons so we already have monotonically increasing estimates and
 906       * the purpose of conservative estimates is not to let short term
 907       * fluctuations lower our estimates by too much.
 908       */
 909      double halfEst = estimateCombinedFee(confTarget/2, HALF_SUCCESS_PCT, true, &tempResult);
 910      if (feeCalc) {
 911          feeCalc->est = tempResult;
 912          feeCalc->reason = FeeReason::HALF_ESTIMATE;
 913      }
 914      median = halfEst;
 915      double actualEst = estimateCombinedFee(confTarget, SUCCESS_PCT, true, &tempResult);
 916      if (actualEst > median) {
 917          median = actualEst;
 918          if (feeCalc) {
 919              feeCalc->est = tempResult;
 920              feeCalc->reason = FeeReason::FULL_ESTIMATE;
 921          }
 922      }
 923      double doubleEst = estimateCombinedFee(2 * confTarget, DOUBLE_SUCCESS_PCT, !conservative, &tempResult);
 924      if (doubleEst > median) {
 925          median = doubleEst;
 926          if (feeCalc) {
 927              feeCalc->est = tempResult;
 928              feeCalc->reason = FeeReason::DOUBLE_ESTIMATE;
 929          }
 930      }
 931  
 932      if (conservative || median == -1) {
 933          double consEst =  estimateConservativeFee(2 * confTarget, &tempResult);
 934          if (consEst > median) {
 935              median = consEst;
 936              if (feeCalc) {
 937                  feeCalc->est = tempResult;
 938                  feeCalc->reason = FeeReason::CONSERVATIVE;
 939              }
 940          }
 941      }
 942  
 943      if (median < 0) return CFeeRate(0); // error condition
 944  
 945      return CFeeRate(llround(median));
 946  }
 947  
 948  void CBlockPolicyEstimator::Flush() {
 949      FlushUnconfirmed();
 950      FlushFeeEstimates();
 951  }
 952  
 953  bool CBlockPolicyEstimator::FlushFeeEstimates() const
 954  {
 955      AutoFile est_file{fsbridge::fopen(m_estimation_filepath, "wb")};
 956      if (est_file.IsNull() || !Write(est_file)) {
 957          LogWarning("Failed to write fee estimates to %s. Continue anyway.", fs::PathToString(m_estimation_filepath));
 958          (void)est_file.fclose();
 959          return false;
 960      } else if (est_file.fclose() != 0) {
 961          LogWarning("Failed to close fee estimates file %s: %s. Continuing anyway.", fs::PathToString(m_estimation_filepath), SysErrorString(errno));
 962          return false;
 963      } else {
 964          LogPrintf("Flushed fee estimates to %s.\n", fs::PathToString(m_estimation_filepath.filename()));
 965          return true;
 966      }
 967  }
 968  
 969  bool CBlockPolicyEstimator::Write(AutoFile& fileout) const
 970  {
 971      try {
 972          LOCK(m_cs_fee_estimator);
 973          fileout << CURRENT_FEES_FILE_VERSION;
 974          fileout << int{0}; // Unused dummy field. Written files may contain any value in [0, 289900]
 975          fileout << nBestSeenHeight;
 976          if (BlockSpan() > HistoricalBlockSpan()/2) {
 977              fileout << firstRecordedHeight << nBestSeenHeight;
 978          }
 979          else {
 980              fileout << historicalFirst << historicalBest;
 981          }
 982          fileout << Using<VectorFormatter<EncodedDoubleFormatter>>(buckets);
 983          feeStats->Write(fileout);
 984          shortStats->Write(fileout);
 985          longStats->Write(fileout);
 986      }
 987      catch (const std::exception&) {
 988          LogWarning("Unable to write policy estimator data (non-fatal)");
 989          return false;
 990      }
 991      return true;
 992  }
 993  
 994  bool CBlockPolicyEstimator::Read(AutoFile& filein)
 995  {
 996      try {
 997          LOCK(m_cs_fee_estimator);
 998          int nVersionRequired, dummy;
 999          filein >> nVersionRequired >> dummy;
1000          if (nVersionRequired > CURRENT_FEES_FILE_VERSION) {
1001              throw std::runtime_error{strprintf("File version (%d) too high to be read.", nVersionRequired)};
1002          }
1003  
1004          // Read fee estimates file into temporary variables so existing data
1005          // structures aren't corrupted if there is an exception.
1006          unsigned int nFileBestSeenHeight;
1007          filein >> nFileBestSeenHeight;
1008  
1009          if (nVersionRequired < CURRENT_FEES_FILE_VERSION) {
1010              LogWarning("Incompatible old fee estimation data (non-fatal). Version: %d", nVersionRequired);
1011          } else { // nVersionRequired == CURRENT_FEES_FILE_VERSION
1012              unsigned int nFileHistoricalFirst, nFileHistoricalBest;
1013              filein >> nFileHistoricalFirst >> nFileHistoricalBest;
1014              if (nFileHistoricalFirst > nFileHistoricalBest || nFileHistoricalBest > nFileBestSeenHeight) {
1015                  throw std::runtime_error("Corrupt estimates file. Historical block range for estimates is invalid");
1016              }
1017              std::vector<double> fileBuckets;
1018              filein >> Using<VectorFormatter<EncodedDoubleFormatter>>(fileBuckets);
1019              size_t numBuckets = fileBuckets.size();
1020              if (numBuckets <= 1 || numBuckets > 1000) {
1021                  throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets");
1022              }
1023  
1024              std::unique_ptr<TxConfirmStats> fileFeeStats(new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE));
1025              std::unique_ptr<TxConfirmStats> fileShortStats(new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE));
1026              std::unique_ptr<TxConfirmStats> fileLongStats(new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE));
1027              fileFeeStats->Read(filein, numBuckets);
1028              fileShortStats->Read(filein, numBuckets);
1029              fileLongStats->Read(filein, numBuckets);
1030  
1031              // Fee estimates file parsed correctly
1032              // Copy buckets from file and refresh our bucketmap
1033              buckets = fileBuckets;
1034              bucketMap.clear();
1035              for (unsigned int i = 0; i < buckets.size(); i++) {
1036                  bucketMap[buckets[i]] = i;
1037              }
1038  
1039              // Destroy old TxConfirmStats and point to new ones that already reference buckets and bucketMap
1040              feeStats = std::move(fileFeeStats);
1041              shortStats = std::move(fileShortStats);
1042              longStats = std::move(fileLongStats);
1043  
1044              nBestSeenHeight = nFileBestSeenHeight;
1045              historicalFirst = nFileHistoricalFirst;
1046              historicalBest = nFileHistoricalBest;
1047          }
1048      }
1049      catch (const std::exception& e) {
1050          LogWarning("Unable to read policy estimator data (non-fatal): %s", e.what());
1051          return false;
1052      }
1053      return true;
1054  }
1055  
1056  void CBlockPolicyEstimator::FlushUnconfirmed()
1057  {
1058      const auto startclear{SteadyClock::now()};
1059      LOCK(m_cs_fee_estimator);
1060      size_t num_entries = mapMemPoolTxs.size();
1061      // Remove every entry in mapMemPoolTxs
1062      while (!mapMemPoolTxs.empty()) {
1063          auto mi = mapMemPoolTxs.begin();
1064          _removeTx(mi->first, false); // this calls erase() on mapMemPoolTxs
1065      }
1066      const auto endclear{SteadyClock::now()};
1067      LogDebug(BCLog::ESTIMATEFEE, "Recorded %u unconfirmed txs from mempool in %.3fs\n", num_entries, Ticks<SecondsDouble>(endclear - startclear));
1068  }
1069  
1070  std::chrono::hours CBlockPolicyEstimator::GetFeeEstimatorFileAge()
1071  {
1072      auto file_time{fs::last_write_time(m_estimation_filepath)};
1073      auto now{fs::file_time_type::clock::now()};
1074      return std::chrono::duration_cast<std::chrono::hours>(now - file_time);
1075  }
1076  
1077  static std::set<double> MakeFeeSet(const CFeeRate& min_incremental_fee,
1078                                     double max_filter_fee_rate,
1079                                     double fee_filter_spacing)
1080  {
1081      std::set<double> fee_set;
1082  
1083      const CAmount min_fee_limit{std::max(CAmount(1), min_incremental_fee.GetFeePerK() / 2)};
1084      fee_set.insert(0);
1085      for (double bucket_boundary = min_fee_limit;
1086           bucket_boundary <= max_filter_fee_rate;
1087           bucket_boundary *= fee_filter_spacing) {
1088  
1089          fee_set.insert(bucket_boundary);
1090      }
1091  
1092      return fee_set;
1093  }
1094  
1095  FeeFilterRounder::FeeFilterRounder(const CFeeRate& minIncrementalFee, FastRandomContext& rng)
1096      : m_fee_set{MakeFeeSet(minIncrementalFee, MAX_FILTER_FEERATE, FEE_FILTER_SPACING)},
1097        insecure_rand{rng}
1098  {
1099  }
1100  
1101  CAmount FeeFilterRounder::round(CAmount currentMinFee)
1102  {
1103      AssertLockNotHeld(m_insecure_rand_mutex);
1104      std::set<double>::iterator it = m_fee_set.lower_bound(currentMinFee);
1105      if (it == m_fee_set.end() ||
1106          (it != m_fee_set.begin() &&
1107           WITH_LOCK(m_insecure_rand_mutex, return insecure_rand.rand32()) % 3 != 0)) {
1108          --it;
1109      }
1110      return static_cast<CAmount>(*it);
1111  }
1112