watch-gateway.py raw

   1  #!/usr/bin/env python3
   2  """Watch a gateway over the public relay and put it through its paces.
   3  
   4  Every minute: discovery (is it announcing?) and a tunnel probe (is it actually serving?). Every
   5  fifteen minutes, when it is up: a sync from the fork checkpoint through it, which is the same path
   6  a wallet takes, and record how much it delivered.
   7  
   8  Everything lands in watch.jsonl (one record per check) and watch.log (one line per check), so a gap
   9  in either file is itself evidence.
  10  
  11  usage: watch-gateway.py --gateway KEY [--gateway KEY ...] [--relay URL] [--dir DIR]
  12  """
  13  
  14  import argparse
  15  import datetime
  16  import json
  17  import os
  18  import subprocess
  19  import threading
  20  import time
  21  
  22  PROBE = os.path.expanduser("~/.local/bin/nostrino-probe")
  23  DISCOVER = os.path.expanduser("~/.local/bin/nostrino-discover")
  24  NEUTRINO = os.path.expanduser("~/.local/bin/nostrino-neutrino")
  25  SOURCE_PROFILE = os.path.expanduser("~/.local/state/gateway-watch/blake.json")
  26  PROFILE = SOURCE_PROFILE   # the fork profile under the chain name the gateways serve
  27  
  28  
  29  def now():
  30      return datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds")
  31  
  32  
  33  def run(cmd, timeout):
  34      try:
  35          out = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
  36          return out.stdout.strip(), out.stderr.strip(), out.returncode
  37      except subprocess.TimeoutExpired:
  38          return "", "timeout", 124
  39  
  40  
  41  def discovery(relay):
  42      """Returns {pubkey: {age_seconds, tips, uri}} for every gateway the relay lists."""
  43      out, err, code = run([DISCOVER, "-relay", relay, "-all"], 90)
  44  
  45      if code != 0:
  46          return {}, err
  47  
  48      found = {}
  49      current = None
  50  
  51      for line in out.splitlines():
  52          if line.startswith("nostr+chaindata://"):
  53              current = {"uri": line, "age_seconds": None, "tips": {}}
  54          elif "back end pubkey:" in line and current is not None:
  55              found[line.split(":", 1)[1].strip()] = current
  56              current = None
  57  
  58      for line in out.splitlines():
  59          if "updated=" in line and "nostr+chaindata://" in line:
  60              stamp = line.split("updated=", 1)[1].split()[0]
  61              try:
  62                  when = datetime.datetime.fromisoformat(stamp)
  63                  key = None
  64  
  65                  # The pubkey is on the next line, so attribute by URI order.
  66                  uri = line.split()[0]
  67                  for pub, info in found.items():
  68                      if info["uri"].split()[0] == uri:
  69                          key = pub
  70                  if key:
  71                      found[key]["age_seconds"] = (
  72                          datetime.datetime.now(datetime.timezone.utc) - when).total_seconds()
  73              except ValueError:
  74                  pass
  75  
  76              for token in line.split():
  77                  if ":" in token and token.split(":", 1)[0] in ("btc", "blake", "xbt"):
  78                      chain, height = token.split(":", 1)
  79                      for info in found.values():
  80                          if info["uri"].split()[0] == line.split()[0]:
  81                              info["tips"][chain] = height
  82  
  83      return found, ""
  84  
  85  
  86  def probe(relay, gateway, chain):
  87      out, err, code = run([PROBE, "-relay", relay, "-gateway", gateway, "-chain", chain], 45)
  88  
  89      if not out:
  90          return {"ok": False, "error": err or "no output"}
  91  
  92      try:
  93          return json.loads(out.splitlines()[-1])
  94      except json.JSONDecodeError:
  95          return {"ok": False, "error": out[:200]}
  96  
  97  
  98  def pace(gateway, relay, directory):
  99      """Run the wallet's own path through the gateway: the fork from its checkpoint."""
 100      store = os.path.join(directory, "pace-store")
 101      subprocess.run(["rm", "-rf", store], capture_output=True)
 102  
 103      started = time.time()
 104      out, err, code = run([
 105          NEUTRINO, "-gateway", gateway, "-relay", relay, "-profile", PROFILE,
 106          "-store", store, "-timeout", "180s",
 107      ], 200)
 108  
 109      summary = ""
 110      for line in (out + "\n" + err).splitlines():
 111          if line.startswith("added:") or line.startswith("tip:") or "neutrino:" in line:
 112              summary = line.strip()
 113  
 114      subprocess.run(["rm", "-rf", store], capture_output=True)
 115  
 116      return {"ok": code == 0, "seconds": round(time.time() - started, 1),
 117              "summary": summary, "code": code}
 118  
 119  
 120  def main():
 121      parser = argparse.ArgumentParser()
 122      parser.add_argument("--gateway", action="append", required=True)
 123      parser.add_argument("--relay", default="wss://swaprelay.smesh.lol")
 124      parser.add_argument("--dir", default=os.path.expanduser("~/.local/state/gateway-watch"))
 125      parser.add_argument("--interval", type=int, default=60)
 126      parser.add_argument("--pace-every", type=int, default=900)
 127      args = parser.parse_args()
 128  
 129      os.makedirs(args.dir, exist_ok=True)
 130  
 131      # The profile is regenerated by the caller into ~/.local/state; nothing here depends on /tmp,
 132      # which a tmpfiles cleanup can empty in the middle of the night.
 133      if not os.path.exists(PROFILE):
 134          say(f"# missing {PROFILE}: run nostrino-profile -chain blake -out there, with name blake")
 135      jsonl = open(os.path.join(args.dir, "watch.jsonl"), "a", buffering=1)
 136      human = open(os.path.join(args.dir, "watch.log"), "a", buffering=1)
 137  
 138      def say(text):
 139          human.write(text + "\n")
 140          print(text, flush=True)
 141  
 142      say(f"# watch started {now()} relay={args.relay} gateways={args.gateway}")
 143  
 144      last_pace = 0.0
 145      pacing_lock = threading.Lock()
 146  
 147      def pace_in_background(gateways):
 148          # Pacing takes minutes; the one-minute check cadence must not wait for it.
 149          def run():
 150              pace_record = {"ts": now(), "pace": {}}
 151              pace_lines = []
 152  
 153              for gateway in gateways:
 154                  if not gateways[gateway]:
 155                      pace_lines.append(f"pace {gateway[:8]} skipped: not serving")
 156                      continue
 157  
 158                  pacing = pace(gateway, args.relay, args.dir)
 159                  pace_record["pace"][gateway] = pacing
 160                  pace_lines.append(
 161                      f"pace {gateway[:8]} {pacing['seconds']}s ok={pacing['ok']} {pacing['summary'][:150]}")
 162  
 163              jsonl.write(json.dumps(pace_record) + "\n")
 164              say(f"{now()} " + " | ".join(pace_lines))
 165  
 166              with pacing_lock:
 167                  pass
 168  
 169          if pacing_lock.locked():
 170              return
 171  
 172          threading.Thread(target=run, daemon=True).start()
 173  
 174      while True:
 175          found, discovery_err = discovery(args.relay)
 176          record = {"ts": now(), "relay": args.relay, "gateways": {}, "discovery_error": discovery_err}
 177          lines = []
 178  
 179          for gateway in args.gateway:
 180              info = found.get(gateway, {})
 181              blake = probe(args.relay, gateway, "blake")
 182              btc = probe(args.relay, gateway, "btc")
 183  
 184              entry = {
 185                  "announced_age_s": info.get("age_seconds"),
 186                  "announced_tips": info.get("tips", {}),
 187                  "blake": blake,
 188                  "btc": btc,
 189                  "online": blake.get("ok", False) or btc.get("ok", False),
 190              }
 191              record["gateways"][gateway] = entry
 192  
 193              lines.append(
 194                  f"{gateway[:8]} "
 195                  f"announced={entry['announced_age_s'] if entry['announced_age_s'] is None else int(entry['announced_age_s'])}s "
 196                  f"blake={'ok tip %s' % blake.get('tip') if blake.get('ok') else 'FAIL %s' % blake.get('error', '')[:60]} "
 197                  f"btc={'ok tip %s' % btc.get('tip') if btc.get('ok') else 'FAIL %s' % btc.get('error', '')[:60]}"
 198              )
 199  
 200          # The check line is written before any pacing, so the cadence of the file never depends on
 201          # how long a sync takes.
 202          jsonl.write(json.dumps(record) + "\n")
 203          say(f"{now()} " + " | ".join(lines))
 204  
 205          # Pacing is a sync from the checkpoint through each serving gateway: the path a wallet
 206          # takes. It runs on its own thread so the check cadence stays at one line a minute.
 207          if time.time() - last_pace >= args.pace_every and not pacing_lock.locked():
 208              last_pace = time.time()
 209              online = {g: record["gateways"][g]["online"] for g in args.gateway}
 210  
 211              with pacing_lock:
 212                  pace_in_background(online)
 213  
 214          time.sleep(args.interval)
 215  
 216  
 217  main()
 218