1 #!/usr/bin/env python3
2 """Watch a gateway over the public relay and put it through its paces.
3 4 Every minute: discovery (is it announcing?) and a tunnel probe (is it actually serving?). Every
5 fifteen minutes, when it is up: a sync from the fork checkpoint through it, which is the same path
6 a wallet takes, and record how much it delivered.
7 8 Everything lands in watch.jsonl (one record per check) and watch.log (one line per check), so a gap
9 in either file is itself evidence.
10 11 usage: watch-gateway.py --gateway KEY [--gateway KEY ...] [--relay URL] [--dir DIR]
12 """
13 14 import argparse
15 import datetime
16 import json
17 import os
18 import subprocess
19 import threading
20 import time
21 22 PROBE = os.path.expanduser("~/.local/bin/nostrino-probe")
23 DISCOVER = os.path.expanduser("~/.local/bin/nostrino-discover")
24 NEUTRINO = os.path.expanduser("~/.local/bin/nostrino-neutrino")
25 SOURCE_PROFILE = os.path.expanduser("~/.local/state/gateway-watch/blake.json")
26 PROFILE = SOURCE_PROFILE # the fork profile under the chain name the gateways serve
27 28 29 def now():
30 return datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds")
31 32 33 def run(cmd, timeout):
34 try:
35 out = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
36 return out.stdout.strip(), out.stderr.strip(), out.returncode
37 except subprocess.TimeoutExpired:
38 return "", "timeout", 124
39 40 41 def discovery(relay):
42 """Returns {pubkey: {age_seconds, tips, uri}} for every gateway the relay lists."""
43 out, err, code = run([DISCOVER, "-relay", relay, "-all"], 90)
44 45 if code != 0:
46 return {}, err
47 48 found = {}
49 current = None
50 51 for line in out.splitlines():
52 if line.startswith("nostr+chaindata://"):
53 current = {"uri": line, "age_seconds": None, "tips": {}}
54 elif "back end pubkey:" in line and current is not None:
55 found[line.split(":", 1)[1].strip()] = current
56 current = None
57 58 for line in out.splitlines():
59 if "updated=" in line and "nostr+chaindata://" in line:
60 stamp = line.split("updated=", 1)[1].split()[0]
61 try:
62 when = datetime.datetime.fromisoformat(stamp)
63 key = None
64 65 # The pubkey is on the next line, so attribute by URI order.
66 uri = line.split()[0]
67 for pub, info in found.items():
68 if info["uri"].split()[0] == uri:
69 key = pub
70 if key:
71 found[key]["age_seconds"] = (
72 datetime.datetime.now(datetime.timezone.utc) - when).total_seconds()
73 except ValueError:
74 pass
75 76 for token in line.split():
77 if ":" in token and token.split(":", 1)[0] in ("btc", "blake", "xbt"):
78 chain, height = token.split(":", 1)
79 for info in found.values():
80 if info["uri"].split()[0] == line.split()[0]:
81 info["tips"][chain] = height
82 83 return found, ""
84 85 86 def probe(relay, gateway, chain):
87 out, err, code = run([PROBE, "-relay", relay, "-gateway", gateway, "-chain", chain], 45)
88 89 if not out:
90 return {"ok": False, "error": err or "no output"}
91 92 try:
93 return json.loads(out.splitlines()[-1])
94 except json.JSONDecodeError:
95 return {"ok": False, "error": out[:200]}
96 97 98 def pace(gateway, relay, directory):
99 """Run the wallet's own path through the gateway: the fork from its checkpoint."""
100 store = os.path.join(directory, "pace-store")
101 subprocess.run(["rm", "-rf", store], capture_output=True)
102 103 started = time.time()
104 out, err, code = run([
105 NEUTRINO, "-gateway", gateway, "-relay", relay, "-profile", PROFILE,
106 "-store", store, "-timeout", "180s",
107 ], 200)
108 109 summary = ""
110 for line in (out + "\n" + err).splitlines():
111 if line.startswith("added:") or line.startswith("tip:") or "neutrino:" in line:
112 summary = line.strip()
113 114 subprocess.run(["rm", "-rf", store], capture_output=True)
115 116 return {"ok": code == 0, "seconds": round(time.time() - started, 1),
117 "summary": summary, "code": code}
118 119 120 def main():
121 parser = argparse.ArgumentParser()
122 parser.add_argument("--gateway", action="append", required=True)
123 parser.add_argument("--relay", default="wss://swaprelay.smesh.lol")
124 parser.add_argument("--dir", default=os.path.expanduser("~/.local/state/gateway-watch"))
125 parser.add_argument("--interval", type=int, default=60)
126 parser.add_argument("--pace-every", type=int, default=900)
127 args = parser.parse_args()
128 129 os.makedirs(args.dir, exist_ok=True)
130 131 # The profile is regenerated by the caller into ~/.local/state; nothing here depends on /tmp,
132 # which a tmpfiles cleanup can empty in the middle of the night.
133 if not os.path.exists(PROFILE):
134 say(f"# missing {PROFILE}: run nostrino-profile -chain blake -out there, with name blake")
135 jsonl = open(os.path.join(args.dir, "watch.jsonl"), "a", buffering=1)
136 human = open(os.path.join(args.dir, "watch.log"), "a", buffering=1)
137 138 def say(text):
139 human.write(text + "\n")
140 print(text, flush=True)
141 142 say(f"# watch started {now()} relay={args.relay} gateways={args.gateway}")
143 144 last_pace = 0.0
145 pacing_lock = threading.Lock()
146 147 def pace_in_background(gateways):
148 # Pacing takes minutes; the one-minute check cadence must not wait for it.
149 def run():
150 pace_record = {"ts": now(), "pace": {}}
151 pace_lines = []
152 153 for gateway in gateways:
154 if not gateways[gateway]:
155 pace_lines.append(f"pace {gateway[:8]} skipped: not serving")
156 continue
157 158 pacing = pace(gateway, args.relay, args.dir)
159 pace_record["pace"][gateway] = pacing
160 pace_lines.append(
161 f"pace {gateway[:8]} {pacing['seconds']}s ok={pacing['ok']} {pacing['summary'][:150]}")
162 163 jsonl.write(json.dumps(pace_record) + "\n")
164 say(f"{now()} " + " | ".join(pace_lines))
165 166 with pacing_lock:
167 pass
168 169 if pacing_lock.locked():
170 return
171 172 threading.Thread(target=run, daemon=True).start()
173 174 while True:
175 found, discovery_err = discovery(args.relay)
176 record = {"ts": now(), "relay": args.relay, "gateways": {}, "discovery_error": discovery_err}
177 lines = []
178 179 for gateway in args.gateway:
180 info = found.get(gateway, {})
181 blake = probe(args.relay, gateway, "blake")
182 btc = probe(args.relay, gateway, "btc")
183 184 entry = {
185 "announced_age_s": info.get("age_seconds"),
186 "announced_tips": info.get("tips", {}),
187 "blake": blake,
188 "btc": btc,
189 "online": blake.get("ok", False) or btc.get("ok", False),
190 }
191 record["gateways"][gateway] = entry
192 193 lines.append(
194 f"{gateway[:8]} "
195 f"announced={entry['announced_age_s'] if entry['announced_age_s'] is None else int(entry['announced_age_s'])}s "
196 f"blake={'ok tip %s' % blake.get('tip') if blake.get('ok') else 'FAIL %s' % blake.get('error', '')[:60]} "
197 f"btc={'ok tip %s' % btc.get('tip') if btc.get('ok') else 'FAIL %s' % btc.get('error', '')[:60]}"
198 )
199 200 # The check line is written before any pacing, so the cadence of the file never depends on
201 # how long a sync takes.
202 jsonl.write(json.dumps(record) + "\n")
203 say(f"{now()} " + " | ".join(lines))
204 205 # Pacing is a sync from the checkpoint through each serving gateway: the path a wallet
206 # takes. It runs on its own thread so the check cadence stays at one line a minute.
207 if time.time() - last_pace >= args.pace_every and not pacing_lock.locked():
208 last_pace = time.time()
209 online = {g: record["gateways"][g]["online"] for g in args.gateway}
210 211 with pacing_lock:
212 pace_in_background(online)
213 214 time.sleep(args.interval)
215 216 217 main()
218