#!/usr/bin/env python3 """Watch a gateway over the public relay and put it through its paces. Every minute: discovery (is it announcing?) and a tunnel probe (is it actually serving?). Every fifteen minutes, when it is up: a sync from the fork checkpoint through it, which is the same path a wallet takes, and record how much it delivered. Everything lands in watch.jsonl (one record per check) and watch.log (one line per check), so a gap in either file is itself evidence. usage: watch-gateway.py --gateway KEY [--gateway KEY ...] [--relay URL] [--dir DIR] """ import argparse import datetime import json import os import subprocess import threading import time PROBE = os.path.expanduser("~/.local/bin/nostrino-probe") DISCOVER = os.path.expanduser("~/.local/bin/nostrino-discover") NEUTRINO = os.path.expanduser("~/.local/bin/nostrino-neutrino") SOURCE_PROFILE = os.path.expanduser("~/.local/state/gateway-watch/blake.json") PROFILE = SOURCE_PROFILE # the fork profile under the chain name the gateways serve def now(): return datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds") def run(cmd, timeout): try: out = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) return out.stdout.strip(), out.stderr.strip(), out.returncode except subprocess.TimeoutExpired: return "", "timeout", 124 def discovery(relay): """Returns {pubkey: {age_seconds, tips, uri}} for every gateway the relay lists.""" out, err, code = run([DISCOVER, "-relay", relay, "-all"], 90) if code != 0: return {}, err found = {} current = None for line in out.splitlines(): if line.startswith("nostr+chaindata://"): current = {"uri": line, "age_seconds": None, "tips": {}} elif "back end pubkey:" in line and current is not None: found[line.split(":", 1)[1].strip()] = current current = None for line in out.splitlines(): if "updated=" in line and "nostr+chaindata://" in line: stamp = line.split("updated=", 1)[1].split()[0] try: when = datetime.datetime.fromisoformat(stamp) key = None # The pubkey is on the next line, so attribute by URI order. uri = line.split()[0] for pub, info in found.items(): if info["uri"].split()[0] == uri: key = pub if key: found[key]["age_seconds"] = ( datetime.datetime.now(datetime.timezone.utc) - when).total_seconds() except ValueError: pass for token in line.split(): if ":" in token and token.split(":", 1)[0] in ("btc", "blake", "xbt"): chain, height = token.split(":", 1) for info in found.values(): if info["uri"].split()[0] == line.split()[0]: info["tips"][chain] = height return found, "" def probe(relay, gateway, chain): out, err, code = run([PROBE, "-relay", relay, "-gateway", gateway, "-chain", chain], 45) if not out: return {"ok": False, "error": err or "no output"} try: return json.loads(out.splitlines()[-1]) except json.JSONDecodeError: return {"ok": False, "error": out[:200]} def pace(gateway, relay, directory): """Run the wallet's own path through the gateway: the fork from its checkpoint.""" store = os.path.join(directory, "pace-store") subprocess.run(["rm", "-rf", store], capture_output=True) started = time.time() out, err, code = run([ NEUTRINO, "-gateway", gateway, "-relay", relay, "-profile", PROFILE, "-store", store, "-timeout", "180s", ], 200) summary = "" for line in (out + "\n" + err).splitlines(): if line.startswith("added:") or line.startswith("tip:") or "neutrino:" in line: summary = line.strip() subprocess.run(["rm", "-rf", store], capture_output=True) return {"ok": code == 0, "seconds": round(time.time() - started, 1), "summary": summary, "code": code} def main(): parser = argparse.ArgumentParser() parser.add_argument("--gateway", action="append", required=True) parser.add_argument("--relay", default="wss://swaprelay.smesh.lol") parser.add_argument("--dir", default=os.path.expanduser("~/.local/state/gateway-watch")) parser.add_argument("--interval", type=int, default=60) parser.add_argument("--pace-every", type=int, default=900) args = parser.parse_args() os.makedirs(args.dir, exist_ok=True) # The profile is regenerated by the caller into ~/.local/state; nothing here depends on /tmp, # which a tmpfiles cleanup can empty in the middle of the night. if not os.path.exists(PROFILE): say(f"# missing {PROFILE}: run nostrino-profile -chain blake -out there, with name blake") jsonl = open(os.path.join(args.dir, "watch.jsonl"), "a", buffering=1) human = open(os.path.join(args.dir, "watch.log"), "a", buffering=1) def say(text): human.write(text + "\n") print(text, flush=True) say(f"# watch started {now()} relay={args.relay} gateways={args.gateway}") last_pace = 0.0 pacing_lock = threading.Lock() def pace_in_background(gateways): # Pacing takes minutes; the one-minute check cadence must not wait for it. def run(): pace_record = {"ts": now(), "pace": {}} pace_lines = [] for gateway in gateways: if not gateways[gateway]: pace_lines.append(f"pace {gateway[:8]} skipped: not serving") continue pacing = pace(gateway, args.relay, args.dir) pace_record["pace"][gateway] = pacing pace_lines.append( f"pace {gateway[:8]} {pacing['seconds']}s ok={pacing['ok']} {pacing['summary'][:150]}") jsonl.write(json.dumps(pace_record) + "\n") say(f"{now()} " + " | ".join(pace_lines)) with pacing_lock: pass if pacing_lock.locked(): return threading.Thread(target=run, daemon=True).start() while True: found, discovery_err = discovery(args.relay) record = {"ts": now(), "relay": args.relay, "gateways": {}, "discovery_error": discovery_err} lines = [] for gateway in args.gateway: info = found.get(gateway, {}) blake = probe(args.relay, gateway, "blake") btc = probe(args.relay, gateway, "btc") entry = { "announced_age_s": info.get("age_seconds"), "announced_tips": info.get("tips", {}), "blake": blake, "btc": btc, "online": blake.get("ok", False) or btc.get("ok", False), } record["gateways"][gateway] = entry lines.append( f"{gateway[:8]} " f"announced={entry['announced_age_s'] if entry['announced_age_s'] is None else int(entry['announced_age_s'])}s " f"blake={'ok tip %s' % blake.get('tip') if blake.get('ok') else 'FAIL %s' % blake.get('error', '')[:60]} " f"btc={'ok tip %s' % btc.get('tip') if btc.get('ok') else 'FAIL %s' % btc.get('error', '')[:60]}" ) # The check line is written before any pacing, so the cadence of the file never depends on # how long a sync takes. jsonl.write(json.dumps(record) + "\n") say(f"{now()} " + " | ".join(lines)) # Pacing is a sync from the checkpoint through each serving gateway: the path a wallet # takes. It runs on its own thread so the check cadence stays at one line a minute. if time.time() - last_pace >= args.pace_every and not pacing_lock.locked(): last_pace = time.time() online = {g: record["gateways"][g]["online"] for g in args.gateway} with pacing_lock: pace_in_background(online) time.sleep(args.interval) main()