1 #!/usr/bin/env python3
2 # Copyright (c) 2013-present The Bitcoin Core developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 #
6 # Generate seeds.txt from Pieter's DNS seeder
7 #
8 9 import argparse
10 import collections
11 import ipaddress
12 from pathlib import Path
13 import random
14 import re
15 import sys
16 from typing import Union
17 18 asmap_dir = Path(__file__).parent.parent / "asmap"
19 sys.path.append(str(asmap_dir))
20 from asmap import ASMap, net_to_prefix # noqa: E402
21 22 NSEEDS=512
23 24 MAX_SEEDS_PER_ASN = {
25 'ipv4': 2,
26 'ipv6': 10,
27 }
28 29 MIN_BLOCKS = 910000
30 31 PATTERN_IPV4 = re.compile(r"^(([0-2]?\d{1,2})\.([0-2]?\d{1,2})\.([0-2]?\d{1,2})\.([0-2]?\d{1,2})):(\d{1,5})$")
32 PATTERN_IPV6 = re.compile(r"^\[([\da-f:]+)]:(\d{1,5})$", re.IGNORECASE)
33 PATTERN_ONION = re.compile(r"^([a-z2-7]{56}\.onion):(\d+)$")
34 PATTERN_I2P = re.compile(r"^([a-z2-7]{52}\.b32\.i2p):(\d{1,5})$")
35 PATTERN_AGENT = re.compile(
36 r"^/Satoshi:("
37 r"0\.14\.(0|1|2|3|99)"
38 r"|0\.15\.(0|1|2|99)"
39 r"|0\.16\.(0|1|2|3|99)"
40 r"|0\.17\.(0|0\.1|1|2|99)"
41 r"|0\.18\.(0|1|99)"
42 r"|0\.19\.(0|1|2|99)"
43 r"|0\.20\.(0|1|2|99)"
44 r"|0\.21\.(0|1|2|99)"
45 r"|22\.(0|1|99)\.0"
46 r"|23\.(0|1|2|99)\.0"
47 r"|24\.(0|1|2|99)\.(0|1)"
48 r"|25\.(0|1|2|99)\.0"
49 r"|26\.(0|1|2|99)\.0"
50 r"|27\.(0|1|2|99)\.0"
51 r"|28\.(0|1|2|3|4|99)\.0"
52 r"|29\.(0|1|2|3|99)\.0"
53 r"|30\.(0|1|2|99)\.0"
54 r")")
55 56 def parseline(line: str) -> Union[dict, None]:
57 """ Parses a line from `seeds_main.txt` into a dictionary of details for that line.
58 or `None`, if the line could not be parsed.
59 """
60 if line.startswith('#'):
61 # Ignore line that starts with comment
62 return None
63 sline = line.split()
64 if len(sline) < 11:
65 # line too short to be valid, skip it.
66 return None
67 # Skip bad results.
68 if int(sline[1]) == 0:
69 return None
70 m = PATTERN_IPV4.match(sline[0])
71 sortkey = None
72 ip = None
73 if m is None:
74 m = PATTERN_IPV6.match(sline[0])
75 if m is None:
76 m = PATTERN_ONION.match(sline[0])
77 if m is None:
78 m = PATTERN_I2P.match(sline[0])
79 if m is None:
80 return None
81 else:
82 net = 'i2p'
83 ipstr = sortkey = m.group(1)
84 port = int(m.group(2))
85 else:
86 net = 'onion'
87 ipstr = sortkey = m.group(1)
88 port = int(m.group(2))
89 else:
90 net = 'ipv6'
91 if m.group(1) in ['::']: # Not interested in localhost
92 return None
93 ipstr = m.group(1)
94 if ipstr.startswith("fc"): # cjdns looks like ipv6 but always begins with fc
95 net = "cjdns"
96 sortkey = ipstr # XXX parse IPv6 into number, could use name_to_ipv6 from generate-seeds
97 port = int(m.group(2))
98 else:
99 # Do IPv4 sanity check
100 ip = 0
101 for i in range(0,4):
102 if int(m.group(i+2)) < 0 or int(m.group(i+2)) > 255:
103 return None
104 ip = ip + (int(m.group(i+2)) << (8*(3-i)))
105 if ip == 0:
106 return None
107 net = 'ipv4'
108 sortkey = ip
109 ipstr = m.group(1)
110 port = int(m.group(6))
111 # Extract uptime %.
112 uptime30 = float(sline[7][:-1])
113 # Extract Unix timestamp of last success.
114 lastsuccess = int(sline[2])
115 # Extract protocol version.
116 version = int(sline[10])
117 # Extract user agent.
118 agent = sline[11][1:-1]
119 # Extract service flags.
120 service = int(sline[9], 16)
121 # Extract blocks.
122 blocks = int(sline[8])
123 # Construct result.
124 return {
125 'net': net,
126 'ip': ipstr,
127 'port': port,
128 'ipnum': ip,
129 'uptime': uptime30,
130 'lastsuccess': lastsuccess,
131 'version': version,
132 'agent': agent,
133 'service': service,
134 'blocks': blocks,
135 'sortkey': sortkey,
136 }
137 138 def dedup(ips: list[dict]) -> list[dict]:
139 """ Remove duplicates from `ips` where multiple ips share address and port. """
140 d = {}
141 for ip in ips:
142 ip_port = (ip["ip"], ip["port"])
143 if ip_port not in d or ip["lastsuccess"] > d[ip_port]["lastsuccess"]:
144 d[ip_port] = ip
145 return list(d.values())
146 147 def filtermultiport(ips: list[dict]) -> list[dict]:
148 """ Filter out hosts with more nodes per IP"""
149 hist = collections.defaultdict(list)
150 for ip in ips:
151 hist[ip['sortkey']].append(ip)
152 return [value[0] for (key,value) in list(hist.items()) if len(value)==1]
153 154 # Based on Greg Maxwell's seed_filter.py
155 def filterbyasn(asmap: ASMap, ips: list[dict], max_per_asn: dict, max_per_net: int) -> list[dict]:
156 """ Prunes `ips` by
157 (a) trimming ips to have at most `max_per_net` ips from each net (e.g. ipv4, ipv6); and
158 (b) trimming ips to have at most `max_per_asn` ips from each asn in each net.
159 """
160 # Sift out ips by type
161 ips_ipv46 = [ip for ip in ips if ip['net'] in ['ipv4', 'ipv6']]
162 ips_onion = [ip for ip in ips if ip['net'] == 'onion']
163 ips_i2p = [ip for ip in ips if ip['net'] == 'i2p']
164 ips_cjdns = [ip for ip in ips if ip["net"] == "cjdns"]
165 166 # Filter IPv46 by ASN, and limit to max_per_net per network
167 result = []
168 net_count: dict[str, int] = collections.defaultdict(int)
169 asn_count: dict[int, int] = collections.defaultdict(int)
170 171 for i, ip in enumerate(ips_ipv46):
172 if net_count[ip['net']] == max_per_net:
173 # do not add this ip as we already too many
174 # ips from this network
175 continue
176 asn = asmap.lookup(net_to_prefix(ipaddress.ip_network(ip['ip'])))
177 if not asn or asn_count[ip['net'], asn] == max_per_asn[ip['net']]:
178 # do not add this ip as we already have too many
179 # ips from this ASN on this network
180 continue
181 asn_count[ip['net'], asn] += 1
182 net_count[ip['net']] += 1
183 ip['asn'] = asn
184 result.append(ip)
185 186 # Add back Onions (up to max_per_net)
187 result.extend(ips_onion[0:max_per_net])
188 result.extend(ips_i2p[0:max_per_net])
189 result.extend(ips_cjdns[0:max_per_net])
190 return result
191 192 def ip_stats(ips: list[dict]) -> str:
193 """ Format and return pretty string from `ips`. """
194 hist: dict[str, int] = collections.defaultdict(int)
195 for ip in ips:
196 if ip is not None:
197 hist[ip['net']] += 1
198 199 return f"{hist['ipv4']:6d} {hist['ipv6']:6d} {hist['onion']:6d} {hist['i2p']:6d} {hist['cjdns']:6d}"
200 201 def parse_args():
202 argparser = argparse.ArgumentParser(description='Generate a list of bitcoin node seed ip addresses.')
203 argparser.add_argument("-a","--asmap", help='the location of the asmap asn database file (required)', required=True)
204 argparser.add_argument("-s","--seeds", help='the location of the DNS seeds file (required)', required=True)
205 argparser.add_argument("-m", "--minblocks", help="The minimum number of blocks each node must have", default=MIN_BLOCKS, type=int)
206 return argparser.parse_args()
207 208 def main():
209 args = parse_args()
210 211 print(f'Loading asmap database "{args.asmap}"…', end='', file=sys.stderr, flush=True)
212 with open(args.asmap, 'rb') as f:
213 asmap = ASMap.from_binary(f.read())
214 print('Done.', file=sys.stderr)
215 216 print('Loading and parsing DNS seeds…', end='', file=sys.stderr, flush=True)
217 with open(args.seeds, 'r') as f:
218 lines = f.readlines()
219 ips = [parseline(line) for line in lines]
220 random.shuffle(ips)
221 print('Done.', file=sys.stderr)
222 223 print('\x1b[7m IPv4 IPv6 Onion I2P CJDNS Pass \x1b[0m', file=sys.stderr)
224 print(f'{ip_stats(ips):s} Initial', file=sys.stderr)
225 # Skip entries with invalid address.
226 ips = [ip for ip in ips if ip is not None]
227 print(f'{ip_stats(ips):s} Skip entries with invalid address', file=sys.stderr)
228 # Skip duplicates (in case multiple seeds files were concatenated)
229 ips = dedup(ips)
230 print(f'{ip_stats(ips):s} After removing duplicates', file=sys.stderr)
231 # Enforce minimal number of blocks.
232 ips = [ip for ip in ips if ip['blocks'] >= args.minblocks]
233 print(f'{ip_stats(ips):s} Enforce minimal number of blocks', file=sys.stderr)
234 # Require service bit 1.
235 ips = [ip for ip in ips if (ip['service'] & 1) == 1]
236 print(f'{ip_stats(ips):s} Require service bit 1', file=sys.stderr)
237 # Require at least 50% 30-day uptime for clearnet, onion and i2p; 10% for cjdns
238 req_uptime = {
239 'ipv4': 50,
240 'ipv6': 50,
241 'onion': 50,
242 'i2p': 50,
243 'cjdns': 10,
244 }
245 ips = [ip for ip in ips if ip['uptime'] > req_uptime[ip['net']]]
246 print(f'{ip_stats(ips):s} Require minimum uptime', file=sys.stderr)
247 # Require a known and recent user agent.
248 ips = [ip for ip in ips if PATTERN_AGENT.match(ip['agent'])]
249 print(f'{ip_stats(ips):s} Require a known and recent user agent', file=sys.stderr)
250 # Sort by availability (and use last success as tie breaker)
251 ips.sort(key=lambda x: (x['uptime'], x['lastsuccess'], x['ip']), reverse=True)
252 # Filter out hosts with multiple bitcoin ports, these are likely abusive
253 ips = filtermultiport(ips)
254 print(f'{ip_stats(ips):s} Filter out hosts with multiple bitcoin ports', file=sys.stderr)
255 # Look up ASNs and limit results, both per ASN and globally.
256 ips = filterbyasn(asmap, ips, MAX_SEEDS_PER_ASN, NSEEDS)
257 print(f'{ip_stats(ips):s} Look up ASNs and limit results per ASN and per net', file=sys.stderr)
258 # Sort the results by IP address (for deterministic output).
259 ips.sort(key=lambda x: (x['net'], x['sortkey']))
260 for ip in ips:
261 if ip['net'] == 'ipv6' or ip["net"] == "cjdns":
262 print(f"[{ip['ip']}]:{ip['port']}", end="")
263 else:
264 print(f"{ip['ip']}:{ip['port']}", end="")
265 if 'asn' in ip:
266 print(f" # AS{ip['asn']}", end="")
267 print()
268 269 if __name__ == '__main__':
270 main()
271