1 #!/usr/bin/env python3
2 # Copyright (c) The Bitcoin Core developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 """Test the HTTP server basics."""
6 7 from test_framework.test_framework import BitcoinTestFramework
8 from test_framework.util import assert_equal, str_to_b64str
9 10 import http.client
11 import time
12 import urllib.parse
13 14 # Configuration option for some tests
15 RPCSERVERTIMEOUT = 2
16 # Set in httpserver.h
17 MAX_HEADERS_SIZE = 8192
18 MAX_BODY_SIZE = 32 * 1024 * 1024
19 20 # When a test expects a server disconnection, any of these errors are
21 # acceptable. The specific event is determined by race condition and platform OS.
22 NETWORK_ERRORS = (
23 BrokenPipeError, # write to a closed socket/pipe
24 ConnectionResetError, # connection forcibly closed by peer
25 ConnectionAbortedError, # connection aborted locally or by network stack
26 http.client.ResponseNotReady, # server response not ready or connection out of sync
27 )
28 29 class BitcoinHTTPConnection:
30 def __init__(self, node):
31 self.url = urllib.parse.urlparse(node.url)
32 self.authpair = f'{self.url.username}:{self.url.password}'
33 self.headers = {"Authorization": f"Basic {str_to_b64str(self.authpair)}"}
34 self.reset_conn()
35 36 def reset_conn(self):
37 self.conn = http.client.HTTPConnection(self.url.hostname, self.url.port)
38 self.conn.connect()
39 40 def sock_closed(self):
41 if self.conn.sock is None:
42 return True
43 try:
44 self.conn.request('GET', '/')
45 self.conn.getresponse().read()
46 return False
47 except NETWORK_ERRORS:
48 return True
49 50 def close_sock(self):
51 self.conn.close()
52 53 def set_timeout(self, seconds):
54 self.conn.sock.settimeout(seconds)
55 56 def add_header(self, key, value):
57 self.headers.update({key: value})
58 59 def _request(self, method, path, data, connection_header, **kwargs):
60 headers = self.headers.copy()
61 if connection_header is not None:
62 headers["Connection"] = connection_header
63 self.conn.request(method, path, data, headers, **kwargs)
64 return self.conn.getresponse()
65 66 def post(self, path, data, connection_header=None, **kwargs):
67 return self._request('POST', path, data, connection_header, **kwargs)
68 69 def get(self, path, connection_header=None):
70 return self._request('GET', path, '', connection_header)
71 72 def send_raw(self, data):
73 self.conn.sock.sendall(data)
74 75 def post_raw(self, path, data):
76 data_bytes = data.encode("utf-8")
77 req = f"POST {path} HTTP/1.1\r\n"
78 req += f'Authorization: Basic {str_to_b64str(self.authpair)}\r\n'
79 req += f'Content-Length: {len(data_bytes)}\r\n\r\n'
80 self.send_raw(req.encode("ascii") + data_bytes)
81 82 def recv_raw(self):
83 '''
84 Blocking socket will wait until data is received and return up to 1024 bytes
85 '''
86 return self.conn.sock.recv(1024)
87 88 def expect_timeout(self, seconds):
89 # Wait for response, but expect a timeout disconnection
90 start = time.time()
91 response1 = self.recv_raw()
92 stop = time.time()
93 # Server disconnected with EOF
94 assert_equal(response1, b"")
95 # Server disconnected within an acceptable range of time:
96 # not immediately, and not too far over the configured duration.
97 # This allows for some jitter in the test between client and server.
98 duration = stop - start
99 assert duration <= seconds + 2, f"Server disconnected too slow: {duration} > {seconds}"
100 assert duration >= seconds - 1, f"Server disconnected too fast: {duration} < {seconds}"
101 # The connection is definitely closed.
102 assert self.sock_closed()
103 104 class HTTPBasicsTest (BitcoinTestFramework):
105 def set_test_params(self):
106 self.num_nodes = 1
107 108 def setup_network(self):
109 self.setup_nodes()
110 self.node = self.nodes[0]
111 112 def run_test(self):
113 # The test framework typically reuses a single persistent HTTP connection
114 # for all RPCs to a TestNode. Because we are setting -rpcservertimeout
115 # so low on this one node, its connection will quickly timeout and get dropped by
116 # the server. Negating this setting will force the AuthServiceProxy
117 # for this node to create a fresh new HTTP connection for every command
118 # called for the remainder of this test.
119 self.node.reuse_http_connections = False
120 121 self.check_default_connection()
122 self.check_keepalive_connection()
123 self.check_close_connection()
124 self.check_excessive_request_size()
125 self.check_pipelining()
126 self.check_chunked_transfer()
127 self.check_idle_timeout()
128 self.check_server_busy_idle_timeout()
129 self.check_auth_required()
130 self.check_wrong_credentials()
131 self.check_malformed_auth_headers()
132 self.check_disallowed_http_methods()
133 self.check_path_traversal()
134 self.check_request_smuggling_cl_te()
135 self.check_duplicate_content_length()
136 self.check_null_byte_in_uri()
137 self.check_invalid_http_version()
138 self.check_whitespace_in_headers()
139 140 141 def check_default_connection(self):
142 self.log.info("Checking default HTTP/1.1 connection persistence")
143 conn = BitcoinHTTPConnection(self.node)
144 # Make request without explicit "Connection" header
145 response1 = conn.post('/', '{"method": "getbestblockhash"}').read()
146 assert b'"error":null' in response1
147 # Connection still open after request
148 assert not conn.sock_closed()
149 # Make second request without explicit "Connection" header
150 response2 = conn.post('/', '{"method": "getchaintips"}').read()
151 assert b'"error":null' in response2
152 # Connection still open after second request
153 assert not conn.sock_closed()
154 # Close
155 conn.close_sock()
156 assert conn.sock_closed()
157 158 159 def check_keepalive_connection(self):
160 self.log.info("Checking keep-alive connection persistence")
161 conn = BitcoinHTTPConnection(self.node)
162 # Make request with explicit "Connection: keep-alive" header
163 response1 = conn.post('/', '{"method": "getbestblockhash"}', connection_header='keep-alive').read()
164 assert b'"error":null' in response1
165 # Connection still open after request
166 assert not conn.sock_closed()
167 # Make second request with explicit "Connection" header
168 response2 = conn.post('/', '{"method": "getchaintips"}', connection_header='keep-alive').read()
169 assert b'"error":null' in response2
170 # Connection still open after second request
171 assert not conn.sock_closed()
172 # Close
173 conn.close_sock()
174 assert conn.sock_closed()
175 176 177 def check_close_connection(self):
178 self.log.info("Checking close connection after response")
179 conn = BitcoinHTTPConnection(self.node)
180 # Make request with explicit "Connection: close" header
181 response1 = conn.post('/', '{"method": "getbestblockhash"}', connection_header='close').read()
182 assert b'"error":null' in response1
183 # Connection closed after response
184 assert conn.sock_closed()
185 186 187 def check_excessive_request_size(self):
188 self.log.info("Checking excessive request size")
189 190 # Large URI plus up to 1000 bytes of default headers
191 # added by python's http.client still below total limit.
192 conn = BitcoinHTTPConnection(self.node)
193 response1 = conn.get(f'/{"x" * (MAX_HEADERS_SIZE - 1000)}')
194 assert_equal(response1.status, http.client.NOT_FOUND)
195 196 # Excessive URI size plus default headers breaks the limit.
197 conn = BitcoinHTTPConnection(self.node)
198 response2 = conn.get(f'/{"x" * MAX_HEADERS_SIZE}')
199 assert_equal(response2.status, http.client.BAD_REQUEST)
200 201 # Compute how many short header lines need to be added to http.client
202 # default headers to make / break the total limit in a single request.
203 header_line_length = len("header_0000: foo\r\n")
204 headers_below_limit = (MAX_HEADERS_SIZE - 1000) // header_line_length
205 headers_above_limit = MAX_HEADERS_SIZE // header_line_length
206 207 # Many small header lines is ok
208 conn = BitcoinHTTPConnection(self.node)
209 for i in range(headers_below_limit):
210 conn.add_header(f"header_{i:04}", "foo")
211 response3 = conn.get('/x')
212 assert_equal(response3.status, http.client.NOT_FOUND)
213 214 # Too many small header lines exceeds total headers size allowed
215 conn = BitcoinHTTPConnection(self.node)
216 for i in range(headers_above_limit):
217 conn.add_header(f"header_{i:04}", "foo")
218 response3 = conn.get('/x')
219 assert_equal(response3.status, http.client.BAD_REQUEST)
220 221 # Compute how much data we can add to a request message body
222 # to make / break the limit.
223 base_request_body_size = len('{"jsonrpc": "2.0", "id": "0", "method": "submitblock", "params": [""]}}')
224 bytes_below_limit = MAX_BODY_SIZE - base_request_body_size
225 bytes_above_limit = MAX_BODY_SIZE - base_request_body_size + 2
226 227 # Large request body size is ok
228 conn = BitcoinHTTPConnection(self.node)
229 response4 = conn.post('/', f'{{"jsonrpc": "2.0", "id": "0", "method": "submitblock", "params": ["{"0" * bytes_below_limit}"]}}')
230 assert_equal(response4.status, http.client.OK)
231 232 conn = BitcoinHTTPConnection(self.node)
233 try:
234 # Excessive body size is invalid
235 conn.post_raw('/', f'{{"jsonrpc": "2.0", "id": "0", "method": "submitblock", "params": ["{"0" * bytes_above_limit}"]}}')
236 self.log.info("Client finished sending request before connection was terminated")
237 except NETWORK_ERRORS:
238 self.log.info("Client did not finish sending request before connection was terminated")
239 240 # The server will send a 413 response and disconnect but due to a race
241 # condition, the python client may or may not read the response before
242 # detecting the broken socket (which it may still be trying to write to).
243 try:
244 response5 = conn.conn.getresponse()
245 assert_equal(response5.status, http.client.REQUEST_ENTITY_TOO_LARGE)
246 self.log.info(f"Client got expected response status {response5.status}")
247 assert conn.sock_closed()
248 except NETWORK_ERRORS:
249 self.log.info("Client did not read response before disconnecting")
250 251 252 def check_pipelining(self):
253 """
254 Requests are responded to in the order in which they were received
255 See https://www.rfc-editor.org/rfc/rfc7230#section-6.3.2
256 """
257 self.log.info("Check pipelining")
258 tip_height = self.node.getblockcount()
259 conn = BitcoinHTTPConnection(self.node)
260 conn.set_timeout(5)
261 262 # Send two requests in a row.
263 # The first request will block the second indefinitely
264 conn.post_raw('/', f'{{"method": "waitforblockheight", "params": [{tip_height + 1}]}}')
265 conn.post_raw('/', '{"method": "getblockcount"}')
266 267 try:
268 # The server should not respond to the second request until the first
269 # request has been handled. Since the server will not respond at all
270 # to the first request until we generate a block we expect a socket timeout.
271 conn.recv_raw()
272 assert False
273 except TimeoutError:
274 pass
275 276 # Use a separate http connection to generate a block
277 self.generate(self.node, 1, sync_fun=self.no_op)
278 279 # Wait for two responses to be received
280 res = b""
281 while res.count(b"result") != 2:
282 res += conn.recv_raw()
283 284 # waitforblockheight was responded to first, and then getblockcount
285 # which includes the block added after the request was made
286 chunks = res.split(b'"result":')
287 assert chunks[1].startswith(b'{"hash":')
288 assert chunks[2].startswith(bytes(f'{tip_height + 1}', 'utf8'))
289 290 291 def check_chunked_transfer(self):
292 self.log.info("Check HTTP request encoded with chunked transfer")
293 conn = BitcoinHTTPConnection(self.node)
294 headers_chunked = conn.headers.copy()
295 headers_chunked.update({"Transfer-encoding": "chunked"})
296 body_chunked = [
297 b'{"method": "submitblock", "params": ["',
298 b'0' * 1000000,
299 b'1' * 1000000,
300 b'2' * 1000000,
301 b'3' * 1000000,
302 b'"]}'
303 ]
304 conn.conn.request(
305 method='POST',
306 url='/',
307 body=iter(body_chunked),
308 headers=headers_chunked,
309 encode_chunked=True)
310 response1 = conn.recv_raw()
311 assert b'{"result":"high-hash","error":null}\n' in response1
312 313 self.log.info("Check excessive size HTTP request encoded with chunked transfer")
314 conn = BitcoinHTTPConnection(self.node)
315 headers_chunked = conn.headers.copy()
316 headers_chunked.update({"Transfer-encoding": "chunked"})
317 body_chunked = [
318 b'{"method": "submitblock", "params": ["',
319 b'0' * 10000000,
320 b'1' * 10000000,
321 b'2' * 10000000,
322 b'3' * 10000000,
323 b'"]}'
324 ]
325 try:
326 conn.conn.request(
327 method='POST',
328 url='/',
329 body=iter(body_chunked),
330 headers=headers_chunked,
331 encode_chunked=True)
332 self.log.info("Client finished sending request before connection was terminated")
333 except NETWORK_ERRORS:
334 self.log.info("Client did not finish sending request before connection was terminated")
335 336 # The server will send a 413 response and disconnect but due to a race
337 # condition, the python client may or may not read the response before
338 # detecting the broken socket (which it may still be trying to write to).
339 try:
340 response2 = conn.conn.getresponse()
341 assert_equal(response2.status, http.client.REQUEST_ENTITY_TOO_LARGE)
342 self.log.info(f"Client got expected response status {response2.status}")
343 assert conn.sock_closed()
344 except NETWORK_ERRORS:
345 self.log.info("Client did not read response before disconnecting")
346 347 348 def check_idle_timeout(self):
349 self.log.info("Check -rpcservertimeout")
350 351 # This is the amount of time the server will wait for a client to
352 # send a complete request. Test it by sending an incomplete but
353 # so-far otherwise well-formed HTTP request, and never finishing it.
354 self.restart_node(0, extra_args=[f"-rpcservertimeout={RPCSERVERTIMEOUT}"])
355 356 # Copied from http_incomplete_test_() in regress_http.c in libevent.
357 # A complete request would have an additional "\r\n" at the end.
358 bad_http_request = "GET /test1 HTTP/1.1\r\nHost: somehost\r\n"
359 conn = BitcoinHTTPConnection(self.node)
360 conn.send_raw(bad_http_request.encode("ascii"))
361 362 conn.expect_timeout(RPCSERVERTIMEOUT)
363 364 # Sanity check -- complete requests don't timeout waiting for completion
365 good_http_request = "GET /test2 HTTP/1.1\r\nHost: somehost\r\n\r\n"
366 conn.reset_conn()
367 conn.send_raw(good_http_request.encode("ascii"))
368 response = conn.recv_raw()
369 assert response.startswith(b"HTTP/1.1 404 Not Found")
370 371 # Still open
372 assert not conn.sock_closed()
373 374 375 def check_server_busy_idle_timeout(self):
376 self.log.info("Check that -rpcservertimeout won't close on a delayed response")
377 378 self.restart_node(0, extra_args=[f"-rpcservertimeout={RPCSERVERTIMEOUT}"])
379 380 tip_height = self.node.getblockcount()
381 conn = BitcoinHTTPConnection(self.node)
382 conn.post_raw('/', f'{{"method": "waitforblockheight", "params": [{tip_height + 1}]}}')
383 384 # Wait until after the timeout, then generate a block with a second HTTP connection
385 time.sleep(RPCSERVERTIMEOUT + 1)
386 generated_block = self.generate(self.node, 1, sync_fun=self.no_op)[0]
387 388 # The first connection gets the response it is patiently waiting for
389 response1 = conn.recv_raw().decode()
390 assert generated_block in response1
391 # The connection is still open
392 assert not conn.sock_closed()
393 394 # Now it will actually close due to idle timeout
395 conn.expect_timeout(RPCSERVERTIMEOUT)
396 397 398 def check_auth_required(self):
399 self.log.info("Check that requests without credentials return 401 Unauthorized with WWW-Authenticate")
400 conn = BitcoinHTTPConnection(self.node)
401 conn.headers = {}
402 response = conn.post('/', '{"method": "getbestblockhash"}')
403 assert_equal(response.status, http.client.UNAUTHORIZED)
404 assert response.getheader('WWW-Authenticate') is not None
405 406 407 def check_wrong_credentials(self):
408 self.log.info("Check that incorrect credentials return 401 Unauthorized")
409 conn = BitcoinHTTPConnection(self.node)
410 wrong_pair = f"{conn.url.username}:wrong_password"
411 conn.headers = {"Authorization": f"Basic {str_to_b64str(wrong_pair)}"}
412 response = conn.post('/', '{"method": "getbestblockhash"}')
413 assert_equal(response.status, http.client.UNAUTHORIZED)
414 assert response.getheader('WWW-Authenticate') is not None
415 416 417 def check_malformed_auth_headers(self):
418 self.log.info("Check that malformed Authorization headers return 401 Unauthorized")
419 cases = [
420 "Bearer sometoken123",
421 'Digest username="user", realm="test"',
422 "Basic !!!notbase64!!!",
423 f"Basic {str_to_b64str('nocolon')}",
424 "Basic ",
425 ]
426 for auth_value in cases:
427 conn = BitcoinHTTPConnection(self.node)
428 conn.headers = {"Authorization": f"{auth_value}"}
429 response = conn.post('/', '{"method": "getbestblockhash"}')
430 assert_equal(response.status, http.client.UNAUTHORIZED)
431 assert response.getheader('WWW-Authenticate') is not None
432 433 434 def check_disallowed_http_methods(self):
435 self.log.info("Check that unsafe or unsupported HTTP methods are rejected")
436 for method, err in [
437 ['TRACE', http.client.METHOD_NOT_ALLOWED],
438 ['CONNECT', http.client.METHOD_NOT_ALLOWED],
439 ['DELETE', http.client.METHOD_NOT_ALLOWED],
440 ['PATCH', http.client.METHOD_NOT_ALLOWED],
441 ['OPTIONS', http.client.METHOD_NOT_ALLOWED],
442 ['GET', http.client.METHOD_NOT_ALLOWED] # RPC endpoint '/' only handles POST
443 ]:
444 conn = BitcoinHTTPConnection(self.node)
445 response = conn._request(method, '/', data=None, connection_header=None)
446 assert_equal(response.status, err)
447 448 449 def check_path_traversal(self):
450 self.log.info("Check that path traversal attempts are safely rejected")
451 traversal_paths = [
452 '/../etc/passwd',
453 '/../../etc/shadow',
454 '/%2e%2e/%2e%2e/etc/passwd', # URL-encoded dots
455 '/..%2Fetc%2Fpasswd', # URL-encoded slash
456 '/.%2e/.%2e/etc/passwd', # mixed encoding
457 '/valid/../../../etc/passwd',
458 ]
459 for path in traversal_paths:
460 conn = BitcoinHTTPConnection(self.node)
461 response = conn.get(path)
462 assert_equal(response.status, http.client.NOT_FOUND)
463 464 465 def check_request_smuggling_cl_te(self):
466 self.log.info("Check request smuggling is not possible")
467 # https://www.rfc-editor.org/rfc/rfc7230#section-3.3.3
468 # Transfer-Encoding takes precedence over Content-Length.
469 # Sending both creates a smuggling vector: a front-end proxy that
470 # uses Content-Length while the back-end uses Transfer-Encoding lets an
471 # attacker prepend arbitrary bytes to the next victim's request.
472 473 # The real JSON-RPC body sent as a single chunk.
474 body = b'{"method":"getblockcount"}'
475 # Content-Length is set to the length of just the chunk-size line
476 # ("1a\r\n" = 4 bytes), not the full chunked body — the ambiguity that
477 # smuggling exploits. A server using Transfer-Encoding reads the complete
478 # chunk and responds with the block count; a server confused by the
479 # mismatch may stall, close the connection, or return an error.
480 chunk_size_line = f"{len(body):x}\r\n".encode("ascii")
481 # Signals end of body
482 empty_chunk = b'\r\n0\r\n\r\n'
483 chunk_body = chunk_size_line + body + empty_chunk
484 485 conn = BitcoinHTTPConnection(self.node)
486 raw = (
487 f"POST / HTTP/1.1\r\n"
488 f"Host: {conn.url.hostname}\r\n"
489 f"Authorization: Basic {str_to_b64str(conn.authpair)}\r\n"
490 f"Content-Length: {len(chunk_size_line)}\r\n"
491 f"Transfer-Encoding: chunked\r\n"
492 f"\r\n"
493 ).encode("ascii") + chunk_body
494 conn.send_raw(raw)
495 response = conn.recv_raw().decode()
496 assert "HTTP/1.1 200 OK" in response
497 count = self.node.getblockcount()
498 assert f'"result":{count}' in response
499 500 501 def check_duplicate_content_length(self):
502 self.log.info("Check that duplicate Content-Length headers are handled")
503 # https://www.rfc-editor.org/rfc/rfc7230#section-3.3.3
504 # Multiple Content-Length headers with differing values "MUST"
505 # result in an error.
506 conn = BitcoinHTTPConnection(self.node)
507 body = '{"method":"getblockcount"}'
508 raw = (
509 f"POST / HTTP/1.1\r\n"
510 f"Host: {conn.url.hostname}\r\n"
511 f"Authorization: Basic {str_to_b64str(conn.authpair)}\r\n"
512 f"Content-Length: {len(body)}\r\n"
513 f"Content-Length: 999\r\n"
514 f"\r\n"
515 f"{body}"
516 ).encode("ascii")
517 conn.send_raw(raw)
518 response = conn.recv_raw().decode()
519 assert response.startswith("HTTP/1.1 400")
520 521 522 def check_null_byte_in_uri(self):
523 self.log.info("Check that null bytes in the URI are safely rejected")
524 # Null-byte injection can truncate the path string in C environments,
525 # bypassing suffix/extension checks and causing unexpected file access.
526 conn = BitcoinHTTPConnection(self.node)
527 raw = (
528 "GET /safe\x00/../etc/passwd HTTP/1.1\r\n"
529 f"Host: {conn.url.hostname}\r\n"
530 f"Authorization: Basic {str_to_b64str(conn.authpair)}\r\n"
531 "\r\n"
532 ).encode("ascii")
533 conn.send_raw(raw)
534 response = conn.recv_raw().decode()
535 assert response.startswith("HTTP/1.1 400")
536 537 538 def check_invalid_http_version(self):
539 self.log.info("Check that requests with invalid HTTP versions are safely rejected")
540 cases = [
541 b"GET / \r\n\r\n", # HTTP/0.9 — no version
542 b"GET / HTTP/9.9\r\nHost: localhost\r\n\r\n", # far-future version
543 b"GET / HTTP/INVALID\r\nHost: localhost\r\n\r\n", # non-numeric version
544 b"GET / NOTHTTP/1.1\r\nHost: localhost\r\n\r\n", # wrong protocol name
545 ]
546 for raw in cases:
547 conn = BitcoinHTTPConnection(self.node)
548 conn.send_raw(raw)
549 response = conn.recv_raw().decode()
550 assert response.startswith("HTTP/1.1 400")
551 552 553 def check_whitespace_in_headers(self):
554 self.log.info("Check that requests with whitespace in headers are rejected")
555 # Extra whitespace before colon in header.
556 conn = BitcoinHTTPConnection(self.node)
557 conn.headers = {"Authorization ": f"Basic {str_to_b64str(conn.authpair)}"}
558 response = conn.post('/', '{"method": "getbestblockhash"}')
559 assert_equal(response.status, http.client.BAD_REQUEST)
560 561 # Extra whitespace at start of new line.
562 # "line folding" as defined in
563 # https://www.rfc-editor.org/rfc/rfc2616#section-2.2
564 # is considered unsafe and is explicitly deprecated in
565 # https://www.rfc-editor.org/rfc/rfc7230#section-3.2.4
566 conn = BitcoinHTTPConnection(self.node)
567 conn.headers = {"Authorization": f"Basic \n {str_to_b64str(conn.authpair)}"}
568 response = conn.post('/', '{"method": "getbestblockhash"}')
569 assert_equal(response.status, http.client.BAD_REQUEST)
570 571 572 if __name__ == '__main__':
573 HTTPBasicsTest(__file__).main()
574