1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
|
#!/usr/bin/env python3
"""Research 140: HTTP/1.1 requests read a packet at a time, written in aup (aup's examples/http.aup).
Builds it with aup (AUP, default the sibling checkout) and feeds it streams of pipelined requests (methods, targets,
HTTP/1.1 and a last HTTP/1.0, fields with mixed-case names and spaces around their values, bodies by Content-Length or
chunked with extensions and trailers, some streams ending in part of a request), each cut into packets: whole, at
random boundaries, and a byte at a time for the shorter ones, each packet an event. The requests read must be h11's (a
known implementation that reads as bytes come) and the bytes left those after the last of them, on every cut, and the
three paths (batches, rounds, all compiled) the same. Then damaged streams (cut short, a bit flipped) fed whole against http_requests.py, which reads
them the same way: the same requests and bytes left when it takes one, a failed event when it refuses it. Then counts
lines (code lines: neither blank nor a comment; brace lines: a closing brace alone) of http.aup and of
http_requests.py's algorithm. One JSON line. Given a path, it also writes the whole streams there for http_time.c; given a directory after it, Pure's
tests/specs/http there."""
import json, os, random, subprocess, sys, tempfile
import h11
here = os.path.dirname(os.path.abspath(__file__))
pure = os.path.normpath(os.path.join(here, '../../..'))
aup = os.environ.get('AUP', os.path.join(pure, '..', 'aup'))
sys.path.insert(0, here)
import http_requests # noqa: E402
TOKEN = "!#$%&'*+-.^_`|~0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
def lines(path, stop=None):
code = braces = 0
for line in open(path):
if stop and line.startswith(stop): break
text = line.strip()
if not text or text.startswith('#'): continue
code += 1
braces += text == '}'
return code, braces
def streams(chooser, count):
def word(chars, low, high):
return ''.join(chooser.choices(chars, k=chooser.randrange(low, high))).encode()
def ows():
return chooser.choice([b'', b'', b' ', b'\t', b' \t'])
def request(minor):
method = chooser.choice([b'GET', b'POST', b'PUT', b'DELETE', b'PATCH', b'OPTIONS', b'HEAD', word(TOKEN, 1, 8)])
target = b'/' + bytes(chooser.randrange(0x21, 0x7f) for _ in range(chooser.randrange(0, 30)))
fields = [(chooser.choice([b'Host', b'host', b'HOST']), b'example.com')]
for _ in range(chooser.randrange(0, 6)):
name = word(TOKEN, 1, 12)
if name.lower() in (b'host', b'content-length', b'transfer-encoding', b'connection', b'expect', b'upgrade'):
continue
value = (b' ' if chooser.random() < 0.5 else b'\t').join(
bytes(chooser.randrange(0x21, 0x7f) for _ in range(chooser.randrange(1, 10))) for _ in range(chooser.randrange(0, 4)))
fields.append((name, value))
chooser.shuffle(fields)
body, framing = b'', []
pick = chooser.random()
if pick < 0.35:
body = bytes(chooser.randrange(256) for _ in range(chooser.randrange(0, 300)))
framing = [(b'Content-Length', str(len(body)).encode())]
elif pick < 0.65 and minor == 1:
framing = [(chooser.choice([b'Transfer-Encoding', b'transfer-encoding']), chooser.choice([b'chunked', b'Chunked']))]
chunks = [bytes(chooser.randrange(256) for _ in range(chooser.randrange(1, 120))) for _ in range(chooser.randrange(0, 5))]
for c in chunks:
size = (f'{len(c):x}' if chooser.random() < 0.5 else f'{len(c):X}').encode()
size = b'0' * chooser.choice([0, 0, 1, 3]) + size
tail = b';' + word('abc=', 0, 6) if chooser.random() < 0.3 else ows()
body += size + tail + b'\r\n' + c + b'\r\n'
trailers = b''.join(word(TOKEN, 1, 8) + b':' + ows() + word('xyz', 0, 6) + b'\r\n' for _ in range(chooser.randrange(0, 3)))
body += b'0' + (b';x' if chooser.random() < 0.2 else b'') + b'\r\n' + trailers + b'\r\n'
head = method + b' ' + target + b' HTTP/1.' + str(minor).encode() + b'\r\n'
head += b''.join(n + b':' + ows() + v + ows() + b'\r\n' for n, v in fields + framing)
return head + b'\r\n' + body
out = []
for _ in range(count):
stream = b''.join(request(1) for _ in range(chooser.randrange(1, 5)))
if chooser.random() < 0.2:
stream += request(0) # an HTTP/1.0 request ends its connection: last
elif chooser.random() < 0.3:
part = request(1)
stream += part[:chooser.randrange(1, len(part))]
out.append(stream)
return out
def cuts(chooser, stream):
"""The stream whole, at random boundaries, and (if it is short) a byte at a time."""
at = sorted(chooser.sample(range(1, len(stream)), min(len(stream) - 1, chooser.randrange(1, 12))))
out = [[stream], [stream[a:b] for a, b in zip([0] + at, at + [len(stream)])]]
if len(stream) < 400:
out.append([stream[i:i + 1] for i in range(len(stream))])
return out
def h11_reading(packets):
"""The requests h11 reads whole from the packets, each as http_requests' tuples. (h11 hands a request's line, fields
and body on as they come, so the bytes left are those after the last whole request, as http_requests keeps them.)"""
conn, out, current = h11.Connection(h11.SERVER), [], None
for packet in packets:
conn.receive_data(packet)
while True:
event = conn.next_event()
if event in (h11.NEED_DATA, h11.PAUSED):
break
if isinstance(event, h11.Request):
current = [event.method, event.target, int(event.http_version[-1:]), list(event.headers), b'', []]
elif isinstance(event, h11.Data):
current[4] += event.data
elif isinstance(event, h11.EndOfMessage):
current[5] = list(event.headers)
out.append(tuple(current))
conn.send(h11.Response(status_code=200, headers=[(b'content-length', b'0')]))
conn.send(h11.EndOfMessage())
if conn.our_state is h11.DONE and conn.their_state is h11.DONE:
conn.start_next_cycle()
return out
def normal(requests):
"""Requests with transfer-encoding's value lowercased, as h11 hands it on (that field alone)."""
return [(m, t, v, [(n, x.lower() if n == b'transfer-encoding' else x) for n, x in f], b, tr) for m, t, v, f, b, tr in requests]
def run(module, packets, env, work):
given = []
for n, packet in enumerate(packets):
path = os.path.join(work, f'{n}.packet')
with open(path, 'wb') as f:
f.write(packet)
given.append(f'message packet @{path}\n')
out = subprocess.run([os.path.join(pure, 'build', 'pure-run'), module], input=''.join(given), capture_output=True,
text=True, env=dict(os.environ, **env), timeout=600).stdout.splitlines()
state = json.loads(out[-1])
s = state['states']
pairs = lambda fs: [(bytes.fromhex(f['name']), bytes.fromhex(f['value'])) for f in fs]
requests = [(bytes.fromhex(r['method']), bytes.fromhex(r['target']), r['minor'], pairs(r['fields']), bytes.fromhex(r['body']),
pairs(r['trailers'])) for r in s['requests']]
left = bytes.fromhex(s['pending'][0]['data']) if s['pending'] else b''
return out, (requests, left), any(json.loads(line)['fault'] != 'none' for line in out)
chooser = random.Random(140)
made = streams(chooser, 200)
if len(sys.argv) > 1: # the whole streams (no request left in part) for http_time.c: each a u32 big-endian length, then it
with open(sys.argv[1], 'wb') as f:
f.write(b''.join(len(s).to_bytes(4, 'big') + s for s in made if not http_requests.feed(b'', s)[1]))
if len(sys.argv) > 2: # Pure's tests/specs/http: the first 80 streams so, and what http_requests (h11's reading) reads
with open(os.path.join(sys.argv[2], 'streams.bin'), 'wb') as f:
f.write(b''.join(len(s).to_bytes(4, 'big') + s for s in made[:80]))
hexed = lambda fs: [[n.hex(), v.hex()] for n, v in fs]
with open(os.path.join(sys.argv[2], 'read.json'), 'w') as f:
json.dump([{'requests': [[m.hex(), t.hex(), v, hexed(fs), b.hex(), hexed(tr)] for m, t, v, fs, b, tr in got], 'left': left.hex()}
for got, left in (http_requests.feed(b'', s) for s in made[:80])], f, separators=(',', ':'))
f.write('\n')
paths = {'batches': 0, 'rounds': 0, 'all': 0}
fed = {'whole': 0, 'random': 0, 'bytes': 0}
same_paths = True
with tempfile.TemporaryDirectory() as work:
module = os.path.join(work, 'http.pmod')
subprocess.run([os.path.join(aup, 'build', 'aup'), 'build', os.path.join(aup, 'examples', 'http.aup'), '-o', module],
check=True)
for stream in made:
for kind, packets in zip(('whole', 'random', 'bytes'), cuts(chooser, stream)):
want = http_requests.feed(b'', stream)
assert normal(want[0]) == h11_reading(packets), 'the reference differs from h11'
envs = (('batches', {'PURE_JIT': '0'}), ('rounds', {'PURE_JIT': '0', 'PURE_BATCH': '0'}), ('all', {'PURE_JIT': 'all'})) \
if kind == 'random' else (('batches', {'PURE_JIT': '0'}),)
raws = []
for name, env in envs:
raw, got, failed = run(module, packets, env, work)
right = not failed and got == want
fed[kind] += right and name == 'batches'
if kind == 'random':
paths[name] += right
raws.append(raw)
same_paths = same_paths and all(r == raws[0] for r in raws)
broken = []
for stream in chooser.sample(made, 100):
flipped = bytearray(stream)
flipped[chooser.randrange(len(stream))] ^= 1 << chooser.randrange(8)
broken += [stream[:chooser.randrange(1, len(stream))], bytes(flipped)]
agreed = refused = 0
for stream in broken:
try:
want = http_requests.feed(b'', stream)
except ValueError:
want = None
refused += want is None
_, got, failed = run(module, [stream], {}, work)
agreed += (want is None and failed) or (want is not None and not failed and got == want)
aup_code, aup_braces = lines(os.path.join(aup, 'examples', 'http.aup'))
python_code, _ = lines(os.path.join(here, 'http_requests.py'), stop='if __name__')
ok = all(n == len(made) for n in paths.values()) and fed['whole'] == fed['random'] == len(made) and same_paths and \
fed['bytes'] == sum(len(s) < 400 for s in made) and agreed == len(broken)
print(json.dumps({'streams': len(made), 'bytes': sum(map(len, made)),
'requests': sum(len(http_requests.feed(b'', s)[0]) for s in made), 'right': fed, 'paths': paths,
'paths_same': same_paths, 'damaged': len(broken), 'damaged_refused': refused, 'damaged_agreed': agreed,
'aup_lines': aup_code, 'aup_brace_lines': aup_braces, 'python_lines': python_code}))
sys.exit(0 if ok else 1)
|
