Check that a value comes out the same on every path (108)

Values at the edges of what JSON, rows, a query and a plain body can say
go in to one node, over a link to a second that takes them by name in
another order and wider, and over HTTP to a third; each must come out
the same from views, a query, an answer, a restart, a restart after a
checkpoint and a replaced module: a number's bits (zero without its
sign), a text's bytes. What must be refused is refused on every way in
and leaves nothing behind.

1 files changed, 281 insertions, 1 deletions

diff --git a/tests/check.py b/tests/check.py
index 7ccafc6..6fbf1a8 100644
--- a/tests/check.py
+++ b/tests/check.py
@@ -63,7 +63,8 @@ class HttpHost:
         assert ready, 'HTTP host did not start'
         line = self.process.stdout.readline()
         assert line, self.process.communicate(timeout=5)
-        self.port = json.loads(line)['port']
+        self.info = json.loads(line)  # its port, name, public key
+        self.port = self.info['port']
 
     def request(self, method, key, body=b'', expected=None):
         import http.client
@@ -3296,6 +3297,284 @@ def server_app(aup, pure, work, seeds, operations=200, threads=1):
     return compared, sum(acknowledged)
 
 
+MEANING_KEPT = """record Kept {
+  id = 0
+  n = 0
+  u = 0
+  t: text
+  x = 0
+  label: text
+}
+state kept: list of Kept by id
+"""
+MEANING_ROWS = """view rows = each k in kept {
+  id = k.id
+  n = k.n
+  u = k.u
+  t = k.t
+  x = k.x
+  label = k.label
+}
+"""
+# A: what comes in (a JSON body, its rows, or a query and a plain body) is kept, and sent on; a put is answered
+# with its text.
+MEANING_ORIGIN = MEANING_KEPT + """record Part {
+  id = 0
+  n = 0
+  u = 0
+  t: text
+}
+input put = message {
+  id: u32
+  n: number
+  u: u32
+  t: text
+  at: {
+    x: number
+    label: text
+  }
+}
+input put_plain = message {
+  id: u32
+  n: number
+  u: u32
+  t: text
+}
+input put_rows = message {
+  parts: list of Part
+}
+input touch = message {
+  id: u32
+  t: text
+}
+output passed = message like put
+on put {
+  kept.add(Kept { id: put.id, n: put.n, u: put.u, t: put.t, x: put.at.x, label: put.at.label })
+  send passed { ..put }
+  reply put.t
+}
+on put_rows {
+  for p in put_rows.parts {
+    kept.add(Kept { id: p.id, n: p.n, u: p.u, t: p.t, x: 0, label: "rows" })
+    send passed { id: p.id, n: p.n, u: p.u, t: p.t, at: { x: 0, label: "rows" } }
+  }
+}
+on put_plain {
+  kept.add(Kept { id: put_plain.id, n: put_plain.n, u: put_plain.u, t: put_plain.t, x: 0, label: "" })
+  send passed { id: put_plain.id, n: put_plain.n, u: put_plain.u, t: put_plain.t, at: { x: 0, label: "" } }
+}
+on touch {
+  kept[touch.id].t = touch.t
+}
+""" + MEANING_ROWS
+# B: A's sends over the link, taken by name (fields in another order, u a number), kept and sent on over HTTP.
+MEANING_MIRROR = MEANING_KEPT + """input passed = message {
+  at: {
+    label: text
+    x: number
+  }
+  t: text
+  u: number
+  n: number
+  id: u32
+}
+output onward = message like passed
+on passed {
+  kept.add(Kept { id: passed.id, n: passed.n, u: passed.u, t: passed.t, x: passed.at.x, label: passed.at.label })
+  send onward { ..passed }
+}
+""" + MEANING_ROWS
+# C: B's sends over HTTP.
+MEANING_END = MEANING_KEPT + """input onward = message {
+  id: u32
+  n: number
+  u: number
+  t: text
+  at: {
+    x: number
+    label: text
+  }
+}
+on onward {
+  kept.add(Kept { id: onward.id, n: onward.n, u: onward.u, t: onward.t, x: onward.at.x, label: onward.at.label })
+}
+""" + MEANING_ROWS
+# The value set: every number a JSON reader must round correctly, u32s to its ends, texts JSON escapes and UTF-8.
+MEANING_NUMBERS = ['0', '-0', '1', '-1', '0.1', '0.3333333333333333', '9007199254740991', '9007199254740992',
+                   '9007199254740993', '-9007199254740993', '1e21', '1e20', '1e-7', '123456789012345678901234567890',
+                   '1.7976931348623157e308', '1.7976931348623158e308', '2.2250738585072014e-308',
+                   '2.2250738585072011e-308', '5e-324', '2.4703282292062328e-324', '1E+2', '-2.5e-10']
+MEANING_U32 = [0, 1, 2147483648, 4294967295]
+MEANING_TEXTS = ['', 'a', '한글 텍스트', 'emoji 🎉 and 𝄞', 'quote " backslash \\ slash /', 'line\nfeed\ttab\rreturn',
+                 'control \x01\x1f\x7f', 'nul \x00 inside', 'separator \u2028 and \u2029', '\ufeffbom',
+                 '\uffff not a character', 'x' * 4096]
+
+
+def server_meaning(aup, pure, work):
+    """One value, every path (docs/research/108): values at the edges of what
+    JSON, a query and a plain body can say come in to A; A keeps them and sends
+    them over a link to B, which takes them by name (fields in another order,
+    u widened to a number) and sends them over HTTP to C. Each value must come
+    out the same at every exit: A's view, a query on A, B's and C's views, A
+    after a restart and after a checkpoint, B after its module was replaced.
+    A number keeps its bits (zero has no sign: -0 shows as 0 and counts as 0),
+    a text its bytes. What must be refused is refused on every way in, and
+    leaves nothing behind (no row, no send)."""
+    import socket
+    import struct
+    import time
+    from http.client import HTTPConnection
+    from urllib.parse import quote
+    modules = {}
+    for name, text in (('origin', MEANING_ORIGIN), ('mirror', MEANING_MIRROR), ('end', MEANING_END),
+                       ('mirror-2', MEANING_MIRROR.replace('  id = 0\n  n = 0\n  u = 0\n  t: text\n  x = 0\n  label: text\n',
+                                                           '  label: text\n  x = 0\n  t: text\n  u = 0\n  n = 0\n  id = 0\n'))):
+        source = work / f'meaning-{name}.aup'
+        source.write_text(text)
+        compile_module(aup, source, source.with_suffix('.pmod'))
+        modules[name] = source.with_suffix('.pmod')
+    serve = pure / 'build/pure-serve'
+    keys = {}
+    for name in ('a', 'b', 'c'):
+        host = HttpHost([serve, '--module', modules['end'], '--store', work / 'meaning-keys', '--key', work / f'meaning-{name}.key'])
+        keys[name] = host.info['public_key']
+        host.stop()
+    with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as probe:
+        probe.bind(('127.0.0.1', 0))
+        link_port = probe.getsockname()[1]
+
+    def start(name, module):
+        args = [serve, '--module', modules[module], '--store', work / f'meaning-{name}', '--key', work / f'meaning-{name}.key',
+                '--name', name]
+        if name == 'a':
+            args += ['--send', f'passed=pure://127.0.0.1:{link_port}/passed', '--seal', f'passed={keys["b"]}']
+        if name == 'b':
+            args += ['--link-port', link_port, '--sender', f'a.passed={keys["a"]}',
+                     '--send', f'onward=http://127.0.0.1:{hosts["c"].port}/inputs/onward', '--seal', f'onward={keys["c"]}']
+        if name == 'c':
+            args += ['--sender', f'b.onward={keys["b"]}']
+        hosts[name] = HttpHost(args)
+
+    def send(port, path, body, kind, answer=None):
+        connection = HTTPConnection('127.0.0.1', port, timeout=30)
+        try:
+            connection.request('POST', path, body, {'Content-Length': str(len(body)), 'Content-Type': kind})
+            response = connection.getresponse()
+            got = response.read()
+            assert answer is None or response.status != 200 or got == answer, (path, got[:80], answer[:80])
+            return response.status
+        finally:
+            connection.close()
+
+    def rows(name, path='/views/rows'):
+        status, _, _, body = http(hosts[name].port, 'GET', path)
+        assert status == 200, (name, path, status, body[:200])
+        return {row['id']: row for row in json.loads(body)['items']}
+
+    def same(shown, want):  # a number's bits, zero having no sign
+        return float(shown) == want == 0 or struct.pack('<d', float(shown)) == struct.pack('<d', want)
+
+    def differing(found, expected):
+        wrong = [(key, found.get(key)) for key, (n, u, t, x, label) in expected.items()
+                 if key not in found or not (same(found[key]['n'], n) and float(found[key]['u']) == u and
+                                             found[key]['t'] == t and same(found[key]['x'], x) and found[key]['label'] == label)]
+        return wrong + [(key, 'not sent') for key in set(found) - set(expected)]
+
+    hosts, expected, refused = {}, {}, 0
+    try:
+        start('c', 'end')
+        start('b', 'mirror')
+        start('a', 'origin')
+        a = hosts['a'].port
+        count = len(MEANING_NUMBERS)
+        # In as JSON, each string once escaped to ASCII and once as UTF-8, a group's fields inside.
+        for i, n in enumerate(MEANING_NUMBERS):
+            u, t = MEANING_U32[i % len(MEANING_U32)], MEANING_TEXTS[i % len(MEANING_TEXTS)]
+            x, label = MEANING_NUMBERS[(i + 5) % count], MEANING_TEXTS[(i + 3) % len(MEANING_TEXTS)]
+            ascii_only = i % 2 == 0
+            body = '{"id": %d, "n": %s, "u": %d, "t": %s, "at": {"x": %s, "label": %s}}' % (
+                i + 1, n, u, json.dumps(t, ensure_ascii=ascii_only), x, json.dumps(label, ensure_ascii=ascii_only))
+            assert send(a, '/inputs/put', body.encode(), 'application/json', t.encode()) == 200, body[:200]
+            expected[i + 1] = (float(n), u, t, float(x), label)
+        # In as rows of one message: a JSON array, placed as its list's stored form.
+        parts = [(501 + i, n, MEANING_U32[(i + 2) % len(MEANING_U32)], MEANING_TEXTS[(i + 4) % len(MEANING_TEXTS)])
+                 for i, n in enumerate(MEANING_NUMBERS)]
+        body = '{"parts": [%s]}' % ', '.join('{"id": %d, "n": %s, "u": %d, "t": %s}' % (key, n, u, json.dumps(t))
+                                             for key, n, u, t in parts)
+        assert send(a, '/inputs/put_rows', body.encode(), 'application/json') == 200, body[:200]
+        expected.update({key: (float(n), u, t, 0.0, 'rows') for key, n, u, t in parts})
+        # In as a query (%XX where a client would write it so) and a plain body.
+        for i, n in enumerate(MEANING_NUMBERS):
+            u, t = MEANING_U32[(i + 1) % len(MEANING_U32)], MEANING_TEXTS[(i + 2) % len(MEANING_TEXTS)]
+            assert send(a, f'/inputs/put_plain?id={101 + i}&n={quote(n, safe="")}&u={u}', t.encode(),
+                        'text/plain; charset=utf-8') == 200, n
+            expected[101 + i] = (float(n), u, t, 0.0, '')
+        # What must be refused: numbers past the finite, u32s out of range or not whole, texts that are no UTF-8.
+        good = '{"id": %d, "n": %s, "u": %s, "t": %s, "at": {"x": 1, "label": ""}}'
+        bad_json = [good % (201, '1e999', '1', '"t"'), good % (202, '-1e999', '1', '"t"'), good % (203, 'NaN', '1', '"t"'),
+                    good % (204, '00', '1', '"t"'), good % (205, '+1', '1', '"t"'), good % (206, '.5', '1', '"t"'),
+                    good % (207, '1', '4294967296', '"t"'), good % (208, '1', '-1', '"t"'), good % (209, '1', '1.5', '"t"'),
+                    good % (210, '1', '1e3', '"t"'), good % (211, '1', '-0', '"t"'), good % (212, '1', '1', '"\\ud800"'),
+                    good % (213, '1', '1', '"a\\ud83d"'), good % (214, '1', '1', '"\\udc00\\ud800"')]
+        for body in bad_json:
+            assert send(a, '/inputs/put', body.encode(), 'application/json') == 400, body
+        no_utf8 = [b'\xff', b'\xc3', b'\xed\xa0\x80', b'\xc0\xaf', b'\xf4\x90\x80\x80']  # a surrogate, too long, past U+10FFFF
+        for i, bad in enumerate(no_utf8):
+            body = b'{"id": %d, "n": 1, "u": 1, "t": "' % (215 + i) + bad + b'", "at": {"x": 1, "label": ""}}'
+            assert send(a, '/inputs/put', body, 'application/json') == 400, body
+        bad_query = [('1e999', '1'), ('inf', '1'), ('nan', '1'), ('0x10', '1'), ('', '1'), ('1', '4294967296'), ('1', '-1'),
+                     ('1', '1.5'), ('1', '1e3'), ('1', '%2D0'), ('1', '%201'), ('1', '%zz')]
+        for i, (n, u) in enumerate(bad_query):
+            assert send(a, f'/inputs/put_plain?id={301 + i}&n={n}&u={u}', b't', 'text/plain') == 400, (n, u)
+        for i, bad in enumerate(no_utf8 + [b'ok \xc3']):
+            assert send(a, f'/inputs/put_plain?id={401 + i}&n=1&u=1', bad, 'text/plain') == 422, bad
+        # Rows go whole or not at all: one number past the finite refuses them all; a u that a row holds (a
+        # number) but a u32 cannot fails the event where it is sent, and none is kept.
+        assert send(a, '/inputs/put_rows', b'{"parts": [{"id": 601, "n": 1, "u": 1, "t": "a"}, '
+                                           b'{"id": 602, "n": 1e999, "u": 1, "t": "b"}]}', 'application/json') == 400
+        assert send(a, '/inputs/put_rows', b'{"parts": [{"id": 603, "n": 1, "u": 1, "t": "a"}, '
+                                           b'{"id": 604, "n": 1, "u": 1.5, "t": "b"}]}', 'application/json') != 200
+        refused = len(bad_json) + len(no_utf8) + len(bad_query) + len(no_utf8) + 1 + 2
+        # A row to change often on A alone, until a checkpoint writes its list anew; it ends as it began.
+        assert send(a, '/inputs/put', json.dumps({'id': 900, 'n': 9, 'u': 9, 't': 'filler',
+                                                  'at': {'x': 9, 'label': 'filler'}}).encode(), 'application/json') == 200
+        expected[900] = (9.0, 9, 'filler', 9.0, 'filler')
+        exits = {'a': rows('a')}
+        deadline = time.time() + 30
+        while time.time() < deadline and (len(rows('b')) < len(expected) or len(rows('c')) < len(expected)):
+            time.sleep(0.05)
+        time.sleep(0.3)  # anything more (a refused event sent after all) would be here by now
+        exits['b'], exits['c'] = rows('b'), rows('c')
+        query = work / 'meaning-query.aup'
+        query.write_text(MEANING_ROWS.replace('view rows =', 'view rows_asked ='))
+        run(aup, 'query', work / 'meaning-origin.aup', query, '-o', work / 'meaning-query.pmod')
+        status, _, _, body = http(a, 'PUT', '/queries/rows_asked', (work / 'meaning-query.pmod').read_bytes())
+        assert status == 200, body[:200]
+        exits['a queried'] = rows('a', '/queries/rows_asked')
+        hosts.pop('a').stop()
+        start('a', 'origin')
+        exits['a restarted'] = rows('a')
+        for step in range(400):  # 1.6 MiB of changes to a list of a few KiB: its image is written anew
+            assert send(hosts['a'].port, '/inputs/touch', json.dumps({'id': 900, 't': f'{step:04}' * 1024}).encode(),
+                        'application/json') == 200
+        assert send(hosts['a'].port, '/inputs/touch', json.dumps({'id': 900, 't': 'filler'}).encode(),
+                    'application/json') == 200
+        checkpoints = hosts.pop('a').stop()[-1]['checkpoints']
+        assert checkpoints >= 1, checkpoints
+        start('a', 'origin')
+        exits['a after a checkpoint'] = rows('a')
+        hosts.pop('b').stop()
+        start('b', 'mirror-2')
+        exits['b replaced'] = rows('b')
+        for where, found in exits.items():
+            wrong = differing(found, expected)
+            assert not wrong, (where, wrong[:3])
+    finally:
+        for host in hosts.values():
+            host.stop()
+    return {'values': len(expected), 'refused': refused, 'exits': len(exits), 'checkpoints': checkpoints}
+
+
 def main():
     global EDITOR
     parser = argparse.ArgumentParser()
@@ -3343,6 +3622,7 @@ def main():
         report['server_arguments'] = server_arguments(aup, pure, work)
         report['server_answers'] = server_answers(aup, pure, work)
         report['server_answers_memory'] = server_answers_memory(aup, pure, work)
+        report['server_meaning'] = server_meaning(aup, pure, work)
     print(json.dumps(report))