Remember each sender's secret beside the node's key: no X25519 at restart

A listener lets a quiet sender's secret go with its active state, so a
node started again made one X25519 for every sender that came back (169
us on this Pi, most of a wake's CPU): asymmetric work in proportion to the
links, which design 011 counts as a reason to design again. The secret is
now remembered in a file beside the key (KEY.links, 0600, begun again for
another node's key): a place of 64 bytes a sender in the order allowed,
mapped, so a sender at rest takes no memory for it. A place is the first
28 bytes of the sender's key, a check (FNV-1a of the key and the secret)
and the secret, written secret first and key last; a place whose check
fails is made again. The UDP link checks see a receiver started again
make no X25519.

4 files changed, 89 insertions, 10 deletions

diff --git a/serve/link.c b/serve/link.c
index ed1b695..e8c8a7a 100644
--- a/serve/link.c
+++ b/serve/link.c
@@ -13,8 +13,10 @@
 #include <stdlib.h>
 #include <string.h>
 #include <sys/eventfd.h>
+#include <sys/mman.h>
 #include <sys/random.h>
 #include <sys/socket.h>
+#include <sys/stat.h>
 #include <time.h>
 #include <unistd.h>
 
@@ -783,6 +785,8 @@ struct PureLinkListener {
     unsigned port;
     uint64_t run, idle, swept, trimmed, untrimmed;
     PureAead *aead; /* keyed by each packet's run */
+    unsigned char *memo; /* the secrets remembered (pure_link_remember), mapped */
+    size_t memo_size;
     int fd, wake;
     pthread_t thread;
     bool started;
@@ -926,12 +930,71 @@ static void drop_held(Stream *st, size_t k) {
     --st->held_count;
 }
 
-/* A peer's active state, made at its first packet: the secret its key shares with ours. */
+/* The memo of secrets (pure_link_remember): a header, then a place a peer
+ * in the order allowed, each the first 28 bytes of its key, a check (FNV-1a
+ * of its whole key and the secret) and the secret. A place whose check
+ * fails (never made, another key, cut short by a crash) is made again. */
+enum { MEMO_HEAD = 64, MEMO_PLACE = 64 };
+static const unsigned char memo_magic[8] = {'P', 'L', 'M', '1', 0, 0, 0, 0};
+
+static uint32_t memo_check(const unsigned char key[PURE_SEAL_KEY], const unsigned char secret[32]) {
+    return fnv(fnv(2166136261u, key, PURE_SEAL_KEY), secret, 32);
+}
+
+bool pure_link_remember(PureLinkListener *s, const char *path, char *error, size_t size) {
+    int fd = open(path, O_RDWR | O_CREAT | O_NOFOLLOW | O_CLOEXEC, 0600);
+    struct stat info;
+    unsigned char head[MEMO_HEAD];
+    size_t needed = MEMO_HEAD + (s->peer_count ? s->peer_count : 1) * MEMO_PLACE;
+    bool ok = fd >= 0 && !fstat(fd, &info) && S_ISREG(info.st_mode) && !(info.st_mode & 077);
+    /* Another node's key, or none yet: begun again. */
+    bool ours = ok && pread(fd, head, MEMO_HEAD, 0) == MEMO_HEAD && !memcmp(head, memo_magic, 8) &&
+                !memcmp(head + 8, s->own_public, PURE_SEAL_KEY);
+    if (ok && !ours) {
+        memset(head, 0, MEMO_HEAD);
+        memcpy(head, memo_magic, 8);
+        memcpy(head + 8, s->own_public, PURE_SEAL_KEY);
+        ok = !ftruncate(fd, 0) && pwrite(fd, head, MEMO_HEAD, 0) == MEMO_HEAD;
+    }
+    ok = ok && (ours && (size_t)info.st_size >= needed ? true : !ftruncate(fd, (off_t)needed));
+    void *map = ok ? mmap(NULL, needed, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0) : MAP_FAILED;
+    if (fd >= 0) close(fd);
+    if (map == MAP_FAILED) {
+        snprintf(error, size, "%s: cannot keep secrets there (a file only its owner may read)", path);
+        return false;
+    }
+    s->memo = map;
+    s->memo_size = needed;
+    return true;
+}
+
+/* A peer's active state, made at its first packet: the secret its key
+ * shares with ours, remembered or made (one X25519, then remembered: the
+ * secret first, the key that makes the place its own last). */
 static Active *activate(PureLinkListener *s, Peer *peer, uint64_t now) {
     Active *a = calloc(1, sizeof(*a)), **actives = room(s->actives, &s->active_capacity, s->active_count, sizeof(*actives), 64);
     if (actives) s->actives = actives;
-    if (!a || !actives || !pure_link_secret(s->own, peer->key, a->secret)) {
-        if (a) explicit_bzero(a->secret, sizeof(a->secret));
+    if (!a || !actives) {
+        free(a);
+        return NULL;
+    }
+    size_t index = (size_t)(peer - s->peers);
+    unsigned char *place = s->memo && MEMO_HEAD + (index + 1) * MEMO_PLACE <= s->memo_size ?
+                           s->memo + MEMO_HEAD + index * MEMO_PLACE : NULL;
+    if (place && !memcmp(place, peer->key, 28) && get32(place + 28) == memo_check(peer->key, place + 32)) {
+        memcpy(a->secret, place + 32, 32);
+        ++s->counts.remembered;
+    } else if (pure_link_secret(s->own, peer->key, a->secret)) {
+        if (place) {
+            memset(place, 0, 28);
+            __atomic_thread_fence(__ATOMIC_RELEASE);
+            memcpy(place + 32, a->secret, 32);
+            put32(place + 28, memo_check(peer->key, a->secret));
+            __atomic_thread_fence(__ATOMIC_RELEASE);
+            memcpy(place, peer->key, 28);
+        }
+    } else {
+        explicit_bzero(a, sizeof(*a));
         free(a);
         return NULL;
     }
@@ -1409,6 +1472,7 @@ void pure_link_listener_close(PureLinkListener *s, PureLinkCounts *counts) {
     }
     while (s->active_count) let_go(s, s->actives[0]);
     pure_aead_free(s->aead);
+    if (s->memo) munmap(s->memo, s->memo_size);
     free(s->actives);
     free(s->peers);
     free(s->index);
diff --git a/serve/link.h b/serve/link.h
index 51f12bd..b831ab4 100644
--- a/serve/link.h
+++ b/serve/link.h
@@ -89,6 +89,12 @@ bool pure_link_allow(PureLinkListener *, const char *source, const unsigned char
  * keys, addresses and streams go: an idle link costs the listener only its
  * allowing, and the sender's next packet makes them again (one X25519). */
 void pure_link_idle(PureLinkListener *, uint64_t ms);
+/* Remember the secret each sender's key shares with the node's in a file
+ * (path, 0600; begun again for another node's key): a place a sender in
+ * the order allowed, mapped, so a sender at rest takes no memory for it.
+ * A node that starts again then makes no X25519 for a sender it heard from
+ * before. After the senders are allowed, before serving. */
+bool pure_link_remember(PureLinkListener *, const char *path, char *error, size_t size);
 bool pure_link_serve(PureLinkListener *, char *error, size_t size);
 /* stored: the delivery's save succeeded; gap: it stopped at a gap. */
 void pure_link_done(PureLinkListener *, void *token, uint64_t heard, bool stored, bool gap);
@@ -96,10 +102,10 @@ void pure_link_done(PureLinkListener *, void *token, uint64_t heard, bool stored
 /* Counts for a host's report: packets and records sent, records sent again,
  * packets lost and acknowledged, packets received, deliveries handed over,
  * the last number stored; a listener's senders woken (their active state
- * made), let go when idle, and active when it closed. A listener's are
- * given when it closes. */
+ * made), of them with a secret remembered (no X25519), let go when idle,
+ * and active when it closed. A listener's are given when it closes. */
 typedef struct {
-    uint64_t packets, records, resent, lost, acknowledged, received, deliveries, stored, woken, swept, active;
+    uint64_t packets, records, resent, lost, acknowledged, received, deliveries, stored, woken, remembered, swept, active;
 } PureLinkCounts;
 void pure_link_counts(PureLink *, PureLinkCounts *);
 void pure_link_listener_close(PureLinkListener *, PureLinkCounts *counts);
diff --git a/serve/serve.c b/serve/serve.c
index 5309f02..93406fd 100644
--- a/serve/serve.c
+++ b/serve/serve.c
@@ -2662,6 +2662,9 @@ int main(int argc, char **argv) {
         for (unsigned i = 0; allowed && i < sender_count; ++i)
             allowed = pure_link_allow(link_listener, senders[i].name, senders[i].key, error, sizeof(error));
         if (allowed && link_idle_ms) pure_link_idle(link_listener, link_idle_ms);
+        char memo[4096];
+        snprintf(memo, sizeof(memo), "%s.links", key_path); /* beside the key the secrets come from */
+        allowed = allowed && pure_link_remember(link_listener, memo, error, sizeof(error));
         if (!allowed || !pure_link_serve(link_listener, error, sizeof(error))) {
             fprintf(stderr, "--link-port: %s\n", error);
             return 2;
@@ -2736,11 +2739,12 @@ int main(int argc, char **argv) {
             ",\"link_packets\":%" PRIu64 ",\"link_records\":%" PRIu64 ",\"link_resent\":%" PRIu64
             ",\"link_lost\":%" PRIu64 ",\"link_acknowledged\":%" PRIu64 ",\"link_stored\":%" PRIu64
             ",\"listener_packets\":%" PRIu64 ",\"listener_received\":%" PRIu64 ",\"listener_deliveries\":%" PRIu64
-            ",\"listener_woken\":%" PRIu64 ",\"listener_swept\":%" PRIu64 ",\"listener_active\":%" PRIu64 "}\n",
+            ",\"listener_woken\":%" PRIu64 ",\"listener_remembered\":%" PRIu64 ",\"listener_swept\":%" PRIu64
+            ",\"listener_active\":%" PRIu64 "}\n",
             active, retained, accepted, requests, bursts, events, ticks, expired, threads, owned ? "true" : "false",
             owner.executor.publications, shown, delivered, attempts, owner.executor.answers, groups, grouped,
             checkpoints, sent.packets, sent.records, sent.resent, sent.lost, sent.acknowledged, sent.stored,
-            heard.packets, heard.received, heard.deliveries, heard.woken, heard.swept, heard.active);
+            heard.packets, heard.received, heard.deliveries, heard.woken, heard.remembered, heard.swept, heard.active);
     free(hosts);
     free(thread);
     return 0;
diff --git a/tests/check.py b/tests/check.py
index e46622b..7be32d7 100644
--- a/tests/check.py
+++ b/tests/check.py
@@ -784,7 +784,9 @@ def server_links_udp(aup, pure, work, netem=None):
     delays, loses and reorders packets meanwhile. The receiver takes its
     senders from a file (--senders, the sender among a thousand others) and
     lets a sender's keys and streams go after 400 ms of quiet
-    (--link-idle-ms): the order after a longer pause arrives all the same."""
+    (--link-idle-ms): the order after a longer pause arrives all the same.
+    Started again, it makes no X25519: the secret is remembered beside its
+    key (KEY.links)."""
     import select
     import signal
     import socket
@@ -903,11 +905,14 @@ def server_links_udp(aup, pure, work, netem=None):
             _, error = process.communicate(timeout=30)
             assert process.returncode == 0, error[-1000:]
             stats[name] = [json.loads(line) for line in error.decode().splitlines() if line.startswith('{"active"')][-1]
+    # The receiver, started again, remembered the sender's secret: no X25519 for it.
     assert stats['b']['listener_swept'] >= 1 and stats['b']['listener_woken'] >= 2, stats['b']
+    assert stats['b']['listener_remembered'] == stats['b']['listener_woken'], stats['b']
     return {'orders': sent, 'acknowledged_total': sum(acknowledged),
             'packets_after_restart': stats['a']['link_packets'], 'resent_after_restart': stats['a']['link_resent'],
             'lost_after_restart': stats['a']['link_lost'], 'received_after_restart': stats['b']['listener_received'],
-            'woken_after_restart': stats['b']['listener_woken'], 'swept_after_restart': stats['b']['listener_swept']}
+            'woken_after_restart': stats['b']['listener_woken'], 'swept_after_restart': stats['b']['listener_swept'],
+            'remembered_after_restart': stats['b']['listener_remembered']}
 
 
 def server_links_udp_lossy(aup, pure, work):