diff options
| -rw-r--r-- | link/link-protocol.c | 101 | ||||
| -rw-r--r-- | link/link-protocol.h | 20 | ||||
| -rw-r--r-- | link/test-link.c | 65 | ||||
| -rw-r--r-- | research/experiments/019-idle-links/2026-09-30-lost-linux-aarch64.jsonl | 2 |
4 files changed, 154 insertions, 34 deletions
diff --git a/link/link-protocol.c b/link/link-protocol.c index 5b9eaa3..44aa3e9 100644 --- a/link/link-protocol.c +++ b/link/link-protocol.c @@ -19,7 +19,7 @@ enum { DATAGRAM_MOST = PURE_LINK_DATAGRAM_MOST, BODY_MOST = DATAGRAM_MOST - HEAD * at least this much of it fits. */ enum { PART_LEAST = 64 }; enum { F_PADDING = 0, F_STREAM = 1, F_RECORD = 2, F_ACK = 3, F_STORED = 4, F_PING = 5, F_RECORDS = 6, F_CHALLENGE = 7, - F_RESPONSE = 8, F_OFFER = 9, F_ACCEPT = 10, STORED_GAP = 1 }; + F_RESPONSE = 8, F_OFFER = 9, F_ACCEPT = 10, F_LOST = 11, STORED_GAP = 1 }; /* A packet's first byte is VERSION with the sending side's epoch (mod 16) * in its low four bits; OFFER and ACCEPT are an epoch's number (u32) and a * public key made for it, and ACCEPT the first bytes of the key it answers @@ -42,7 +42,7 @@ enum { REACH_TRIES = 3, HOLE_LOST = 3, HOLE_PROBES = 2 }; enum { RESPONSES = 4 }; /* Keys made for runs that do not open, and new epochs accepted: a second, * and at once. */ -enum { KEYINGS = 256, ACCEPTS = 256, ACCEPTS_AT_ONCE = 8 }; +enum { KEYINGS = 256, ACCEPTS = 256, ACCEPTS_AT_ONCE = 8, LOSTS = 4096 }; /* A receiver accepts a new epoch for a sender at most a second. */ static const uint64_t ACCEPT_EVERY_NS = 1000000000u, OURS_LIFE = 1000000000u; /* CUBIC's constants (RFC 9438): the cubic's scale, and the window kept at a @@ -300,6 +300,7 @@ struct PureLinkSender { static bool seal_in(PureLinkSender *l, uint32_t n, const unsigned char secret[32]); static bool offer(PureLinkSender *l, uint32_t n); +static void epoch_lost(PureLinkSender *l); static void begin_epoch(PureLinkSender *l, const unsigned char their[PURE_SEAL_KEY], uint64_t now); /* The rings of records and flights are powers of two long (grown by @@ -691,14 +692,15 @@ void pure_link_sender_take(PureLinkSender *l, const unsigned char *packet, size_ get32(packet + 1) != l->peer_id) return; /* not ours, or an error from an earlier send (nobody listening) */ uint64_t run = get64(packet + 5), pn = get64(packet + 13); - /* Answers come in its epoch or the one before. */ + /* Answers come in its epoch, the one before, or 0 (LOST: a receiver that + * knows no epoch of this run says so under the nodes' keys). */ unsigned e = packet[0] & EPOCH_MASK; - uint32_t epoch = e == (l->epoch & EPOCH_MASK) ? l->epoch : l->before_epoch; + uint32_t epoch = e == (l->epoch & EPOCH_MASK) ? l->epoch : e == (l->before_epoch & EPOCH_MASK) ? l->before_epoch : 0; if (e != (epoch & EPOCH_MASK)) return; if (!l->in || l->in_run != run || l->in_epoch != epoch) { pure_aead_free(l->in); - l->in = aead_of(epoch == l->epoch ? l->epoch_secret : l->before_secret, l->peer, l->own_public, run, l->run, - l->in_base); + l->in = aead_of(epoch == l->epoch ? l->epoch_secret : epoch == l->before_epoch ? l->before_secret : l->secret, + l->peer, l->own_public, run, l->run, l->in_base); ++l->counts.keyed; l->in_run = run; l->in_epoch = epoch; @@ -737,6 +739,11 @@ void pure_link_sender_take(PureLinkSender *l, const unsigned char *packet, size_ !memcmp(plain + at + EPOCH_FRAME - 1, l->offer_public, ANSWERS)) begin_epoch(l, plain + at + 4, now); at += ACCEPT_FRAME - 1; + } else if (type == F_LOST) { /* of a packet still out: the receiver lost this run's epoch */ + if (at + 8 > size) break; + Flight *f = flight_of(l, get64(plain + at)); + if (f && !f->done && (l->epoch || l->renegotiating)) epoch_lost(l); + at += 8; } else if (type != F_PING) break; /* padding, or what this side does not know: the rest */ } @@ -1045,6 +1052,32 @@ static bool offer(PureLinkSender *l, uint32_t n) { return true; } +/* The receiver knows no epoch of this run (it started again without its + * memo), and said so under the nodes' keys of a packet still out: begun + * again as a run begins, in epoch 0 with records until the next epoch is + * accepted (only a receiver can say it; a word of the past names a packet + * no longer out). What it did not store goes again, what flies went + * nowhere: no congestion counted for it. Silence alone is not this: that + * could be an attacker's doing, and then only offers go (tick). */ +static void epoch_lost(PureLinkSender *l) { + if (l->epoch) { + seal_in(l, 0, l->secret); + offer(l, 1); + ++l->counts.fallbacks; + } + l->renegotiating = false; + l->pto_count = 0; + for (size_t k = 0; k < l->flight_count; ++k) { + Flight *f = &l->flights[(l->flight_head + k) & (l->flight_capacity - 1)]; + if (!f->done) settle(l, f), f->count = 0; + } + for (size_t k = 0; k < l->count; ++k) { + Record *r = record_at(l, k); + for (uint32_t i = 0; i < r->parts; ++i) + if (part_of(r, i)->state != UNSENT) set_part(l, r, part_of(r, i), UNSENT), ++l->counts.resent; + } +} + /* The receiver accepted the offer with its public key: the epoch begins, * the private half gone. */ static void begin_epoch(PureLinkSender *l, const unsigned char their[PURE_SEAL_KEY], uint64_t now) { @@ -1233,6 +1266,9 @@ struct PureLinkReceiver { * whose packets open (a thousand waking at once) cost none. */ double key_tokens; uint64_t key_at; + /* LOSTs (lost) come out of a bucket of their own: LOSTS a second. */ + double lost_tokens; + uint64_t lost_at; /* The pair its epochs are accepted with: one for a second (the X25519 * of making it shared by the offers of that second), then the private * half is gone. An accept costs an X25519 on each side (220 µs on a @@ -1250,6 +1286,10 @@ struct PureLinkReceiver { /* Runs whose offers wait for the turn (a slot may hold another run by then). */ struct { Run *run; uint64_t id; } *offers; size_t offer_count, offer_capacity; + /* Answers sealed where there was no run to answer from, sent in the turn: + * LOST, to a sender whose packet came in an epoch it does not know. */ + struct { unsigned char packet[HEADER + 9 + PURE_AEAD_TAG], to[PURE_LINK_ADDRESS]; size_t to_size; } *notices; + size_t notice_count, notice_capacity; PureAead *aead; /* keyed by each packet's run */ unsigned char open_key[PURE_AEAD_KEY], seal_key[PURE_AEAD_KEY]; /* the keys it has now */ bool open_keyed, seal_keyed; @@ -2002,6 +2042,34 @@ static void accept_waiting(PureLinkReceiver *s, uint64_t now) { s->offer_count = 0; } +/* A packet of a run it does not know, in an epoch past 0 it knows nothing + * of (it started again without its memo, or the memo kept another run of + * the sender's): LOST, sealed in epoch 0, naming the packet's number, to + * where it came from, in the turn. The sender, if that packet is one of + * those it has out, begins again from epoch 0 as a run begins. As QUIC's + * stateless reset (RFC 9000 §10.3) it answers a packet it cannot open, so + * it is smaller than what asked for it, and LOSTS a second at most. */ +enum { LOST_SIZE = HEADER + 9 + PURE_AEAD_TAG }; +static void lost(PureLinkReceiver *s, const Peer *peer, const Active *a, uint64_t run, uint64_t pn, const void *from, + size_t from_size) { + void *notices = room(s->notices, &s->notice_capacity, s->notice_count, sizeof(*s->notices), 16); + unsigned char key[PURE_AEAD_KEY], base[PURE_AEAD_NONCE], body[9] = {F_LOST}; + uint64_t answer_run = random64(); + if (!notices || s->notice_count >= LOSTS || + !derive(a->secret, s->own_public, peer->key, answer_run, run, key, base)) + return; + s->notices = notices; + put64(body + 1, pn); + size_t at = s->notice_count, to_size = from_size < PURE_LINK_ADDRESS ? from_size : PURE_LINK_ADDRESS; + memcpy(s->seal_key, key, PURE_AEAD_KEY), s->seal_keyed = true; + if (seal_packet(s->aead, key, base, s->own_id, 0, answer_run, 0, body, sizeof(body), s->notices[at].packet)) { + memcpy(s->notices[at].to, from, to_size); + s->notices[at].to_size = to_size; + ++s->notice_count; + } + explicit_bzero(key, sizeof(key)); +} + /* Whether keys may be made for a run not seen (a token is there). */ static bool may_key(PureLinkReceiver *s, uint64_t now) { return has_token(&s->key_tokens, &s->key_at, KEYINGS, KEYINGS, now); @@ -2021,8 +2089,8 @@ static bool take_from(PureLinkReceiver *s, Peer *peer, const unsigned char *pack Run made = {.active = a, .run = run, .seen = now, .used = true}, *k = r ? r : &made; if (!r) { made.answer_run = random64(); - ++s->counts.keyed; - if (!run_begun(s, peer, a, &made, e)) k = NULL; + if (run_begun(s, peer, a, &made, e)) ++s->counts.keyed; + else k = NULL; } /* The keys of the packet's epoch: the next, now's, the one before, or * epoch 0's for a sender that begins its epochs again. Two may share @@ -2046,8 +2114,18 @@ static bool take_from(PureLinkReceiver *s, Peer *peer, const unsigned char *pack bool uses_next = k && opened == &k->next; explicit_bzero(&zero, sizeof(zero)); if (!opened) { + /* A run whose epoch it does not know (no keys made for it): LOST. A + * token (KEYINGS) goes for keys made for nothing, and for a secret + * made and not kept (no memo): not for this, as the secret made for + * it is remembered, once a sender. */ + bool unknown = !k && e; + if (unknown && n > LOST_SIZE && has_token(&s->lost_tokens, &s->lost_at, LOSTS, LOSTS, now)) { + --s->lost_tokens; + lost(s, peer, a, run, pn, from, from_size); + } explicit_bzero(&made, sizeof(made)); - if (!r) s->key_tokens = s->key_tokens > 1 ? s->key_tokens - 1 : 0; /* keys for nothing */ + if (!r && (!unknown || (woke && !a->remembered && !s->memo))) + s->key_tokens = s->key_tokens > 1 ? s->key_tokens - 1 : 0; /* keys for nothing */ if (woke) let_go(s, a); return false; } @@ -2273,6 +2351,10 @@ void pure_link_receiver_turn(PureLinkReceiver *s, const PureLinkDone *done, size } apply_done(s, done, count); hand_over_ready(s); + for (size_t k = 0; k < s->notice_count; ++k) + if (send(context, s->notices[k].packet, sizeof(s->notices[k].packet), s->notices[k].to, s->notices[k].to_size)) + ++s->counts.packets; + s->notice_count = 0; answer(s, now, send, context); if (s->offer_count) { /* the offers' X25519 after the answers, then their ACCEPTs */ accept_waiting(s, now); @@ -2296,6 +2378,7 @@ void pure_link_receiver_free(PureLinkReceiver *s) { pure_aead_free(s->aead); explicit_bzero(s->ours_private, sizeof(s->ours_private)); free(s->offers); + free(s->notices); if (s->memo) munmap(s->memo, s->memo_size); free(s->actives); free(s->peers); diff --git a/link/link-protocol.h b/link/link-protocol.h index b2995e5..25ff12f 100644 --- a/link/link-protocol.h +++ b/link/link-protocol.h @@ -38,18 +38,24 @@ * go. The sender seals in it from the ACCEPT on; the receiver takes it once * a packet of it opens, and opens the late ones of the epoch before. So what * went in epoch 1 and on cannot be opened by one who later has both nodes' - * keys: only a run's first packets, before its first answer, are sealed + * keys: only a run's first packets, before its first epoch, are sealed * under the static keys. A new epoch every hour or GiB * (pure_link_sender_epochs). A receiver that remembers (pure_link_remember) * writes the epoch it accepted for each sender's run, and goes on in it after - * it starts again; a sender that hears nothing for 2 s over 4 probes (the - * receiver forgot) offers again in epoch 0, with no records, and the chain - * begins again from the nodes' keys. + * it starts again. One that does not know a run's epoch (it forgot) answers + * its packet with LOST, under the nodes' keys, naming the packet: its + * sender, if the packet is one it has out, begins again as a run begins, in + * epoch 0 with records until an epoch is accepted. A sender that hears + * nothing for 2 s over 4 probes (an attacker may keep the answers from it) + * offers again in epoch 0 with no records, and the chain begins again from + * the nodes' keys. * * What an attacker on the path may do, and what it gets (link/test-link.c * does each): - * - change a packet: it does not open, and it is answered by nothing - * (a packet is opened before it is looked at). + * - change a packet: it does not open, and it is answered by nothing but, + * when its run is one the receiver does not know in an epoch past 0, a + * LOST (as QUIC's stateless reset: smaller than the packet, LOSTS a + * second at most, telling nothing but that). * - send a packet of the past again: its records are applied once (the * heard numbers), its word is taken only as the most said (STORED's * limit), and it moves nothing: answers go where the newest packet of @@ -98,6 +104,8 @@ * same; one a second for a sender), then the first four * bytes of the key offered it answers (a sender that took * its epoch for lost offers epoch 1 again with another) + * 0x0b LOST u64 a packet's number: a receiver knows no epoch of its run + * (sealed in epoch 0, in a run of answers of its own) * 0x00 PADDING to the end of the packet: a full packet sent in a burst is * a whole datagram * Losses are detected as RFC 9002 says, with RACK's patience for packets diff --git a/link/test-link.c b/link/test-link.c index c409063..1d84804 100644 --- a/link/test-link.c +++ b/link/test-link.c @@ -90,6 +90,7 @@ typedef struct { /* A black hole: from when the sender's datagram grew, the path carries * nothing larger than PURE_LINK_DATAGRAM (and says nothing of it). */ bool black_hole, hole; + uint64_t forgot_at; /* when the receiver started again with no memo (it lost its epochs) */ /* The receiving app: its last heard (kept across the receiver's * restarts), what it said of deliveries (given at the next turn). */ uint64_t heard, applied, deliveries; @@ -200,9 +201,15 @@ typedef struct { bool black_hole; /* larger packets lost from when the path was found to carry them */ } Case; +/* Answers to a packet no one should answer: counted, but a LOST (a packet + * of 46 bytes: a run unknown here, in an epoch past 0, as QUIC's stateless + * reset) apart. */ +typedef struct { unsigned answers, lost; size_t lost_bytes; } Answers; static bool nothing_sent(void *context, const unsigned char *packet, size_t size, const unsigned char *to, size_t to_size) { - (void)packet, (void)size, (void)to, (void)to_size; - ++*(unsigned *)context; + (void)packet, (void)to, (void)to_size; + Answers *a = context; + if (size == 46) ++a->lost, a->lost_bytes += size; + else ++a->answers; return true; } @@ -218,7 +225,7 @@ static void tamper(World *w, PureLinkSender *s, PureLinkReceiver *r, const Packe else if (change == 1) size -= 1 + next_random() % 16; else bent[size++] = (unsigned char)next_random(); PureLinkCounts before, after; - unsigned answers = 0; + Answers answers = {0}; ++w->bent; if (p->to_receiver) { pure_link_receiver_counts(r, &before); @@ -230,8 +237,9 @@ static void tamper(World *w, PureLinkSender *s, PureLinkReceiver *r, const Packe pure_link_sender_take(s, bent, size, w->now); pure_link_sender_counts(s, &after); } - CHECK(after.received == before.received && !answers, "a packet changed on the way (%d, to the %s) %s", change, - p->to_receiver ? "receiver" : "sender", answers ? "made an answer" : "opened"); + CHECK(after.received == before.received && !answers.answers && answers.lost_bytes < size, + "a packet changed on the way (%d, to the %s) %s", change, p->to_receiver ? "receiver" : "sender", + answers.answers || answers.lost_bytes >= size ? "made an answer" : "opened"); } } @@ -275,7 +283,10 @@ static bool static_keys(const unsigned char secret[32], const unsigned char *fro /* Each packet the sender sent, tried under the static keys: one of an epoch * past 0 must not open, and one of epoch 0 sent after its run's first packet - * of a later epoch may carry no record. The number that opened. */ + * of a later epoch may carry no record. But for this: once the receiver + * started again with no memo (and said it knows no epoch), the run begins + * again, in epoch 0, until its next packet of a later epoch. The number + * that opened. */ static unsigned later_with_keys(World *w, const PureSealKey *a, const PureSealKey *b, const unsigned char *public_a, const unsigned char *public_b, const char *name) { unsigned char secret[32], again[32], key[PURE_AEAD_KEY], base[PURE_AEAD_NONCE], nonce[PURE_AEAD_NONCE], @@ -285,18 +296,30 @@ static unsigned later_with_keys(World *w, const PureSealKey *a, const PureSealKe unsigned opened = 0; for (size_t k = 0; k < w->sent_log_count; ++k) { const Packet *p = &w->sent_log[k]; - uint64_t run = get64le(p->bytes + 5), pn = get64le(p->bytes + 13), first_later = UINT64_MAX; - for (size_t j = 0; j < w->sent_log_count; ++j) /* its run's first packet of a later epoch */ - if (get64le(w->sent_log[j].bytes + 5) == run && (w->sent_log[j].bytes[0] & 15) && - get64le(w->sent_log[j].bytes + 13) < first_later) - first_later = get64le(w->sent_log[j].bytes + 13); + /* Its run's first packet of a later epoch; after the receiver forgot, + * its first packet of epoch 0 (begun again), and the first of a later + * epoch after that one. */ + uint64_t run = get64le(p->bytes + 5), pn = get64le(p->bytes + 13), first_later = UINT64_MAX, + again = UINT64_MAX, again_later = UINT64_MAX; + for (size_t j = 0; j < w->sent_log_count; ++j) { + const Packet *q = &w->sent_log[j]; + uint64_t n = get64le(q->bytes + 13); + if (get64le(q->bytes + 5) != run) continue; + if ((q->bytes[0] & 15) && n < first_later) first_later = n; + if (w->forgot_at && q->at > w->forgot_at && !(q->bytes[0] & 15) && n < again) again = n; + } + for (size_t j = 0; j < w->sent_log_count; ++j) { + const Packet *q = &w->sent_log[j]; + uint64_t n = get64le(q->bytes + 13); + if (get64le(q->bytes + 5) == run && (q->bytes[0] & 15) && n > again && n < again_later) again_later = n; + } if (!static_keys(secret, public_a, public_b, run, key, base)) exit(1); memcpy(nonce, base, PURE_AEAD_NONCE); for (int i = 0; i < 8; ++i) nonce[4 + i] ^= (unsigned char)(pn >> (56 - 8 * i)); if (!pure_aead_open(aead, key, nonce, p->bytes, 21, p->bytes + 21, p->size - 21, plain)) continue; ++opened; CHECK(!(p->bytes[0] & 15), "%s: a packet of epoch %u opened under the static keys", name, p->bytes[0] & 15); - if (pn < first_later) continue; + if (pn < first_later || (pn >= again && pn < again_later)) continue; size_t size = p->size - 21 - 16; for (size_t at = 0; at < size;) { /* its frames: none a record */ unsigned type = plain[at++]; @@ -391,7 +414,7 @@ static Outcome run(const Case *c, const char *dir) { w.sent_log_capacity = w.sent_log_capacity ? 2 * w.sent_log_capacity : 1024; if (!(w.sent_log = realloc(w.sent_log, w.sent_log_capacity * sizeof(Packet)))) exit(1); } - w.sent_log[w.sent_log_count] = (Packet){.size = size}; + w.sent_log[w.sent_log_count] = (Packet){.at = w.now, .size = size}; /* at: when it was sent */ memcpy(w.sent_log[w.sent_log_count++].bytes, out, size); put_packet(&w, true, out, size); if (w.black_hole && pure_link_sender_datagram(s) > PURE_LINK_DATAGRAM) w.hole = true; @@ -454,6 +477,7 @@ static Outcome run(const Case *c, const char *dir) { pure_link_receiver_free(r); w.done_count = 0; r = receiver_of(&w, b, public_b, public_a, keep, c->idle_ms); + if (!keep) w.forgot_at = w.now; } if (!sender_restarted && pure_link_sender_stored(s) >= c->sender_restart_after) { sender_restarted = true; @@ -574,7 +598,7 @@ static void flood(const char *dir) { for (int i = 0; i < (getenv("TEST_LINK_NO_FLOOD") ? 0 : 10000); ++i, w.now += 10000) { size_t size = 64 + next_random() % 1100; for (size_t k = 0; k < size; ++k) packet[k] = (unsigned char)next_random(); - packet[0] = 0x70; + packet[0] = (unsigned char)(0x70 | (i & 1)); /* half in an epoch past 0: a LOST each, while tokens last */ for (int j = 0; j < 4; ++j) packet[1 + j] = (unsigned char)(id >> 8 * j); w.attacker_in += size; pure_link_receiver_take(r, packet, size, "attacker", 8, w.now); @@ -605,9 +629,11 @@ static void flood(const char *dir) { (unsigned long long)pure_link_sender_due(s)); } } - CHECK(flooded.keyed <= 300 && flooded.secrets <= 1 && !w.attacker_out, /* 256 at once, 256 a second after */ - "flood: %llu runs keyed, %llu secrets made, %llu bytes to the attacker", (unsigned long long)flooded.keyed, - (unsigned long long)flooded.secrets, (unsigned long long)w.attacker_out); + CHECK(flooded.keyed <= 300 && flooded.secrets <= 1 && w.attacker_out <= 46 * (4096 + 410) && + w.attacker_out < w.attacker_in, /* keys: 256 at once, 256 a second; LOSTs: 4,096 at once, 4,096 a second */ + "flood: %llu runs keyed, %llu secrets made, %llu bytes to the attacker for its %llu", + (unsigned long long)flooded.keyed, (unsigned long long)flooded.secrets, (unsigned long long)w.attacker_out, + (unsigned long long)w.attacker_in); CHECK(w.heard == 200, "flood: the sender heard after it: %llu of 200", (unsigned long long)w.heard); if (getenv("TEST_LINK_DEBUG")) { PureLinkCounts rc, sc; @@ -617,8 +643,9 @@ static void flood(const char *dir) { (unsigned long long)rc.keyed, (unsigned long long)rc.active, (unsigned long long)sc.packets, (unsigned long long)sc.received, (unsigned long long)sc.stored); } - printf("{\"case\":\"flood\",\"packets\":10000,\"keyed\":%llu,\"secrets\":%llu,\"heard_after\":%llu}\n", - (unsigned long long)flooded.keyed, (unsigned long long)flooded.secrets, (unsigned long long)w.heard); + printf("{\"case\":\"flood\",\"packets\":10000,\"keyed\":%llu,\"secrets\":%llu,\"to_attacker\":%llu,\"heard_after\":%llu}\n", + (unsigned long long)flooded.keyed, (unsigned long long)flooded.secrets, (unsigned long long)w.attacker_out, + (unsigned long long)w.heard); pure_link_sender_free(s); pure_link_receiver_free(r); pure_seal_key_free(a), pure_seal_key_free(b); diff --git a/research/experiments/019-idle-links/2026-09-30-lost-linux-aarch64.jsonl b/research/experiments/019-idle-links/2026-09-30-lost-linux-aarch64.jsonl new file mode 100644 index 0000000..9c35dc7 --- /dev/null +++ b/research/experiments/019-idle-links/2026-09-30-lost-linux-aarch64.jsonl @@ -0,0 +1,2 @@ +{"kind":"no-memo, LOST","round":1,"row":{"devices": 10000, "period_ms": 5000, "memo": false, "moved": false, "restart_ms": 25.4, "steady_cpu_ms_per_s": 120.0, "senders_steady_cpu_ms_per_s": 70.0, "recovery_cpu_ms_first_10s": 3940.0, "senders_cpu_ms_first_10s": 2380.0, "generator": {"devices": 10000, "made": 160001, "stored": 160001, "sends": 170056, "again": 10055, "stored_ms_p50": 0.999, "stored_ms_p99": 5.748, "stored_ms_max": 57.352, "longest_gap_ms": 0, "gap_from_ms": 0, "keyed": 78339, "secrets": 0, "epochs": 14810, "p50_ms_by_second_made": [4.58, 4.65, 4.66, 4.68, 4.72, 0.96, 1.26, 1.32, 1.31, 1.35, 1.28, 1.26, 1.32, 1.29, 1.38, 1.23, 1.29, 1.28, 1.3, 1.4, 1.24, 1.32, 0.97, 0.86, 0.87, 0.75, 0.8, 1.21, 1.34, 1.39, 1.23, 1.21, 1.29, 1.32, 1.37, 1.04, 1.06, 1.12, 0.84, 0.81, 0.31, 0.33, 0.35, 0.47, 0.5, 0.37, 0.39, 0.42, 0.45, 0.46, 0.37, 0.41, 0.33, 0.34, 0.38, 0.29, 0.31, 0.42, 0.45, 0.47, 0.38, 4.96, 5.15, 5.23, 5.23, 5.11, 1.32, 1.35, 1.35, 1.4, 0.93, 0.73, 0.82, 0.81, 0.84, 0.97, 1.28, 1.32, 1.32, 1.35, 1.43], "p99_ms_by_second_made": [5.24, 5.6, 6.63, 5.76, 6.52, 2.92, 2.16, 1.91, 1.86, 1.74, 1.64, 1.82, 1.72, 1.91, 1.98, 1.57, 1.69, 1.75, 1.83, 1.84, 1.75, 3.89, 2.49, 1.32, 1.24, 1.81, 1.17, 1.88, 1.94, 1.86, 1.86, 1.85, 2.02, 1.88, 1.88, 2.52, 2.32, 2.28, 2.48, 1.62, 0.66, 0.65, 0.61, 0.75, 0.81, 0.76, 0.74, 0.72, 0.74, 0.74, 0.9, 1.24, 0.73, 0.8, 1.02, 0.58, 0.79, 0.87, 0.98, 0.76, 0.65, 57.28, 6.47, 7.21, 5.56, 6.66, 6.44, 1.75, 1.78, 1.94, 2.29, 1.13, 1.2, 1.19, 1.22, 1.52, 1.56, 1.68, 1.85, 1.85, 1.43]}, "timeline_100ms": [185, 190, 206, 206, 190, 209, 197, 202, 204, 188, 208, 203, 198, 202, 187, 204, 207, 186, 215, 204, 187, 195, 212, 188, 212, 205, 198, 189, 204, 203, 196, 203, 199, 204, 186, 220, 190, 201, 200, 201, 194, 202, 212, 197, 183, 215, 199, 192, 203, 194, 222, 191, 202, 202, 201, 202, 202, 194, 203, 202, 200, 192, 206, 195, 207, 193, 206, 196, 208, 194, 195, 208, 198, 195, 207, 195, 197, 198, 208, 195, 198, 208, 200, 197, 197, 197, 210, 198, 195, 199, 207, 197, 196, 199, 209, 197, 200, 198, 196, 200, 204, 205, 193, 206, 192, 205, 203, 193, 204, 205, 195, 195, 203, 207, 191, 203, 204, 195, 196, 208, 196, 207, 195, 195, 207, 194, 205, 196, 206, 197, 195, 199, 206, 197, 207, 196, 194, 211, 196, 196, 198, 211, 198, 202, 195, 197, 207, 200, 194, 196, 208, 203, 191, 209, 190, 204, 195, 204, 206, 192, 206, 198, 205, 195, 204, 192, 198, 206, 193, 211, 194, 197, 205, 198, 196, 205, 201, 203, 196, 197, 201, 198, 199, 206, 195, 198, 207, 195, 210, 194, 197, 198, 208, 197, 197, 196, 211, 196, 200, 198, 195, 207, 201, 194, 206, 194, 202, 206, 191, 207, 204, 194, 196, 208, 194, 206, 193, 199, 206, 189, 215, 190, 208, 197, 205, 194, 205, 194, 206, 193, 208, 196, 205, 195, 195, 207, 193, 208, 193, 206, 194, 208, 194, 196, 206, 196, 207, 196, 196, 205, 205, 196, 202, 193, 203, 204, 192, 203, 204, 193, 203, 207, 191, 205, 204, 194, 203, 193, 205, 205, 193, 197, 203, 206, 192, 198, 207, 196, 196, 210, 196, 198, 196, 213, 196, 195, 196, 201, 207, 196, 198, 201, 198, 209, 199, 199, 199, 199, 198, 201, 197, 209, 194, 195, 208, 194, 199, 211, 193, 196, 209, 198, 197, 197, 207, 195, 197, 206, 198, 207, 197, 199, 203, 198, 199, 195, 197, 209, 200, 197, 199, 197, 201, 196, 212, 200, 192, 200, 209, 197, 198, 197, 200, 199, 198, 201, 200, 198, 211, 198, 194, 204, 202, 193, 202, 206, 193, 201, 201, 203, 194, 206, 192, 205, 205, 194, 206, 202, 193, 194, 206, 197, 210, 192, 195, 203, 208, 193, 206, 196, 199, 208, 193, 206, 195, 206, 192, 203, 201, 196, 202, 198, 201, 207, 191, 205, 203, 197, 202, 195, 208, 197, 196, 207, 195, 205, 196, 196, 206, 196, 196, 209, 195, 197, 207, 196, 206, 196, 196, 208, 196, 198, 197, 207, 197, 197, 207, 197, 197, 198, 208, 191, 201, 202, 202, 202, 202, 191, 202, 203, 202, 193, 201, 205, 202, 192, 202, 204, 202, 193, 208, 200, 200, 200, 198, 199, 198, 198, 199, 210, 198, 200, 198, 200, 199, 199, 200, 200, 199, 200, 201, 201, 199, 201, 200, 200, 200, 201, 200, 201, 200, 199, 193, 203, 201, 201, 200, 202, 201, 201, 201, 191, 202, 202, 202, 201, 201, 191, 202, 202, 198, 200, 208, 193, 199, 200, 208, 198, 199, 198, 199, 201, 197, 203, 199, 199, 200, 198, 209, 197, 196, 197, 210, 199, 195, 196, 207, 195, 208, 197, 196, 198, 206, 198, 198, 207, 196, 196, 206, 191, 197, 207, 199, 208, 198, 197, 196, 208, 198, 198, 205, 197, 194, 206, 195, 204, 205, 194, 205, 195, 205, 197, 195, 208, 196, 195, 205, 195, 201, 208, 199, 201, 200, 191, 202, 199, 200, 200, 203, 203, 201, 191, 201, 204, 202, 201, 201, 200, 201, 192, 201, 203, 201, 202, 191, 202, 202, 202, 201, 203, 199, 200, 198, 200, 199, 197, 198, 200, 208, 199, 199, 200, 193, 188, 212, 202, 192, 197, 197, 212, 195, 207, 198, 196, 196, 195, 213, 197, 206, 181, 200, 214, 199, 194, 208, 198, 189, 208, 202, 197, 200, 202, 200, 198, 201, 200, 202, 200, 203, 203, 183, 202, 209, 200, 193, 204, 207, 193, 204, 197, 197, 213, 200, 195, 201, 207, 192, 209, 192, 205, 195, 197, 205, 195, 209, 194, 206, 194, 209, 195, 195, 207, 194, 209, 194, 195, 208, 198, 197, 207, 194, 199, 197, 209, 198, 196, 206, 199, 195, 203, 202, 195, 204, 202, 199, 203, 190, 202, 202, 202, 202, 192, 201, 202, 202, 203, 203, 191, 204, 202, 193, 203, 204, 193, 206, 194, 202, 204, 194, 205, 203, 195, 205, 192, 205, 204, 192, 205, 194, 205, 204, 195, 205, 196, 194, 206, 194, 205, 195, 206, 201, 192, 203, 202, 204, 191, 207, 194, 204, 202, 193, 205, 201, 194, 206, 195, 205, 195, 208, 202, 193, 196, 206, 194, 209, 195, 206, 196, 197, 206, 195, 197, 205, 193, 211, 194, 207, 195, 197, 197, 207, 198, 195, 209, 195, 197, 209, 196, 199, 196, 13], "killed_ms": 61202, "recovery": {"steady_per_100ms": 200.0, "silent_100ms_steps": 0, "caught_up_ms": 400}, "applied": 160001, "listener": {"listener_packets": 89974, "listener_received": 37582, "listener_deliveries": 37582, "listener_woken": 10000, "listener_remembered": 10000, "listener_swept": 0, "listener_active": 10000, "listener_keyed": 10000, "listener_secrets": 14829, "listener_epochs": 3529}}} +{"kind":"no-memo, LOST","round":2,"row":{"devices": 10000, "period_ms": 5000, "memo": false, "moved": false, "restart_ms": 18.3, "steady_cpu_ms_per_s": 100.0, "senders_steady_cpu_ms_per_s": 50.0, "recovery_cpu_ms_first_10s": 3780.0, "senders_cpu_ms_first_10s": 2220.0, "generator": {"devices": 10000, "made": 159999, "stored": 159999, "sends": 170032, "again": 10033, "stored_ms_p50": 0.959, "stored_ms_p99": 5.903, "stored_ms_max": 48.279, "longest_gap_ms": 0, "gap_from_ms": 0, "keyed": 78341, "secrets": 0, "epochs": 14811, "p50_ms_by_second_made": [4.77, 4.62, 4.69, 4.69, 4.72, 0.75, 0.91, 1.33, 1.32, 1.35, 1.22, 1.26, 1.3, 1.3, 1.31, 1.26, 1.22, 1.31, 1.27, 0.86, 0.72, 0.78, 0.82, 0.96, 1.38, 1.18, 1.27, 1.24, 1.17, 0.87, 0.77, 0.8, 0.82, 1.06, 1.35, 0.71, 0.76, 0.75, 0.79, 0.88, 0.31, 0.33, 0.37, 0.37, 0.4, 0.38, 0.4, 0.42, 0.45, 0.46, 0.38, 0.31, 0.32, 0.34, 0.36, 0.31, 0.4, 0.42, 0.45, 0.47, 0.3, 4.86, 5.08, 5.16, 5.21, 5.03, 1.03, 0.85, 0.82, 0.86, 1.26, 1.27, 1.29, 1.31, 1.35, 1.21, 1.27, 1.27, 1.35, 1.33, 0.0], "p99_ms_by_second_made": [5.23, 5.91, 20.4, 5.68, 5.98, 1.15, 1.62, 1.8, 1.77, 1.95, 1.58, 1.63, 1.88, 2.01, 1.74, 1.6, 1.71, 1.75, 2.18, 1.27, 1.13, 1.29, 1.22, 1.81, 1.99, 1.76, 1.71, 1.66, 1.98, 1.25, 1.27, 1.4, 1.38, 1.81, 2.47, 1.44, 1.46, 1.62, 1.54, 2.35, 0.65, 0.75, 1.02, 0.87, 0.82, 1.08, 0.72, 0.78, 0.77, 0.8, 1.41, 0.59, 0.56, 0.68, 0.66, 0.83, 0.95, 0.81, 0.77, 4.71, 1.59, 42.86, 6.34, 6.62, 11.12, 6.33, 6.07, 1.46, 1.26, 1.36, 1.64, 1.98, 1.68, 1.68, 1.84, 1.65, 1.65, 1.75, 1.69, 1.7, 0.0]}, "timeline_100ms": [180, 211, 198, 185, 206, 208, 193, 204, 202, 186, 210, 204, 198, 205, 188, 213, 191, 212, 183, 202, 206, 187, 218, 201, 189, 208, 184, 216, 192, 212, 184, 219, 182, 205, 208, 193, 189, 215, 188, 215, 184, 215, 188, 208, 192, 213, 196, 186, 216, 185, 225, 190, 202, 202, 202, 203, 191, 202, 202, 203, 202, 191, 204, 204, 192, 204, 203, 200, 192, 208, 194, 207, 194, 207, 196, 196, 209, 198, 197, 194, 200, 206, 194, 206, 195, 199, 208, 200, 197, 195, 201, 199, 199, 208, 200, 198, 196, 202, 197, 210, 192, 205, 193, 202, 201, 207, 190, 208, 191, 204, 197, 206, 204, 193, 205, 193, 207, 195, 208, 190, 209, 199, 195, 196, 207, 197, 195, 210, 193, 205, 195, 207, 194, 196, 208, 195, 198, 200, 211, 194, 196, 198, 210, 197, 196, 197, 209, 199, 195, 207, 196, 203, 194, 202, 203, 195, 206, 195, 204, 202, 193, 206, 194, 206, 193, 208, 191, 210, 194, 206, 193, 207, 195, 195, 208, 197, 194, 198, 210, 195, 208, 195, 194, 209, 195, 200, 204, 203, 192, 206, 195, 207, 194, 197, 206, 196, 204, 196, 208, 194, 204, 201, 197, 195, 201, 203, 203, 192, 204, 204, 193, 204, 203, 193, 203, 204, 192, 206, 194, 202, 204, 193, 206, 205, 191, 205, 205, 195, 195, 207, 194, 205, 194, 205, 194, 208, 193, 205, 200, 194, 202, 198, 200, 198, 210, 196, 197, 201, 200, 198, 208, 191, 205, 206, 193, 203, 195, 197, 207, 204, 195, 196, 204, 195, 206, 195, 205, 192, 202, 202, 197, 200, 206, 199, 205, 198, 196, 196, 212, 192, 196, 209, 197, 195, 200, 207, 194, 201, 210, 194, 195, 206, 196, 207, 193, 207, 195, 195, 210, 195, 206, 194, 203, 204, 192, 205, 193, 204, 205, 192, 204, 204, 192, 204, 196, 207, 195, 200, 201, 205, 194, 195, 207, 195, 205, 194, 205, 197, 208, 194, 195, 206, 195, 199, 205, 198, 196, 203, 198, 202, 201, 202, 199, 198, 197, 202, 205, 202, 197, 197, 209, 195, 202, 201, 203, 189, 203, 201, 204, 203, 190, 203, 201, 205, 190, 203, 205, 193, 203, 202, 203, 192, 206, 202, 190, 207, 203, 193, 204, 202, 192, 206, 203, 191, 204, 205, 192, 205, 193, 206, 194, 204, 205, 196, 202, 204, 192, 197, 205, 203, 198, 207, 195, 195, 207, 197, 199, 203, 196, 207, 195, 197, 208, 196, 195, 209, 195, 197, 208, 195, 197, 198, 209, 198, 197, 207, 196, 199, 202, 198, 194, 208, 197, 198, 207, 199, 198, 199, 200, 197, 198, 210, 198, 198, 198, 197, 202, 196, 200, 202, 201, 199, 209, 197, 198, 201, 202, 199, 199, 198, 199, 201, 200, 199, 199, 199, 199, 202, 199, 199, 200, 200, 200, 200, 200, 200, 200, 202, 202, 200, 201, 201, 201, 201, 201, 190, 200, 203, 203, 203, 201, 191, 203, 201, 201, 202, 201, 192, 202, 201, 198, 199, 208, 198, 197, 200, 203, 202, 201, 196, 195, 207, 195, 206, 195, 202, 200, 198, 207, 195, 196, 207, 197, 195, 206, 195, 207, 197, 198, 206, 197, 197, 197, 207, 195, 207, 197, 199, 197, 197, 209, 196, 197, 208, 196, 197, 197, 209, 198, 197, 206, 195, 205, 194, 195, 210, 196, 198, 201, 198, 200, 199, 199, 206, 204, 200, 199, 193, 201, 198, 200, 199, 202, 205, 203, 200, 190, 203, 201, 200, 200, 201, 203, 200, 201, 201, 200, 191, 205, 202, 202, 190, 202, 201, 202, 201, 191, 206, 206, 199, 197, 206, 192, 205, 195, 208, 195, 197, 207, 194, 206, 196, 181, 211, 195, 191, 216, 190, 200, 196, 214, 186, 202, 207, 204, 190, 210, 198, 206, 194, 196, 196, 209, 205, 194, 193, 210, 198, 204, 199, 197, 184, 206, 199, 204, 194, 214, 187, 210, 207, 187, 206, 197, 201, 197, 195, 206, 209, 191, 206, 199, 196, 209, 202, 203, 192, 207, 197, 196, 203, 207, 190, 204, 200, 203, 193, 202, 207, 193, 205, 193, 203, 205, 192, 204, 205, 194, 207, 195, 204, 193, 206, 194, 206, 193, 206, 194, 209, 195, 204, 190, 202, 204, 194, 201, 203, 201, 194, 205, 203, 194, 206, 194, 206, 203, 194, 206, 199, 192, 203, 205, 192, 205, 194, 203, 207, 193, 199, 205, 193, 210, 193, 197, 205, 195, 207, 193, 210, 194, 198, 204, 195, 198, 205, 196, 206, 197, 197, 199, 209, 192, 200, 204, 206, 190, 206, 204, 193, 203, 205, 192, 206, 191, 203, 207, 194, 206, 195, 195, 201, 206, 193, 204, 195, 208, 193, 206, 199, 195, 206, 194, 207, 197, 196, 204, 196, 205, 198, 196, 209, 195, 206, 196, 195, 206, 194, 211, 197, 197, 196, 11], "killed_ms": 61202, "recovery": {"steady_per_100ms": 200.1, "silent_100ms_steps": 0, "caught_up_ms": 500}, "applied": 159999, "listener": {"listener_packets": 89955, "listener_received": 37572, "listener_deliveries": 37572, "listener_woken": 10000, "listener_remembered": 10000, "listener_swept": 0, "listener_active": 10000, "listener_keyed": 10000, "listener_secrets": 14830, "listener_epochs": 3530}}} |