Send no packet only to probe the path or say a challenge back

M2 (ten thousand devices, a record each every 5 s) sent 118k packets for
80k records. A link whose packets were never full still probed for a
1,452-byte path: nothing larger would have been carried. And a sender
said a receiver's challenge back in a packet of its own, one for each
device each time the receiver started. Now a link probes once a packet
of its was full, and a challenge rides in the next packet that goes
(records, or a probe). Until then the receiver's answers there are at
most three times what came, which the answers to one packet are not;
when they are held back, the sender's probe or record sent again says it
back. M2 sends 80,036 again (docs/research/099-link-epochs.md).

2 files changed, 24 insertions, 14 deletions

diff --git a/link/link-protocol.c b/link/link-protocol.c
index 92cd149..3cb8e07 100644
--- a/link/link-protocol.c
+++ b/link/link-protocol.c
@@ -241,7 +241,7 @@ struct PureLinkSender {
     /* The datagram's size now; a probe of the largest flies; when the next
      * may go, and how many were lost (RFC 8899). */
     uint32_t datagram;
-    bool reaching;
+    bool reaching, filled; /* filled: a packet was full with more to go (a larger one would carry more) */
     uint64_t reach_at;
     unsigned reach_lost;
     uint64_t received;    /* parts received and not yet stored */
@@ -666,20 +666,24 @@ void pure_link_sender_take(PureLinkSender *l, const unsigned char *packet, size_
     }
 }
 
-/* A packet of what is not sent yet, in order, within the receiver's credit:
- * its body's size, or 0 when there is none. */
-static size_t fill(PureLinkSender *l, unsigned char *body, Flight *f, bool pad) {
+/* A packet of what is not sent yet, in order, within the receiver's credit
+ * (or a probe's, which goes all the same): its body's size, or 0 when there
+ * is none. */
+static size_t fill(PureLinkSender *l, unsigned char *body, Flight *f, bool pad, bool probe) {
     size_t at = 0, most = l->datagram - HEADER - PURE_AEAD_TAG;
     body[at++] = F_STREAM;
     at += put_names(body + at, l->source, l->input);
     put32(body + at, l->shape);
     at += 4;
-    bool responded = l->responding;
-    for (unsigned k = 0; k < l->responding; ++k, at += 9) { /* the receiver's challenges, said back */
+    /* The receiver's challenges, said back in the next packet that goes,
+     * never in one of their own: that would be a packet more for each
+     * address checked. Until then the answers there are at most three
+     * times what came from it, and the answers to a packet are less; when
+     * they are held back, a probe or a record sent again says it back. */
+    for (unsigned k = 0; k < l->responding; ++k, at += 9) { /* let go once the packet goes */
         body[at] = F_RESPONSE;
         put64(body + at + 1, l->respond[k]);
     }
-    l->responding = 0;
     /* The credit counts from the oldest record not stored: after a restart
      * the sender does not know yet how far the receiver stored. */
     uint64_t base = l->count ? record_at(l, 0)->seq - 1 : l->stored;
@@ -741,6 +745,7 @@ static size_t fill(PureLinkSender *l, unsigned char *body, Flight *f, bool pad)
         if (before && !full) l->unsent_from = r->seq + 1;
         before = before && !full;
         if (full) { /* the packet is full: whole, so it can go in a burst */
+            l->filled = true;
             if (pad && f->count) {
                 memset(body + at, F_PADDING, most - at);
                 at = most;
@@ -748,7 +753,9 @@ static size_t fill(PureLinkSender *l, unsigned char *body, Flight *f, bool pad)
             break;
         }
     }
-    return f->count || responded ? at : 0;
+    if (!f->count && !(probe && l->responding)) return 0;
+    l->responding = 0;
+    return at;
 }
 
 /* Its number is spent all the same. */
@@ -813,7 +820,7 @@ size_t pure_link_sender_packet(PureLinkSender *l, unsigned char out[PURE_LINK_DA
     if (!probe && !paced(l, now)) return 0;
     unsigned char body[BODY_MOST];
     Flight f = {.pn = l->next_pn, .eliciting = true};
-    if (l->datagram < DATAGRAM_MOST && !l->reaching && l->sampled && now >= l->reach_at &&
+    if (l->datagram < DATAGRAM_MOST && l->filled && !l->reaching && l->sampled && now >= l->reach_at &&
         l->in_flight + DATAGRAM_MOST <= l->window) { /* the path may carry more: a PING of the most, padded */
         body[0] = F_PING;
         memset(body + 1, F_PADDING, BODY_MOST - 1);
@@ -824,7 +831,7 @@ size_t pure_link_sender_packet(PureLinkSender *l, unsigned char out[PURE_LINK_DA
         *number = f.pn;
         return total;
     }
-    size_t size = fill(l, body, &f, whole);
+    size_t size = fill(l, body, &f, whole, probe);
     if (!size) {
         l->unused = true;
         if (!probe) return 0;
diff --git a/link/link-protocol.h b/link/link-protocol.h
index 9b4e63d..f121b2e 100644
--- a/link/link-protocol.h
+++ b/link/link-protocol.h
@@ -68,7 +68,8 @@
  *                whole, with 1 to 3 bytes of framing, not the 19 of RECORD
  *   0x07 CHALLENGE u64: a receiver's, until its sender says it back from
  *                the address the answers go to
- *   0x08 RESPONSE u64: a challenge said back (the last few had)
+ *   0x08 RESPONSE u64: a challenge said back (the last few had), in the next
+ *                packet that goes (records, or a probe)
  *   0x00 PADDING to the end of the packet: a full packet sent in a burst is
  *                a whole datagram
  * Losses are detected as RFC 9002 says, with RACK's patience for packets
@@ -76,9 +77,11 @@
 
 /* A packet is PURE_LINK_DATAGRAM bytes at most at first, and up to
  * PURE_LINK_DATAGRAM_MOST once the path is found to carry that many (a
- * probe of that size acknowledged: DPLPMTUD, RFC 8899); a source's or an
- * input's name is 1 to PURE_LINK_NAME bytes; a receiver keeps where a
- * packet came from (the socket's address) in PURE_LINK_ADDRESS bytes. */
+ * probe of that size acknowledged: DPLPMTUD, RFC 8899; a link probes once
+ * a packet of its was full, else a larger one would carry nothing more); a
+ * source's or an input's name is 1 to PURE_LINK_NAME bytes; a receiver
+ * keeps where a packet came from (the socket's address) in
+ * PURE_LINK_ADDRESS bytes. */
 enum { PURE_LINK_DATAGRAM = 1200, PURE_LINK_DATAGRAM_MOST = 1452, PURE_LINK_NAME = 64, PURE_LINK_ADDRESS = 28 };
 
 /* Counts for a host's report: packets and records sent, records sent again,