commit 4af73d3b7ab2f607d9116ad9e5691ded0d1b2112520a501a03ccffd085f95015 from: Tracey Emery date: Tue Nov 10 16:31:23 2026 UTC per-connection backpressure for slow subscribers A sockets worker now tells the broker when the bytes queued for a connection pass the maximum packet size, and again when they have drained. While a connection is congested the broker drops QoS 0 for it, counting the drop against the session and the broker totals, and parks QoS 1 and 2 in the session queue under the existing "max queued" limit; delivery resumes on drain. Retained messages owed to a new subscription are still written, as [MQTT-3.3.1-9] requires. Until now a subscriber that stopped reading accumulated output until the worker cut it off at four times the packet size. For a Sparkplug primary host that disconnect means a reconnect and a fleet-wide rebirth, so the broker now sheds load per connection first; the overflow disconnect stays as the last resort. "mqttctl show clients" shows the congested state and the per-client drop count. commit - a8f2d00fef2add0808aaaa76f2e20c7662f8d08d3b7aa4e4162b54d6ca6764cc commit + 4af73d3b7ab2f607d9116ad9e5691ded0d1b2112520a501a03ccffd085f95015 blob - e5ce5c43a3ed8dd1f584be733b3b244283830366f7bf798756ba057b481e4378 blob + 97604d4b32cb0e3183e1c028db2ebe108e669dc2fefd7aa95401709b2393ab9c --- mqttctl/mqttctl.c +++ mqttctl/mqttctl.c @@ -565,21 +565,23 @@ show_clients_msg(struct imsg *imsg) memcpy(&ci, imsg->data, sizeof(ci)); if (!header_done) { printf("%-32s %-16s %-40s %-3s %-9s %-6s %-10s %5s %5s " - "%6s %8s %8s\n", "CLIENT ID", "USER", "PEER", "VER", - "STATE", "KEEP", "EXPIRY", "SUBS", "INFL", "QUEUED", - "IN", "OUT"); + "%6s %8s %8s %6s\n", "CLIENT ID", "USER", "PEER", + "VER", "STATE", "KEEP", "EXPIRY", "SUBS", "INFL", + "QUEUED", "IN", "OUT", "DROP"); header_done = 1; } printf("%-32s %-16s %-40s %-3u %-9s %-6u %-10s %5u %5u %6u " - "%8llu %8llu\n", ci.client_id, + "%8llu %8llu %6llu\n", ci.client_id, ci.username[0] != '\0' ? ci.username : "-", ci.connected ? ci.peer : "-", ci.version, - ci.connected ? (ci.tls ? "tls" : "connected") : "offline", + !ci.connected ? "offline" : ci.congested ? "congested" : + ci.tls ? "tls" : "connected", ci.keepalive, ci.expiry == UINT32_MAX ? "never" : fmt_duration(ci.expiry, ebuf, sizeof(ebuf)), ci.nsubs, ci.ninflight, ci.nqueued, (unsigned long long)ci.msgs_in, - (unsigned long long)ci.msgs_out); + (unsigned long long)ci.msgs_out, + (unsigned long long)ci.dropped); break; case IMSG_CTL_SHOW_CLIENTS_END: if (!header_done) blob - 9191faedad566553cb5e16d345bc4033d5bc27b820ad8cf18dbad548b23bfd06 blob + 18bd283bec46709cfcdcc3b9c2cfc2af046694d7b2dfc6234faa78f0259ef4f1 --- mqttd/broker.c +++ mqttd/broker.c @@ -116,6 +116,7 @@ static struct imsgev *broker_proc_iev(uint32_t); static void broker_dispatch_sockets(int, short, void *); static void broker_dispatch_listen(int, short, void *); +static void broker_conn_congested(uint32_t, struct imsg *); static void broker_sighdlr(int, short, void *); static void broker_sweep(int, short, void *); static __dead void @@ -720,6 +721,9 @@ broker_dispatch_sockets(int fd, short event, void *arg case IMSG_CONN_ERROR: broker_conn_error(proc, &imsg); break; + case IMSG_CONN_CONGESTED: + broker_conn_congested(proc, &imsg); + break; case IMSG_BRIDGE_FAILED: bridge_failed(&imsg); break; @@ -1268,7 +1272,37 @@ broker_conn_error(uint32_t proc, struct imsg *imsg) MQTT_RC_PACKET_TOO_LARGE : MQTT_RC_MALFORMED, DETACH_WILL); } +/* + * The worker's output buffer for this connection filled past the + * watermark, or drained again. While congested, session_deliver drops + * QoS 0 and parks QoS 1 and 2 instead of writing to a peer that is not + * reading; the worker's overflow disconnect stays as the last resort. + * Disconnecting a slow primary host would cost a fleet-wide rebirth, so + * this is what stands between a stalled subscriber and that storm. + */ static void +broker_conn_congested(uint32_t proc, struct imsg *imsg) +{ + struct conn_congested cg; + struct bconn *bc; + + IMSG_SIZE_CHECK(imsg, &cg); + memcpy(&cg, imsg->data, sizeof(cg)); + + if ((bc = bconn_find(proc, cg.id)) == NULL) + return; + bc->congested = cg.congested ? 1 : 0; + if (bc->session == NULL) + return; + + log_info("event=%s client=\"%s\" peer=%s", + bc->congested ? "congested" : "drained", bc->session->client_id, + bc->peer); + if (!bc->congested) + session_drain(bc->session); +} + +static void broker_packet(struct bconn *bc, const uint8_t *data, size_t len) { struct mqtt_fixed f; @@ -2656,6 +2690,7 @@ broker_ctl_clients(struct imsg *imsg) strlcpy(ci.peer, s->conn->peer, sizeof(ci.peer)); ci.connected = 1; ci.tls = s->conn->tls; + ci.congested = s->conn->congested; } ci.version = s->version; ci.clean_start = s->clean_start; blob - ed91869bf43db9b52f5b83d8b114cd7bcafdd6bab73822168695d1002e204244 blob + 569a53fd9609ac3c09d9218a8c2c2db072255b9b9aa8b718ead5fdee87d1cb62 --- mqttd/broker.h +++ mqttd/broker.h @@ -97,6 +97,8 @@ struct bconn { uint8_t tls; uint8_t version; uint8_t connected; + /* the worker's output buffer for this peer is over the watermark */ + uint8_t congested; char peer[MQTTD_MAXPEER]; char cert_user[MQTTD_MAXNAME]; struct session *session; blob - 7978b018372beb9abf030ddad5e13e1d12525ebb6c4eae1b54657567c789a535 blob + c6c31160370218d60e19973b6b0a85f172f46419bbefe81289d1aa8c007cd078 --- mqttd/mqttd.8 +++ mqttd/mqttd.8 @@ -145,6 +145,16 @@ authenticator configured with without one, v5 clients asking for it are rejected with reason code 0x8C. Bridges to other brokers and Sparkplug Aware behaviour are configured in .Xr mqttd.conf 5 . +A subscriber that stops reading does not hold anyone else up: once the +bytes queued for its connection exceed +.Ic max packet size , +QoS 0 messages for it are dropped and counted, QoS 1 and 2 messages wait +in its session queue up to +.Ic max queued , +and delivery resumes when the connection has drained. +A connection whose queue nevertheless grows past four times +.Ic max packet size +is closed as a slow consumer. On .Dv SIGHUP the configuration is re-read: limits, users, ACLs, TLS certificates and blob - aa0076eb4971720889d62060520ab510e1b9ec70030b88de60bf07911957203d blob + 7f4393bb862be5c2d156e9f2e8c0a25ad7e68e7235e6210a4468f91df270a816 --- mqttd/mqttd.conf.5 +++ mqttd/mqttd.conf.5 @@ -166,7 +166,8 @@ A block may override it. .It Ic max queued Ar number How many QoS 1 and QoS 2 messages are held for a persistent session -while its client is offline or its inflight window is full. +while its client is offline, its inflight window is full or its +connection is congested. Further messages are dropped. The default is 1000. .It Ic max session expiry Ar number blob - 3d800bb72daeaf70f6da6daadbc86ee090d11b05a477cb48f3ccf8dcbd0c6da5 blob + a7de620c1a192d8e54208be6dd227c9fb49875c475a5a6a0bb210e277182030f --- mqttd/mqttd.h +++ mqttd/mqttd.h @@ -97,6 +97,7 @@ #define MQTTD_CONNECT_TIMEOUT 10 #define MQTTD_RETAIN_SWEEP 60 #define MQTTD_OUTBUF_FACTOR 4 +#define MQTTD_CONGEST_FACTOR 1 #define MQTTD_DEFAULT_SYS_INTERVAL 10 #define MQTTD_VERSION "1.0" @@ -171,6 +172,7 @@ enum imsg_type { IMSG_CONN_CLOSE, IMSG_CONN_ERROR, IMSG_CONN_KEEPALIVE, + IMSG_CONN_CONGESTED, /* listen <-> sockets */ IMSG_CONN_ACCEPT, @@ -563,6 +565,16 @@ struct conn_keepalive { }; /* + * sockets -> broker: the bytes queued for this connection crossed + * MQTTD_CONGEST_FACTOR times the maximum packet size (congested 1), or + * the connection drained again (0). + */ +struct conn_congested { + uint32_t id; + uint8_t congested; +}; + +/* * listen -> sockets: a connected descriptor travels with this message. * bridge_id is set for an outbound bridge the worker has to run TLS and * MQTT over as a client; the PROXY header, if any, is already consumed @@ -663,6 +675,7 @@ struct ctl_client_info { uint8_t connected; uint8_t clean_start; uint8_t tls; + uint8_t congested; uint16_t keepalive; uint32_t expiry; uint32_t nsubs; blob - 5c4c72221089cc4d54e428c9cd24ae301b90169f39263afa0dad89dab3133d37 blob + df38f903acf73f445e297d30f3a94aed3fcaf5d19a5784b74d3a76be37a283bc --- mqttd/session.c +++ mqttd/session.c @@ -617,19 +617,32 @@ session_incoming_remove(struct session *s, uint16_t pi return (0); } +/* + * Hand a message to a session. QoS 0 goes straight out or, when the + * connection is congested, is dropped and counted; at most once delivery + * allows that. A retained message sent because a subscription was just + * made is the exception: the spec requires it to reach the subscriber + * ([MQTT-3.3.1-9]), so it is written regardless. QoS 1 and 2 go out + * while the inflight window has room and the connection is draining, + * otherwise they wait in the queue up to the session's limit. + */ void session_deliver(struct session *s, struct delivery *d) { d->session = s; if (d->qos == 0) { - if (s->conn != NULL) + if (s->conn != NULL && s->conn->congested && !d->retain) { + s->dropped++; + broker_env->stats.pub_dropped++; + } else if (s->conn != NULL) broker_send_publish(s, d, 0); delivery_free(d); return; } - if (s->conn != NULL && s->ninflight < session_window(s)) { + if (s->conn != NULL && !s->conn->congested && + s->ninflight < session_window(s)) { session_inflight_insert(s, d); if (d->pid == 0 || broker_send_publish(s, d, 0) == -1) { session_inflight_remove(s, d); @@ -656,7 +669,8 @@ session_drain(struct session *s) { struct delivery *d; - while (s->conn != NULL && s->ninflight < session_window(s) && + while (s->conn != NULL && !s->conn->congested && + s->ninflight < session_window(s) && (d = TAILQ_FIRST(&s->queue)) != NULL) { TAILQ_REMOVE(&s->queue, d, entry); s->nqueued--; blob - ae1924f7ff00a8f96fb2a9e875ab333f2925a00833f19b9d27f506bb673430e4 blob + a52aca6b81723f55b33587cbb7ff18eee1a8cc213fac3f239f1b92b2d18c6a01 --- mqttd/sockets.c +++ mqttd/sockets.c @@ -92,6 +92,7 @@ struct conn { int closing; int paused; int throttled; + int congested; /* websocket: decoded MQTT bytes in, one frame per packet out */ uint8_t ws; struct evbuffer *wsin; @@ -148,6 +149,7 @@ static void conn_tls_readcb(int, short, void *); static void conn_tls_writecb(int, short, void *); static void conn_cert_user(struct tls *, char *, size_t); static void conn_throttle(struct conn *); +static void conn_congested(struct conn *, int); static void sockets_unthrottle(void); static struct sbridge *sbridge_find(uint32_t); @@ -946,6 +948,27 @@ conn_cert_user(struct tls *ctx, char *buf, size_t len) buf[n] = '\0'; } +/* + * This peer is not reading what we write. Tell the broker, which drops + * QoS 0 for the connection and parks QoS 1 and 2 in the session queue + * until the buffer has drained, instead of piling up bytes here. + */ +static void +conn_congested(struct conn *c, int on) +{ + struct conn_congested cg; + + if (!c->opened) + return; + c->congested = on; + cg.id = c->id; + cg.congested = on; + imsg_compose_event(socks_iev_broker, IMSG_CONN_CONGESTED, 0, 0, -1, + &cg, sizeof(cg)); + log_debug("%s: connection %u from %s %s", __func__, c->id, c->peer, + on ? "congested" : "drained"); +} + /* the broker pipe is full: stop reading from this client for now */ static void conn_throttle(struct conn *c) @@ -1093,8 +1116,14 @@ conn_writecb(struct bufferevent *bev, void *arg) { struct conn *c = arg; - if (c->closing && EVBUFFER_LENGTH(EVBUFFER_OUTPUT(bev)) == 0) + if (EVBUFFER_LENGTH(EVBUFFER_OUTPUT(bev)) != 0) + return; + if (c->closing) { conn_free(c, -1); + return; + } + if (c->congested) + conn_congested(c, 0); } static void @@ -1129,12 +1158,14 @@ conn_timeout(int fd, short event, void *arg) /* * broker -> client: one chunk. Plain connections take it as is; a * websocket collects the packet and sends it as one binary frame. + * A buffer over the congestion watermark is reported to the broker, one + * over the overflow watermark closes the connection. */ static void conn_write_chunk(struct conn *c, const struct conn_pkt *hdr, const uint8_t *data) { - size_t hiwat; + size_t len, hiwat; if (c->ws == WS_OPEN) { if (hdr->offset == 0) @@ -1150,12 +1181,17 @@ conn_write_chunk(struct conn *c, const struct conn_pkt } else bufferevent_write(c->bev, data, hdr->len); + len = EVBUFFER_LENGTH(EVBUFFER_OUTPUT(c->bev)); hiwat = (size_t)env->conf.max_packet_size * MQTTD_OUTBUF_FACTOR; - if (EVBUFFER_LENGTH(EVBUFFER_OUTPUT(c->bev)) > hiwat) { + if (len > hiwat) { log_debug("%s: connection %u from %s is not draining", __func__, c->id, c->peer); conn_free(c, CONN_CLOSE_OVERFLOW); + return; } + if (!c->congested && + len > (size_t)env->conf.max_packet_size * MQTTD_CONGEST_FACTOR) + conn_congested(c, 1); } /* case-insensitive header lookup in a NUL terminated request */