Commit Diff


commit - a8f2d00fef2add0808aaaa76f2e20c7662f8d08d3b7aa4e4162b54d6ca6764cc
commit + 4af73d3b7ab2f607d9116ad9e5691ded0d1b2112520a501a03ccffd085f95015
blob - e5ce5c43a3ed8dd1f584be733b3b244283830366f7bf798756ba057b481e4378
blob + 97604d4b32cb0e3183e1c028db2ebe108e669dc2fefd7aa95401709b2393ab9c
--- mqttctl/mqttctl.c	
+++ mqttctl/mqttctl.c	
@@ -565,21 +565,23 @@ show_clients_msg(struct imsg *imsg)
 		memcpy(&ci, imsg->data, sizeof(ci));
 		if (!header_done) {
 			printf("%-32s %-16s %-40s %-3s %-9s %-6s %-10s %5s %5s "
-			    "%6s %8s %8s\n", "CLIENT ID", "USER", "PEER", "VER",
-			    "STATE", "KEEP", "EXPIRY", "SUBS", "INFL", "QUEUED",
-			    "IN", "OUT");
+			    "%6s %8s %8s %6s\n", "CLIENT ID", "USER", "PEER",
+			    "VER", "STATE", "KEEP", "EXPIRY", "SUBS", "INFL",
+			    "QUEUED", "IN", "OUT", "DROP");
 			header_done = 1;
 		}
 		printf("%-32s %-16s %-40s %-3u %-9s %-6u %-10s %5u %5u %6u "
-		    "%8llu %8llu\n", ci.client_id,
+		    "%8llu %8llu %6llu\n", ci.client_id,
 		    ci.username[0] != '\0' ? ci.username : "-",
 		    ci.connected ? ci.peer : "-", ci.version,
-		    ci.connected ? (ci.tls ? "tls" : "connected") : "offline",
+		    !ci.connected ? "offline" : ci.congested ? "congested" :
+		    ci.tls ? "tls" : "connected",
 		    ci.keepalive, ci.expiry == UINT32_MAX ? "never" :
 		    fmt_duration(ci.expiry, ebuf, sizeof(ebuf)), ci.nsubs,
 		    ci.ninflight,
 		    ci.nqueued, (unsigned long long)ci.msgs_in,
-		    (unsigned long long)ci.msgs_out);
+		    (unsigned long long)ci.msgs_out,
+		    (unsigned long long)ci.dropped);
 		break;
 	case IMSG_CTL_SHOW_CLIENTS_END:
 		if (!header_done)
blob - 9191faedad566553cb5e16d345bc4033d5bc27b820ad8cf18dbad548b23bfd06
blob + 18bd283bec46709cfcdcc3b9c2cfc2af046694d7b2dfc6234faa78f0259ef4f1
--- mqttd/broker.c	
+++ mqttd/broker.c	
@@ -116,6 +116,7 @@ static struct imsgev
 		*broker_proc_iev(uint32_t);
 static void	 broker_dispatch_sockets(int, short, void *);
 static void	 broker_dispatch_listen(int, short, void *);
+static void	 broker_conn_congested(uint32_t, struct imsg *);
 static void	 broker_sighdlr(int, short, void *);
 static void	 broker_sweep(int, short, void *);
 static __dead void
@@ -720,6 +721,9 @@ broker_dispatch_sockets(int fd, short event, void *arg
 		case IMSG_CONN_ERROR:
 			broker_conn_error(proc, &imsg);
 			break;
+		case IMSG_CONN_CONGESTED:
+			broker_conn_congested(proc, &imsg);
+			break;
 		case IMSG_BRIDGE_FAILED:
 			bridge_failed(&imsg);
 			break;
@@ -1268,7 +1272,37 @@ broker_conn_error(uint32_t proc, struct imsg *imsg)
 	    MQTT_RC_PACKET_TOO_LARGE : MQTT_RC_MALFORMED, DETACH_WILL);
 }
 
+/*
+ * The worker's output buffer for this connection filled past the
+ * watermark, or drained again.  While congested, session_deliver drops
+ * QoS 0 and parks QoS 1 and 2 instead of writing to a peer that is not
+ * reading; the worker's overflow disconnect stays as the last resort.
+ * Disconnecting a slow primary host would cost a fleet-wide rebirth, so
+ * this is what stands between a stalled subscriber and that storm.
+ */
 static void
+broker_conn_congested(uint32_t proc, struct imsg *imsg)
+{
+	struct conn_congested	 cg;
+	struct bconn		*bc;
+
+	IMSG_SIZE_CHECK(imsg, &cg);
+	memcpy(&cg, imsg->data, sizeof(cg));
+
+	if ((bc = bconn_find(proc, cg.id)) == NULL)
+		return;
+	bc->congested = cg.congested ? 1 : 0;
+	if (bc->session == NULL)
+		return;
+
+	log_info("event=%s client=\"%s\" peer=%s",
+	    bc->congested ? "congested" : "drained", bc->session->client_id,
+	    bc->peer);
+	if (!bc->congested)
+		session_drain(bc->session);
+}
+
+static void
 broker_packet(struct bconn *bc, const uint8_t *data, size_t len)
 {
 	struct mqtt_fixed	 f;
@@ -2656,6 +2690,7 @@ broker_ctl_clients(struct imsg *imsg)
 			strlcpy(ci.peer, s->conn->peer, sizeof(ci.peer));
 			ci.connected = 1;
 			ci.tls = s->conn->tls;
+			ci.congested = s->conn->congested;
 		}
 		ci.version = s->version;
 		ci.clean_start = s->clean_start;
blob - ed91869bf43db9b52f5b83d8b114cd7bcafdd6bab73822168695d1002e204244
blob + 569a53fd9609ac3c09d9218a8c2c2db072255b9b9aa8b718ead5fdee87d1cb62
--- mqttd/broker.h	
+++ mqttd/broker.h	
@@ -97,6 +97,8 @@ struct bconn {
 	uint8_t			 tls;
 	uint8_t			 version;
 	uint8_t			 connected;
+	/* the worker's output buffer for this peer is over the watermark */
+	uint8_t			 congested;
 	char			 peer[MQTTD_MAXPEER];
 	char			 cert_user[MQTTD_MAXNAME];
 	struct session		*session;
blob - 7978b018372beb9abf030ddad5e13e1d12525ebb6c4eae1b54657567c789a535
blob + c6c31160370218d60e19973b6b0a85f172f46419bbefe81289d1aa8c007cd078
--- mqttd/mqttd.8	
+++ mqttd/mqttd.8	
@@ -145,6 +145,16 @@ authenticator configured with
 without one, v5 clients asking for it are rejected with reason code 0x8C.
 Bridges to other brokers and Sparkplug Aware behaviour are configured in
 .Xr mqttd.conf 5 .
+A subscriber that stops reading does not hold anyone else up: once the
+bytes queued for its connection exceed
+.Ic max packet size ,
+QoS 0 messages for it are dropped and counted, QoS 1 and 2 messages wait
+in its session queue up to
+.Ic max queued ,
+and delivery resumes when the connection has drained.
+A connection whose queue nevertheless grows past four times
+.Ic max packet size
+is closed as a slow consumer.
 On
 .Dv SIGHUP
 the configuration is re-read: limits, users, ACLs, TLS certificates and
blob - aa0076eb4971720889d62060520ab510e1b9ec70030b88de60bf07911957203d
blob + 7f4393bb862be5c2d156e9f2e8c0a25ad7e68e7235e6210a4468f91df270a816
--- mqttd/mqttd.conf.5	
+++ mqttd/mqttd.conf.5	
@@ -166,7 +166,8 @@ A
 block may override it.
 .It Ic max queued Ar number
 How many QoS 1 and QoS 2 messages are held for a persistent session
-while its client is offline or its inflight window is full.
+while its client is offline, its inflight window is full or its
+connection is congested.
 Further messages are dropped.
 The default is 1000.
 .It Ic max session expiry Ar number
blob - 3d800bb72daeaf70f6da6daadbc86ee090d11b05a477cb48f3ccf8dcbd0c6da5
blob + a7de620c1a192d8e54208be6dd227c9fb49875c475a5a6a0bb210e277182030f
--- mqttd/mqttd.h	
+++ mqttd/mqttd.h	
@@ -97,6 +97,7 @@
 #define MQTTD_CONNECT_TIMEOUT		10
 #define MQTTD_RETAIN_SWEEP		60
 #define MQTTD_OUTBUF_FACTOR		4
+#define MQTTD_CONGEST_FACTOR		1
 #define MQTTD_DEFAULT_SYS_INTERVAL	10
 #define MQTTD_VERSION			"1.0"
 
@@ -171,6 +172,7 @@ enum imsg_type {
 	IMSG_CONN_CLOSE,
 	IMSG_CONN_ERROR,
 	IMSG_CONN_KEEPALIVE,
+	IMSG_CONN_CONGESTED,
 
 	/* listen <-> sockets */
 	IMSG_CONN_ACCEPT,
@@ -563,6 +565,16 @@ struct conn_keepalive {
 };
 
 /*
+ * sockets -> broker: the bytes queued for this connection crossed
+ * MQTTD_CONGEST_FACTOR times the maximum packet size (congested 1), or
+ * the connection drained again (0).
+ */
+struct conn_congested {
+	uint32_t		 id;
+	uint8_t			 congested;
+};
+
+/*
  * listen -> sockets: a connected descriptor travels with this message.
  * bridge_id is set for an outbound bridge the worker has to run TLS and
  * MQTT over as a client; the PROXY header, if any, is already consumed
@@ -663,6 +675,7 @@ struct ctl_client_info {
 	uint8_t			 connected;
 	uint8_t			 clean_start;
 	uint8_t			 tls;
+	uint8_t			 congested;
 	uint16_t		 keepalive;
 	uint32_t		 expiry;
 	uint32_t		 nsubs;
blob - 5c4c72221089cc4d54e428c9cd24ae301b90169f39263afa0dad89dab3133d37
blob + df38f903acf73f445e297d30f3a94aed3fcaf5d19a5784b74d3a76be37a283bc
--- mqttd/session.c	
+++ mqttd/session.c	
@@ -617,19 +617,32 @@ session_incoming_remove(struct session *s, uint16_t pi
 	return (0);
 }
 
+/*
+ * Hand a message to a session.  QoS 0 goes straight out or, when the
+ * connection is congested, is dropped and counted; at most once delivery
+ * allows that.  A retained message sent because a subscription was just
+ * made is the exception: the spec requires it to reach the subscriber
+ * ([MQTT-3.3.1-9]), so it is written regardless.  QoS 1 and 2 go out
+ * while the inflight window has room and the connection is draining,
+ * otherwise they wait in the queue up to the session's limit.
+ */
 void
 session_deliver(struct session *s, struct delivery *d)
 {
 	d->session = s;
 
 	if (d->qos == 0) {
-		if (s->conn != NULL)
+		if (s->conn != NULL && s->conn->congested && !d->retain) {
+			s->dropped++;
+			broker_env->stats.pub_dropped++;
+		} else if (s->conn != NULL)
 			broker_send_publish(s, d, 0);
 		delivery_free(d);
 		return;
 	}
 
-	if (s->conn != NULL && s->ninflight < session_window(s)) {
+	if (s->conn != NULL && !s->conn->congested &&
+	    s->ninflight < session_window(s)) {
 		session_inflight_insert(s, d);
 		if (d->pid == 0 || broker_send_publish(s, d, 0) == -1) {
 			session_inflight_remove(s, d);
@@ -656,7 +669,8 @@ session_drain(struct session *s)
 {
 	struct delivery	*d;
 
-	while (s->conn != NULL && s->ninflight < session_window(s) &&
+	while (s->conn != NULL && !s->conn->congested &&
+	    s->ninflight < session_window(s) &&
 	    (d = TAILQ_FIRST(&s->queue)) != NULL) {
 		TAILQ_REMOVE(&s->queue, d, entry);
 		s->nqueued--;
blob - ae1924f7ff00a8f96fb2a9e875ab333f2925a00833f19b9d27f506bb673430e4
blob + a52aca6b81723f55b33587cbb7ff18eee1a8cc213fac3f239f1b92b2d18c6a01
--- mqttd/sockets.c	
+++ mqttd/sockets.c	
@@ -92,6 +92,7 @@ struct conn {
 	int			 closing;
 	int			 paused;
 	int			 throttled;
+	int			 congested;
 	/* websocket: decoded MQTT bytes in, one frame per packet out */
 	uint8_t			 ws;
 	struct evbuffer		*wsin;
@@ -148,6 +149,7 @@ static void	 conn_tls_readcb(int, short, void *);
 static void	 conn_tls_writecb(int, short, void *);
 static void	 conn_cert_user(struct tls *, char *, size_t);
 static void	 conn_throttle(struct conn *);
+static void	 conn_congested(struct conn *, int);
 static void	 sockets_unthrottle(void);
 static struct sbridge
 		*sbridge_find(uint32_t);
@@ -946,6 +948,27 @@ conn_cert_user(struct tls *ctx, char *buf, size_t len)
 	buf[n] = '\0';
 }
 
+/*
+ * This peer is not reading what we write.  Tell the broker, which drops
+ * QoS 0 for the connection and parks QoS 1 and 2 in the session queue
+ * until the buffer has drained, instead of piling up bytes here.
+ */
+static void
+conn_congested(struct conn *c, int on)
+{
+	struct conn_congested	 cg;
+
+	if (!c->opened)
+		return;
+	c->congested = on;
+	cg.id = c->id;
+	cg.congested = on;
+	imsg_compose_event(socks_iev_broker, IMSG_CONN_CONGESTED, 0, 0, -1,
+	    &cg, sizeof(cg));
+	log_debug("%s: connection %u from %s %s", __func__, c->id, c->peer,
+	    on ? "congested" : "drained");
+}
+
 /* the broker pipe is full: stop reading from this client for now */
 static void
 conn_throttle(struct conn *c)
@@ -1093,8 +1116,14 @@ conn_writecb(struct bufferevent *bev, void *arg)
 {
 	struct conn	*c = arg;
 
-	if (c->closing && EVBUFFER_LENGTH(EVBUFFER_OUTPUT(bev)) == 0)
+	if (EVBUFFER_LENGTH(EVBUFFER_OUTPUT(bev)) != 0)
+		return;
+	if (c->closing) {
 		conn_free(c, -1);
+		return;
+	}
+	if (c->congested)
+		conn_congested(c, 0);
 }
 
 static void
@@ -1129,12 +1158,14 @@ conn_timeout(int fd, short event, void *arg)
 /*
  * broker -> client: one chunk.  Plain connections take it as is; a
  * websocket collects the packet and sends it as one binary frame.
+ * A buffer over the congestion watermark is reported to the broker, one
+ * over the overflow watermark closes the connection.
  */
 static void
 conn_write_chunk(struct conn *c, const struct conn_pkt *hdr,
     const uint8_t *data)
 {
-	size_t	 hiwat;
+	size_t	 len, hiwat;
 
 	if (c->ws == WS_OPEN) {
 		if (hdr->offset == 0)
@@ -1150,12 +1181,17 @@ conn_write_chunk(struct conn *c, const struct conn_pkt
 	} else
 		bufferevent_write(c->bev, data, hdr->len);
 
+	len = EVBUFFER_LENGTH(EVBUFFER_OUTPUT(c->bev));
 	hiwat = (size_t)env->conf.max_packet_size * MQTTD_OUTBUF_FACTOR;
-	if (EVBUFFER_LENGTH(EVBUFFER_OUTPUT(c->bev)) > hiwat) {
+	if (len > hiwat) {
 		log_debug("%s: connection %u from %s is not draining",
 		    __func__, c->id, c->peer);
 		conn_free(c, CONN_CLOSE_OVERFLOW);
+		return;
 	}
+	if (!c->congested &&
+	    len > (size_t)env->conf.max_packet_size * MQTTD_CONGEST_FACTOR)
+		conn_congested(c, 1);
 }
 
 /* case-insensitive header lookup in a NUL terminated request */