commit - a8f2d00fef2add0808aaaa76f2e20c7662f8d08d3b7aa4e4162b54d6ca6764cc
commit + 4af73d3b7ab2f607d9116ad9e5691ded0d1b2112520a501a03ccffd085f95015
blob - e5ce5c43a3ed8dd1f584be733b3b244283830366f7bf798756ba057b481e4378
blob + 97604d4b32cb0e3183e1c028db2ebe108e669dc2fefd7aa95401709b2393ab9c
--- mqttctl/mqttctl.c
+++ mqttctl/mqttctl.c
memcpy(&ci, imsg->data, sizeof(ci));
if (!header_done) {
printf("%-32s %-16s %-40s %-3s %-9s %-6s %-10s %5s %5s "
- "%6s %8s %8s\n", "CLIENT ID", "USER", "PEER", "VER",
- "STATE", "KEEP", "EXPIRY", "SUBS", "INFL", "QUEUED",
- "IN", "OUT");
+ "%6s %8s %8s %6s\n", "CLIENT ID", "USER", "PEER",
+ "VER", "STATE", "KEEP", "EXPIRY", "SUBS", "INFL",
+ "QUEUED", "IN", "OUT", "DROP");
header_done = 1;
}
printf("%-32s %-16s %-40s %-3u %-9s %-6u %-10s %5u %5u %6u "
- "%8llu %8llu\n", ci.client_id,
+ "%8llu %8llu %6llu\n", ci.client_id,
ci.username[0] != '\0' ? ci.username : "-",
ci.connected ? ci.peer : "-", ci.version,
- ci.connected ? (ci.tls ? "tls" : "connected") : "offline",
+ !ci.connected ? "offline" : ci.congested ? "congested" :
+ ci.tls ? "tls" : "connected",
ci.keepalive, ci.expiry == UINT32_MAX ? "never" :
fmt_duration(ci.expiry, ebuf, sizeof(ebuf)), ci.nsubs,
ci.ninflight,
ci.nqueued, (unsigned long long)ci.msgs_in,
- (unsigned long long)ci.msgs_out);
+ (unsigned long long)ci.msgs_out,
+ (unsigned long long)ci.dropped);
break;
case IMSG_CTL_SHOW_CLIENTS_END:
if (!header_done)
blob - 9191faedad566553cb5e16d345bc4033d5bc27b820ad8cf18dbad548b23bfd06
blob + 18bd283bec46709cfcdcc3b9c2cfc2af046694d7b2dfc6234faa78f0259ef4f1
--- mqttd/broker.c
+++ mqttd/broker.c
*broker_proc_iev(uint32_t);
static void broker_dispatch_sockets(int, short, void *);
static void broker_dispatch_listen(int, short, void *);
+static void broker_conn_congested(uint32_t, struct imsg *);
static void broker_sighdlr(int, short, void *);
static void broker_sweep(int, short, void *);
static __dead void
case IMSG_CONN_ERROR:
broker_conn_error(proc, &imsg);
break;
+ case IMSG_CONN_CONGESTED:
+ broker_conn_congested(proc, &imsg);
+ break;
case IMSG_BRIDGE_FAILED:
bridge_failed(&imsg);
break;
MQTT_RC_PACKET_TOO_LARGE : MQTT_RC_MALFORMED, DETACH_WILL);
}
+/*
+ * The worker's output buffer for this connection filled past the
+ * watermark, or drained again. While congested, session_deliver drops
+ * QoS 0 and parks QoS 1 and 2 instead of writing to a peer that is not
+ * reading; the worker's overflow disconnect stays as the last resort.
+ * Disconnecting a slow primary host would cost a fleet-wide rebirth, so
+ * this is what stands between a stalled subscriber and that storm.
+ */
static void
+broker_conn_congested(uint32_t proc, struct imsg *imsg)
+{
+ struct conn_congested cg;
+ struct bconn *bc;
+
+ IMSG_SIZE_CHECK(imsg, &cg);
+ memcpy(&cg, imsg->data, sizeof(cg));
+
+ if ((bc = bconn_find(proc, cg.id)) == NULL)
+ return;
+ bc->congested = cg.congested ? 1 : 0;
+ if (bc->session == NULL)
+ return;
+
+ log_info("event=%s client=\"%s\" peer=%s",
+ bc->congested ? "congested" : "drained", bc->session->client_id,
+ bc->peer);
+ if (!bc->congested)
+ session_drain(bc->session);
+}
+
+static void
broker_packet(struct bconn *bc, const uint8_t *data, size_t len)
{
struct mqtt_fixed f;
strlcpy(ci.peer, s->conn->peer, sizeof(ci.peer));
ci.connected = 1;
ci.tls = s->conn->tls;
+ ci.congested = s->conn->congested;
}
ci.version = s->version;
ci.clean_start = s->clean_start;
blob - ed91869bf43db9b52f5b83d8b114cd7bcafdd6bab73822168695d1002e204244
blob + 569a53fd9609ac3c09d9218a8c2c2db072255b9b9aa8b718ead5fdee87d1cb62
--- mqttd/broker.h
+++ mqttd/broker.h
uint8_t tls;
uint8_t version;
uint8_t connected;
+ /* the worker's output buffer for this peer is over the watermark */
+ uint8_t congested;
char peer[MQTTD_MAXPEER];
char cert_user[MQTTD_MAXNAME];
struct session *session;
blob - 7978b018372beb9abf030ddad5e13e1d12525ebb6c4eae1b54657567c789a535
blob + c6c31160370218d60e19973b6b0a85f172f46419bbefe81289d1aa8c007cd078
--- mqttd/mqttd.8
+++ mqttd/mqttd.8
without one, v5 clients asking for it are rejected with reason code 0x8C.
Bridges to other brokers and Sparkplug Aware behaviour are configured in
.Xr mqttd.conf 5 .
+A subscriber that stops reading does not hold anyone else up: once the
+bytes queued for its connection exceed
+.Ic max packet size ,
+QoS 0 messages for it are dropped and counted, QoS 1 and 2 messages wait
+in its session queue up to
+.Ic max queued ,
+and delivery resumes when the connection has drained.
+A connection whose queue nevertheless grows past four times
+.Ic max packet size
+is closed as a slow consumer.
On
.Dv SIGHUP
the configuration is re-read: limits, users, ACLs, TLS certificates and
blob - aa0076eb4971720889d62060520ab510e1b9ec70030b88de60bf07911957203d
blob + 7f4393bb862be5c2d156e9f2e8c0a25ad7e68e7235e6210a4468f91df270a816
--- mqttd/mqttd.conf.5
+++ mqttd/mqttd.conf.5
block may override it.
.It Ic max queued Ar number
How many QoS 1 and QoS 2 messages are held for a persistent session
-while its client is offline or its inflight window is full.
+while its client is offline, its inflight window is full or its
+connection is congested.
Further messages are dropped.
The default is 1000.
.It Ic max session expiry Ar number
blob - 3d800bb72daeaf70f6da6daadbc86ee090d11b05a477cb48f3ccf8dcbd0c6da5
blob + a7de620c1a192d8e54208be6dd227c9fb49875c475a5a6a0bb210e277182030f
--- mqttd/mqttd.h
+++ mqttd/mqttd.h
#define MQTTD_CONNECT_TIMEOUT 10
#define MQTTD_RETAIN_SWEEP 60
#define MQTTD_OUTBUF_FACTOR 4
+#define MQTTD_CONGEST_FACTOR 1
#define MQTTD_DEFAULT_SYS_INTERVAL 10
#define MQTTD_VERSION "1.0"
IMSG_CONN_CLOSE,
IMSG_CONN_ERROR,
IMSG_CONN_KEEPALIVE,
+ IMSG_CONN_CONGESTED,
/* listen <-> sockets */
IMSG_CONN_ACCEPT,
};
/*
+ * sockets -> broker: the bytes queued for this connection crossed
+ * MQTTD_CONGEST_FACTOR times the maximum packet size (congested 1), or
+ * the connection drained again (0).
+ */
+struct conn_congested {
+ uint32_t id;
+ uint8_t congested;
+};
+
+/*
* listen -> sockets: a connected descriptor travels with this message.
* bridge_id is set for an outbound bridge the worker has to run TLS and
* MQTT over as a client; the PROXY header, if any, is already consumed
uint8_t connected;
uint8_t clean_start;
uint8_t tls;
+ uint8_t congested;
uint16_t keepalive;
uint32_t expiry;
uint32_t nsubs;
blob - 5c4c72221089cc4d54e428c9cd24ae301b90169f39263afa0dad89dab3133d37
blob + df38f903acf73f445e297d30f3a94aed3fcaf5d19a5784b74d3a76be37a283bc
--- mqttd/session.c
+++ mqttd/session.c
return (0);
}
+/*
+ * Hand a message to a session. QoS 0 goes straight out or, when the
+ * connection is congested, is dropped and counted; at most once delivery
+ * allows that. A retained message sent because a subscription was just
+ * made is the exception: the spec requires it to reach the subscriber
+ * ([MQTT-3.3.1-9]), so it is written regardless. QoS 1 and 2 go out
+ * while the inflight window has room and the connection is draining,
+ * otherwise they wait in the queue up to the session's limit.
+ */
void
session_deliver(struct session *s, struct delivery *d)
{
d->session = s;
if (d->qos == 0) {
- if (s->conn != NULL)
+ if (s->conn != NULL && s->conn->congested && !d->retain) {
+ s->dropped++;
+ broker_env->stats.pub_dropped++;
+ } else if (s->conn != NULL)
broker_send_publish(s, d, 0);
delivery_free(d);
return;
}
- if (s->conn != NULL && s->ninflight < session_window(s)) {
+ if (s->conn != NULL && !s->conn->congested &&
+ s->ninflight < session_window(s)) {
session_inflight_insert(s, d);
if (d->pid == 0 || broker_send_publish(s, d, 0) == -1) {
session_inflight_remove(s, d);
{
struct delivery *d;
- while (s->conn != NULL && s->ninflight < session_window(s) &&
+ while (s->conn != NULL && !s->conn->congested &&
+ s->ninflight < session_window(s) &&
(d = TAILQ_FIRST(&s->queue)) != NULL) {
TAILQ_REMOVE(&s->queue, d, entry);
s->nqueued--;
blob - ae1924f7ff00a8f96fb2a9e875ab333f2925a00833f19b9d27f506bb673430e4
blob + a52aca6b81723f55b33587cbb7ff18eee1a8cc213fac3f239f1b92b2d18c6a01
--- mqttd/sockets.c
+++ mqttd/sockets.c
int closing;
int paused;
int throttled;
+ int congested;
/* websocket: decoded MQTT bytes in, one frame per packet out */
uint8_t ws;
struct evbuffer *wsin;
static void conn_tls_writecb(int, short, void *);
static void conn_cert_user(struct tls *, char *, size_t);
static void conn_throttle(struct conn *);
+static void conn_congested(struct conn *, int);
static void sockets_unthrottle(void);
static struct sbridge
*sbridge_find(uint32_t);
buf[n] = '\0';
}
+/*
+ * This peer is not reading what we write. Tell the broker, which drops
+ * QoS 0 for the connection and parks QoS 1 and 2 in the session queue
+ * until the buffer has drained, instead of piling up bytes here.
+ */
+static void
+conn_congested(struct conn *c, int on)
+{
+ struct conn_congested cg;
+
+ if (!c->opened)
+ return;
+ c->congested = on;
+ cg.id = c->id;
+ cg.congested = on;
+ imsg_compose_event(socks_iev_broker, IMSG_CONN_CONGESTED, 0, 0, -1,
+ &cg, sizeof(cg));
+ log_debug("%s: connection %u from %s %s", __func__, c->id, c->peer,
+ on ? "congested" : "drained");
+}
+
/* the broker pipe is full: stop reading from this client for now */
static void
conn_throttle(struct conn *c)
{
struct conn *c = arg;
- if (c->closing && EVBUFFER_LENGTH(EVBUFFER_OUTPUT(bev)) == 0)
+ if (EVBUFFER_LENGTH(EVBUFFER_OUTPUT(bev)) != 0)
+ return;
+ if (c->closing) {
conn_free(c, -1);
+ return;
+ }
+ if (c->congested)
+ conn_congested(c, 0);
}
static void
/*
* broker -> client: one chunk. Plain connections take it as is; a
* websocket collects the packet and sends it as one binary frame.
+ * A buffer over the congestion watermark is reported to the broker, one
+ * over the overflow watermark closes the connection.
*/
static void
conn_write_chunk(struct conn *c, const struct conn_pkt *hdr,
const uint8_t *data)
{
- size_t hiwat;
+ size_t len, hiwat;
if (c->ws == WS_OPEN) {
if (hdr->offset == 0)
} else
bufferevent_write(c->bev, data, hdr->len);
+ len = EVBUFFER_LENGTH(EVBUFFER_OUTPUT(c->bev));
hiwat = (size_t)env->conf.max_packet_size * MQTTD_OUTBUF_FACTOR;
- if (EVBUFFER_LENGTH(EVBUFFER_OUTPUT(c->bev)) > hiwat) {
+ if (len > hiwat) {
log_debug("%s: connection %u from %s is not draining",
__func__, c->id, c->peer);
conn_free(c, CONN_CLOSE_OVERFLOW);
+ return;
}
+ if (!c->congested &&
+ len > (size_t)env->conf.max_packet_size * MQTTD_CONGEST_FACTOR)
+ conn_congested(c, 1);
}
/* case-insensitive header lookup in a NUL terminated request */