Commit Diff


commit - 8507d3d5b7832bfe03b50521368f4829ceb782586ac8e40398196e724caec3de
commit + a8f2d00fef2add0808aaaa76f2e20c7662f8d08d3b7aa4e4162b54d6ca6764cc
blob - e8bd3e9700c6d0061d25b3a4b0ddda72267fe3dc9952548d8d8fba7b1b382f2f
blob + 98e930e577b97f059609cedffc4e0acc5b02a469ee71eb2206d691c04e180d7b
--- README.md	
+++ README.md	
@@ -10,8 +10,9 @@ Aware behaviour and topic ACLs with password, certific
 authentication.
 
 The daemon is split into a privileged parent, a control process, a single
-broker process holding all protocol state, and a sockets process that
-owns the network connections and TLS.
+broker process holding all protocol state, a listen process that accepts
+connections and a configurable number of sockets workers that run TLS and
+framing for the connections handed to them.
 The layout and plumbing come from the thingsd proxy daemon.
 
 Prerequisites
blob - 024ceec5cb3ff0c4be2245ce6c3448bcc147ea37bee3d844eff7d1fb398ed58d
blob + e5ce5c43a3ed8dd1f584be733b3b244283830366f7bf798756ba057b481e4378
--- mqttctl/mqttctl.c	
+++ mqttctl/mqttctl.c	
@@ -338,6 +338,7 @@ show_parent_msg(struct imsg *imsg)
 		printf("  started:           %s\n", fmt_time(pi.started));
 		printf("  uptime:            %s\n",
 		    fmt_duration(time(NULL) - pi.started, dbuf, sizeof(dbuf)));
+		printf("  workers:           %u\n", pi.workers);
 		printf("  listeners:         %u\n", pi.nlisteners);
 		break;
 	case IMSG_CTL_SHOW_PARENT_END:
blob - 8af8c13e9d413c115c68638a094f11e0317a8c87781fd260589f8e21903d94f8
blob + 8b7dd5c6faf2308982f443ef48745c8eeaffdfd69f07571b9f34394686c9c64b
--- mqttd/Makefile	
+++ mqttd/Makefile	
@@ -1,7 +1,7 @@
 PROG =		mqttd
 SRCS =		parse.y
 SRCS +=		mqttd.c config.c control.c log.c
-SRCS +=		sockets.c tls.c
+SRCS +=		listen.c sockets.c tls.c
 SRCS +=		broker.c session.c topic.c mqtt.c db.c storage.c auth.c
 SRCS +=		bridge.c sparkplug.c snowflake.c jwt.c
 
blob - 37147be5cc9d0b8834b998b4d930f048680985cf4bcf06c133b5de8a14df983d
blob + d26f4e17a0c3e03bd3ad94c5495a809eb91684be63756b7349ca8a8bad80b27e
--- mqttd/bridge.c	
+++ mqttd/bridge.c	
@@ -15,9 +15,10 @@
  */
 
 /*
- * Bridges: the broker as an MQTT client of another broker.  A sockets
- * process opens the TCP/TLS connection on request; from then on the
- * connection looks like any other, except that we send CONNECT and
+ * Bridges: the broker as an MQTT client of another broker.  The listen
+ * process dials the remote on request and a sockets worker runs TLS over
+ * it; from then on the connection looks like any other, except that we
+ * send CONNECT and
  * SUBSCRIBE and expect CONNACK, SUBACK and PINGRESP.  Local messages for
  * the remote flow through the bridge's own session like any delivery;
  * remote messages are routed locally with the bridge session as origin
@@ -193,8 +194,13 @@ bridge_connect_now(struct bridge *br)
 	br->attempts++;
 	log_debug("bridge %s: connecting to %s (attempt %u)", br->conf.name,
 	    br->conf.host, br->attempts);
-	broker_socks_send(br->conf.proc, IMSG_BRIDGE_CONNECT, &br->conf.id,
-	    sizeof(br->conf.id));
+	/* a sink's remote is the snowflake process, everything else dials */
+	if (br->conf.proc == MQTTD_PROC_SNOW)
+		broker_socks_send(br->conf.proc, IMSG_BRIDGE_CONNECT,
+		    &br->conf.id, sizeof(br->conf.id));
+	else
+		broker_listen_send(IMSG_BRIDGE_CONNECT, &br->conf.id,
+		    sizeof(br->conf.id));
 }
 
 static void
@@ -240,7 +246,7 @@ bridge_failed(struct imsg *imsg)
 	bridge_schedule(br);
 }
 
-/* the sockets process has a connection up; we are the client now */
+/* a sockets worker has the connection up; we are the client now */
 void
 bridge_conn_open(struct bconn *bc, uint32_t id)
 {
blob - c47a03beeff7e1b2c59dddc04d810c80e8011009a1b995987ed801adb8a9a708
blob + 9191faedad566553cb5e16d345bc4033d5bc27b820ad8cf18dbad548b23bfd06
--- mqttd/broker.c	
+++ mqttd/broker.c	
@@ -15,7 +15,7 @@
  */
 
 /*
- * The broker process is the protocol authority.  The sockets processes
+ * The broker process is the protocol authority.  The sockets workers
  * hand it complete control packets and it answers with the packets to
  * write back; it never touches a network socket itself.
  */
@@ -45,6 +45,7 @@ struct imsgev		*broker_iev_storage;
 
 static struct imsgev	*broker_iev_parent;
 static struct imsgev	*broker_iev_control;
+static struct imsgev	*broker_iev_listen;
 static struct imsgev	*broker_iev_socks;
 
 /* CFG_DONE is acknowledged only after storage has replayed its snapshot */
@@ -71,15 +72,15 @@ static struct imsgev	*broker_iev_auth;
 static struct imsgev	*broker_iev_snow;
 static uint32_t		 auth_next_id;
 
-/* the sockets process, or the snowflake process */
+/* sockets workers by index, or the snowflake process */
 static struct imsgev *
 broker_proc_iev(uint32_t proc)
 {
 	if (proc == MQTTD_PROC_SNOW)
 		return (broker_iev_snow);
-	if (proc != 0)
+	if (proc >= env->conf.workers)
 		return (NULL);
-	return (broker_iev_socks);
+	return (&broker_iev_socks[proc]);
 }
 
 struct route_ctx {
@@ -114,6 +115,7 @@ static void	 broker_snow_lost(void);
 static struct imsgev
 		*broker_proc_iev(uint32_t);
 static void	 broker_dispatch_sockets(int, short, void *);
+static void	 broker_dispatch_listen(int, short, void *);
 static void	 broker_sighdlr(int, short, void *);
 static void	 broker_sweep(int, short, void *);
 static __dead void
@@ -181,9 +183,10 @@ user_entry_cmp(struct user_entry *a, struct user_entry
 
 __dead void
 broker(int fd_parent, int fd_control, int fd_storage, int fd_auth,
-    int fd_snow, int fd_socks)
+    int fd_snow, int fd_listen, int *fd_socks)
 {
 	struct timeval	 tv;
+	int		 n;
 
 	log_procinit("broker");
 	proc_privdrop("broker");
@@ -239,10 +242,17 @@ broker(int fd_parent, int fd_control, int fd_storage, 
 		fatal("%s: calloc", __func__);
 	imsg_event_init(broker_iev_snow, fd_snow, broker_dispatch_sockets);
 
-	if ((broker_iev_socks = calloc(1, sizeof(*broker_iev_socks))) == NULL)
+	if ((broker_iev_listen = calloc(1, sizeof(*broker_iev_listen))) == NULL)
 		fatal("%s: calloc", __func__);
-	imsg_event_init(broker_iev_socks, fd_socks, broker_dispatch_sockets);
+	imsg_event_init(broker_iev_listen, fd_listen, broker_dispatch_listen);
 
+	if ((broker_iev_socks = calloc(env->conf.workers,
+	    sizeof(*broker_iev_socks))) == NULL)
+		fatal("%s: calloc", __func__);
+	for (n = 0; n < env->conf.workers; n++)
+		imsg_event_init(&broker_iev_socks[n], fd_socks[n],
+		    broker_dispatch_sockets);
+
 	evtimer_set(&ev_sweep, broker_sweep, NULL);
 	tv.tv_sec = MQTTD_RETAIN_SWEEP;
 	tv.tv_usec = 0;
@@ -297,6 +307,8 @@ broker_sweep(int fd, short event, void *arg)
 static __dead void
 broker_shutdown(void)
 {
+	int	 n;
+
 	/* the last snapshot must reach storage before the pipe closes */
 	db_snapshot_sync();
 
@@ -316,7 +328,10 @@ broker_shutdown(void)
 		imsg_event_shutdown(broker_iev_snow);
 		free(broker_iev_snow);
 	}
-	imsg_event_shutdown(broker_iev_socks);
+	imsg_event_shutdown(broker_iev_listen);
+	free(broker_iev_listen);
+	for (n = 0; n < env->conf.workers; n++)
+		imsg_event_shutdown(&broker_iev_socks[n]);
 	free(broker_iev_socks);
 
 	log_info("broker exiting, pid %d", getpid());
@@ -655,7 +670,8 @@ broker_dispatch_sockets(int fd, short event, void *arg
 	uint32_t	 proc;
 	int		 n;
 
-	proc = iev == broker_iev_snow ? MQTTD_PROC_SNOW : 0;
+	proc = iev == broker_iev_snow ? MQTTD_PROC_SNOW :
+	    (uint32_t)(iev - broker_iev_socks);
 
 	if (event & EV_READ) {
 		if ((n = imsgbuf_read(ibuf)) == -1)
@@ -718,6 +734,64 @@ broker_dispatch_sockets(int fd, short event, void *arg
 	broker_socks_event_add(iev);
 }
 
+/* the listen process only ever reports a bridge it could not dial */
+static void
+broker_dispatch_listen(int fd, short event, void *arg)
+{
+	struct imsgev	*iev = arg;
+	struct imsgbuf	*ibuf = &iev->ibuf;
+	struct imsg	 imsg;
+	int		 n;
+
+	if (event & EV_READ) {
+		if ((n = imsgbuf_read(ibuf)) == -1)
+			fatal("%s: imsgbuf_read", __func__);
+		if (n == 0) {
+			log_warnx("%s: lost listen process", __func__);
+			broker_shutdown();
+		}
+	}
+	if (event & EV_WRITE) {
+		if (imsgbuf_write(ibuf) == -1) {
+			if (errno == EPIPE) {
+				log_warnx("%s: lost listen process", __func__);
+				broker_shutdown();
+			}
+			fatal("%s: imsgbuf_write", __func__);
+		}
+	}
+
+	for (;;) {
+		if ((n = imsgbuf_get(ibuf, &imsg)) == -1)
+			fatal("%s: imsgbuf_get", __func__);
+		if (n == 0)
+			break;
+
+		switch (imsg.hdr.type) {
+		case IMSG_BRIDGE_FAILED:
+			bridge_failed(&imsg);
+			break;
+		default:
+			log_warnx("%s: unexpected imsg %d from listen",
+			    __func__, imsg.hdr.type);
+			break;
+		}
+		imsg_free(&imsg);
+	}
+	imsg_event_add(iev);
+}
+
+void
+broker_listen_send(uint32_t type, const void *data, size_t len)
+{
+	if (imsg_compose(&broker_iev_listen->ibuf, type, 0, 0, -1, data,
+	    len) == -1) {
+		log_warn("%s: imsg_compose", __func__);
+		return;
+	}
+	imsg_event_add(broker_iev_listen);
+}
+
 /* like imsg_event_add, but reading stays off while the broker is paused */
 static void
 broker_socks_event_add(struct imsgev *iev)
@@ -733,22 +807,24 @@ broker_socks_event_add(struct imsgev *iev)
 }
 
 /*
- * Backpressure.  When the sockets pipe or the snowflake pipe holds more
- * than MQTTD_QUEUE_HIGH unsent messages the broker stops reading from
- * both, so publishers stall instead of the broker growing without bound.
- * Reading resumes once both pipes are below MQTTD_QUEUE_LOW.
+ * Backpressure.  When any worker pipe holds more than MQTTD_QUEUE_HIGH
+ * unsent messages the broker stops reading from every worker, so
+ * publishers stall instead of the broker growing without bound.
+ * Reading resumes once every pipe is below MQTTD_QUEUE_LOW.
  */
 static void
 broker_flow_update(void)
 {
 	size_t	 q;
-	int	 over = 0, above_low = 0;
+	int	 n, over = 0, above_low = 0;
 
-	q = imsgbuf_queuelen(&broker_iev_socks->ibuf);
-	if (q > MQTTD_QUEUE_HIGH)
-		over = 1;
-	if (q > MQTTD_QUEUE_LOW)
-		above_low = 1;
+	for (n = 0; n < env->conf.workers; n++) {
+		q = imsgbuf_queuelen(&broker_iev_socks[n].ibuf);
+		if (q > MQTTD_QUEUE_HIGH)
+			over = 1;
+		if (q > MQTTD_QUEUE_LOW)
+			above_low = 1;
+	}
 	if (broker_iev_snow != NULL) {
 		q = imsgbuf_queuelen(&broker_iev_snow->ibuf);
 		if (q > MQTTD_QUEUE_HIGH)
@@ -766,7 +842,8 @@ broker_flow_update(void)
 	} else
 		return;
 
-	broker_socks_event_add(broker_iev_socks);
+	for (n = 0; n < env->conf.workers; n++)
+		broker_socks_event_add(&broker_iev_socks[n]);
 	if (broker_iev_snow != NULL)
 		broker_socks_event_add(broker_iev_snow);
 }
blob - ac35b57826a94aa77801850866dacd5d7c34d140d28c625a15c07fdecff6d0b9
blob + ed91869bf43db9b52f5b83d8b114cd7bcafdd6bab73822168695d1002e204244
--- mqttd/broker.h	
+++ mqttd/broker.h	
@@ -312,6 +312,7 @@ void		 broker_send_pubrel(struct session *, uint16_t, 
 void		 broker_send_ack(struct bconn *, uint8_t, uint16_t, uint8_t);
 void		 broker_close(struct bconn *, int, enum detach_how);
 void		 broker_socks_send(uint32_t, uint32_t, const void *, size_t);
+void		 broker_listen_send(uint32_t, const void *, size_t);
 int		 broker_filter_match(const char *, const char *, size_t);
 int		 broker_route(struct message *);
 #define RC_NONE		-1
blob - acbfedcc388b96d19f6feb8f9e0a07e39414e6086c6beb70cc7405a8ff1e5f83
blob + bf462de1116db0c01bef7e27216bd6a26d59e5bdcf73f04ae90edd0f3ba4b075
--- mqttd/config.c	
+++ mqttd/config.c	
@@ -62,9 +62,10 @@ config_recv_global(struct mqttd *e, struct imsg *imsg)
 }
 
 /*
- * imsg closes the passed descriptor once sent, so every receiver gets
- * its own dup.  The flush is synchronous because the parent pledges
- * without sendfd once configuration is finished.
+ * imsg closes a passed descriptor once it is sent; the listen process
+ * gets the listening socket, the workers get -1 with the configuration
+ * alone.  The flush is synchronous so the listener is in place before
+ * anything that depends on it is sent.
  */
 int
 config_send_listener(struct imsgev *iev, struct listener *l, int fd)
blob - 512395aa375e2bde98bf428ceb550fedaaf686db68bea2c7a85b0dfe00f77c5d
blob + 422026b9f8bbeb9db68ac963a3b72c48b16efb1887fb76181ecf7e30d64fd95a
--- mqttd/control.c	
+++ mqttd/control.c	
@@ -56,7 +56,7 @@ static struct ctl_connlist	 ctl_conns;
 static struct control_sock	 control_cs;
 static struct imsgev		*control_iev_parent;
 static struct imsgev		*control_iev_broker;
-static struct imsgev		*control_iev_socks;
+static struct imsgev		*control_iev_listen;
 
 static struct event		 ev_sigint;
 static struct event		 ev_sigterm;
@@ -74,14 +74,14 @@ static void	 control_close(int, struct control_sock *)
 static void	 control_dispatch_imsg(int, short, void *);
 static void	 control_dispatch_parent(int, short, void *);
 static void	 control_dispatch_broker(int, short, void *);
-static void	 control_dispatch_sockets(int, short, void *);
+static void	 control_dispatch_listen(int, short, void *);
 static void	 control_relay(struct imsg *);
 static void	 control_sighdlr(int, short, void *);
 static __dead void
 		 control_shutdown(void);
 
 __dead void
-control(int fd_parent, int fd_broker, int fd_socks)
+control(int fd_parent, int fd_broker, int fd_listen)
 {
 	log_procinit("control");
 
@@ -117,11 +117,11 @@ control(int fd_parent, int fd_broker, int fd_socks)
 	imsg_event_init(control_iev_broker, fd_broker,
 	    control_dispatch_broker);
 
-	if ((control_iev_socks = calloc(1, sizeof(*control_iev_socks)))
+	if ((control_iev_listen = calloc(1, sizeof(*control_iev_listen)))
 	    == NULL)
 		fatal("%s: calloc", __func__);
-	imsg_event_init(control_iev_socks, fd_socks,
-	    control_dispatch_sockets);
+	imsg_event_init(control_iev_listen, fd_listen,
+	    control_dispatch_listen);
 
 	TAILQ_INIT(&ctl_conns);
 
@@ -170,8 +170,8 @@ control_shutdown(void)
 	free(control_iev_parent);
 	imsg_event_shutdown(control_iev_broker);
 	free(control_iev_broker);
-	imsg_event_shutdown(control_iev_socks);
-	free(control_iev_socks);
+	imsg_event_shutdown(control_iev_listen);
+	free(control_iev_listen);
 
 	log_info("control exiting, pid %d", getpid());
 	exit(0);
@@ -298,7 +298,7 @@ control_dispatch_broker(int fd, short event, void *arg
 }
 
 static void
-control_dispatch_sockets(int fd, short event, void *arg)
+control_dispatch_listen(int fd, short event, void *arg)
 {
 	struct imsgev	*iev = arg;
 	struct imsgbuf	*ibuf = &iev->ibuf;
@@ -309,14 +309,14 @@ control_dispatch_sockets(int fd, short event, void *ar
 		if ((n = imsgbuf_read(ibuf)) == -1)
 			fatal("%s: imsgbuf_read", __func__);
 		if (n == 0) {
-			log_warnx("%s: lost sockets process", __func__);
+			log_warnx("%s: lost listen process", __func__);
 			control_shutdown();
 		}
 	}
 	if (event & EV_WRITE) {
 		if (imsgbuf_write(ibuf) == -1) {
 			if (errno == EPIPE) {
-				log_warnx("%s: lost sockets process", __func__);
+				log_warnx("%s: lost listen process", __func__);
 				control_shutdown();
 			}
 			fatal("%s: imsgbuf_write", __func__);
@@ -335,7 +335,7 @@ control_dispatch_sockets(int fd, short event, void *ar
 			control_relay(&imsg);
 			break;
 		default:
-			log_warnx("%s: unexpected imsg %d from sockets",
+			log_warnx("%s: unexpected imsg %d from listen",
 			    __func__, imsg.hdr.type);
 			break;
 		}
blob - /dev/null
blob + 1f4556a3268c9962b4448d1ed5ea5da85fc235c986e7ce493bdcf46f33a31421 (mode 644)
--- /dev/null	
+++ mqttd/listen.c	
@@ -0,0 +1,1119 @@
+/*
+ * Copyright (c) 2026 Tracey Emery <tracey@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+/*
+ * The listen process owns the listening sockets.  It accepts, consumes
+ * the PROXY protocol header where a listener expects one, dials outbound
+ * bridges, and hands every connected descriptor to the sockets worker
+ * with the fewest connections.  It never sees a TLS record or an MQTT
+ * byte, and it is the one place that knows how many connections each
+ * listener and each worker carries.
+ */
+
+#include <sys/types.h>
+#include <sys/queue.h>
+#include <sys/tree.h>
+#include <sys/time.h>
+#include <sys/socket.h>
+#include <sys/resource.h>
+#include <sys/stat.h>
+#include <sys/uio.h>
+#include <sys/un.h>
+
+#include <netinet/in.h>
+#include <arpa/inet.h>
+
+#include <errno.h>
+#include <event.h>
+#include <imsg.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "mqttd.h"
+
+#define FD_RESERVE		8
+#define PROXY_V1_MAX		107
+#define PROXY_V2_MIN		16
+#define PROXY_PEEK		(PROXY_V2_MIN + 216)
+
+/* outbound bridge target: the address only, the workers hold the TLS side */
+struct lbridge {
+	TAILQ_ENTRY(lbridge)	 entry;
+	struct bridge_config	 conf;
+};
+TAILQ_HEAD(lbridgelist, lbridge);
+
+/* a connection between accept() or connect() and its hand-off */
+struct lconn {
+	RB_ENTRY(lconn)		 entry;
+	uint32_t		 id;
+	int			 fd;
+	struct listener		*l;
+	struct lbridge		*bridge;
+	struct event		 ev;
+	struct event		 timer;
+	char			 peer[MQTTD_MAXPEER];
+};
+RB_HEAD(lconntree, lconn);
+
+static int	 lconn_cmp(struct lconn *, struct lconn *);
+RB_PROTOTYPE_STATIC(lconntree, lconn, entry, lconn_cmp);
+RB_GENERATE_STATIC(lconntree, lconn, entry, lconn_cmp);
+
+static void	 listen_dispatch_parent(int, short, void *);
+static void	 listen_dispatch_control(int, short, void *);
+static void	 listen_dispatch_broker(int, short, void *);
+static void	 listen_dispatch_socks(int, short, void *);
+static void	 listen_sighdlr(int, short, void *);
+static __dead void
+		 listen_shutdown(void);
+static void	 listen_rlimit(void);
+static void	 listen_launch(void);
+static void	 listen_listener_launch(struct listener *);
+static void	 listen_listener_remove(struct listener *);
+static int	 listen_accept_reserve(int, struct sockaddr *, socklen_t *);
+static void	 listen_accept(int, short, void *);
+static void	 listen_accept_paused(int, short, void *);
+static void	 listen_show_listeners(struct imsg *);
+static void	 listen_conn_done(uint32_t, struct imsg *);
+static uint32_t	 listen_pick_worker(void);
+
+static struct lconn
+		*lconn_new(void);
+static struct lconn
+		*lconn_find(uint32_t);
+static void	 lconn_free(struct lconn *);
+static void	 lconn_release(struct lconn *);
+static void	 lconn_handoff(struct lconn *);
+static void	 lconn_set_timer(struct lconn *, time_t);
+static void	 lconn_timeout(int, short, void *);
+static void	 lconn_proxy_read(int, short, void *);
+static int	 proxy_parse_v1(struct lconn *, const uint8_t *, size_t);
+static int	 proxy_parse_v2(struct lconn *, const uint8_t *, size_t);
+
+static struct lbridge
+		*lbridge_find(uint32_t);
+static void	 listen_bridge_config(struct imsg *);
+static void	 listen_bridge_connect(struct imsg *);
+static void	 listen_bridge_connected(int, short, void *);
+static void	 listen_bridge_failed(uint32_t);
+
+static struct imsgev	*listen_iev_parent;
+static struct imsgev	*listen_iev_control;
+static struct imsgev	*listen_iev_broker;
+static struct imsgev	*listen_iev_socks;
+/* live connections per worker, the hand-off decision */
+static uint32_t		*worker_load;
+
+static struct lconntree	 lconns = RB_INITIALIZER(&lconns);
+static struct lbridgelist lbridges = TAILQ_HEAD_INITIALIZER(lbridges);
+static uint32_t		 conn_next_id;
+
+static struct event	 ev_sigint;
+static struct event	 ev_sigterm;
+static struct event	 ev_sighup;
+static struct event	 ev_sigpipe;
+static struct event	 ev_sigusr1;
+
+static int
+lconn_cmp(struct lconn *a, struct lconn *b)
+{
+	if (a->id != b->id)
+		return (a->id < b->id ? -1 : 1);
+	return (0);
+}
+
+__dead void
+listen_main(int fd_parent, int fd_control, int fd_broker, int *fd_socks)
+{
+	int	 n;
+
+	log_procinit("listen");
+	proc_privdrop("listen");
+
+	/* start from empty lists, the listeners arrive from the parent */
+	config_init(env);
+
+	listen_rlimit();
+
+	event_init();
+
+	signal_set(&ev_sigint, SIGINT, listen_sighdlr, NULL);
+	signal_set(&ev_sigterm, SIGTERM, listen_sighdlr, NULL);
+	signal_set(&ev_sighup, SIGHUP, listen_sighdlr, NULL);
+	signal_set(&ev_sigpipe, SIGPIPE, listen_sighdlr, NULL);
+	signal_set(&ev_sigusr1, SIGUSR1, listen_sighdlr, NULL);
+	signal_add(&ev_sigint, NULL);
+	signal_add(&ev_sigterm, NULL);
+	signal_add(&ev_sighup, NULL);
+	signal_add(&ev_sigpipe, NULL);
+	signal_add(&ev_sigusr1, NULL);
+
+	if ((listen_iev_parent = calloc(1, sizeof(*listen_iev_parent))) == NULL)
+		fatal("%s: calloc", __func__);
+	imsg_event_init(listen_iev_parent, fd_parent, listen_dispatch_parent);
+
+	if ((listen_iev_control = calloc(1, sizeof(*listen_iev_control)))
+	    == NULL)
+		fatal("%s: calloc", __func__);
+	imsg_event_init(listen_iev_control, fd_control,
+	    listen_dispatch_control);
+
+	if ((listen_iev_broker = calloc(1, sizeof(*listen_iev_broker))) == NULL)
+		fatal("%s: calloc", __func__);
+	imsg_event_init(listen_iev_broker, fd_broker, listen_dispatch_broker);
+
+	if ((listen_iev_socks = calloc(env->conf.workers,
+	    sizeof(*listen_iev_socks))) == NULL ||
+	    (worker_load = calloc(env->conf.workers,
+	    sizeof(*worker_load))) == NULL)
+		fatal("%s: calloc", __func__);
+	for (n = 0; n < env->conf.workers; n++)
+		imsg_event_init(&listen_iev_socks[n], fd_socks[n],
+		    listen_dispatch_socks);
+
+	/*
+	 * inet and unix for accept(2) and connect(2), sendfd to hand the
+	 * connections to the workers, recvfd for listeners added on reload.
+	 */
+	if (pledge("stdio inet unix sendfd recvfd", NULL) == -1)
+		fatal("pledge");
+
+	event_dispatch();
+
+	listen_shutdown();
+}
+
+static void
+listen_sighdlr(int sig, short event, void *arg)
+{
+	switch (sig) {
+	case SIGINT:
+	case SIGTERM:
+		listen_shutdown();
+		break;
+	case SIGHUP:
+	case SIGPIPE:
+	case SIGUSR1:
+		break;
+	default:
+		fatalx("%s: unexpected signal", __func__);
+	}
+}
+
+static __dead void
+listen_shutdown(void)
+{
+	struct listener	*l, *tl;
+	struct lconn	*c, *tc;
+	struct lbridge	*lb;
+	int		 n;
+
+	RB_FOREACH_SAFE(c, lconntree, &lconns, tc)
+		lconn_free(c);
+
+	while ((lb = TAILQ_FIRST(&lbridges)) != NULL) {
+		TAILQ_REMOVE(&lbridges, lb, entry);
+		free(lb);
+	}
+
+	TAILQ_FOREACH_SAFE(l, &env->listeners, entry, tl) {
+		if (event_initialized(&l->ev))
+			event_del(&l->ev);
+		if (evtimer_initialized(&l->pause))
+			evtimer_del(&l->pause);
+		TAILQ_REMOVE(&env->listeners, l, entry);
+		config_free_listener(l);
+	}
+
+	imsg_event_shutdown(listen_iev_parent);
+	free(listen_iev_parent);
+	imsg_event_shutdown(listen_iev_control);
+	free(listen_iev_control);
+	imsg_event_shutdown(listen_iev_broker);
+	free(listen_iev_broker);
+	for (n = 0; n < env->conf.workers; n++)
+		imsg_event_shutdown(&listen_iev_socks[n]);
+	free(listen_iev_socks);
+	free(worker_load);
+
+	log_info("listen exiting, pid %d", getpid());
+	exit(0);
+}
+
+static void
+listen_rlimit(void)
+{
+	struct rlimit	 rl;
+
+	if (getrlimit(RLIMIT_NOFILE, &rl) == -1)
+		fatal("%s: getrlimit", __func__);
+	log_debug("%s: max open files %llu", __func__,
+	    (unsigned long long)rl.rlim_max);
+
+	/*
+	 * Descriptors only live here until a worker has them, but a
+	 * connect storm can leave many in flight at once.
+	 */
+	rl.rlim_cur = rl.rlim_max;
+	if (setrlimit(RLIMIT_NOFILE, &rl) == -1)
+		fatal("%s: setrlimit", __func__);
+}
+
+/* bind and listen; called by the parent, which still has the privileges */
+int
+listen_bind(struct listener *l)
+{
+	struct sockaddr_storage	*ss = &l->conf.ss;
+	struct sockaddr_un	*sun;
+	int			 fd, on = 1;
+
+	if (ss->ss_family == AF_UNIX) {
+		sun = (struct sockaddr_un *)ss;
+		if (unlink(sun->sun_path) == -1 && errno != ENOENT) {
+			log_warn("%s: unlink %s", __func__, sun->sun_path);
+			return (-1);
+		}
+		fd = socket(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, 0);
+		if (fd == -1) {
+			log_warn("%s: socket", __func__);
+			return (-1);
+		}
+		if (bind(fd, (struct sockaddr *)sun, sizeof(*sun)) == -1) {
+			log_warn("%s: bind %s", __func__, sun->sun_path);
+			goto fail;
+		}
+		/* authentication still applies; let local users reach it */
+		if (chmod(sun->sun_path, MQTTD_UNIX_MODE) == -1) {
+			log_warn("%s: chmod %s", __func__, sun->sun_path);
+			goto fail;
+		}
+		if (listen(fd, SOMAXCONN) == -1) {
+			log_warn("%s: listen", __func__);
+			goto fail;
+		}
+		l->fd = fd;
+		return (0);
+	}
+
+	switch (ss->ss_family) {
+	case AF_INET:
+		((struct sockaddr_in *)ss)->sin_port = l->conf.port;
+		((struct sockaddr_in *)ss)->sin_len = sizeof(struct sockaddr_in);
+		break;
+	case AF_INET6:
+		((struct sockaddr_in6 *)ss)->sin6_port = l->conf.port;
+		((struct sockaddr_in6 *)ss)->sin6_len =
+		    sizeof(struct sockaddr_in6);
+		break;
+	default:
+		log_warnx("%s: unsupported address family", __func__);
+		return (-1);
+	}
+
+	if ((fd = socket(ss->ss_family, SOCK_STREAM | SOCK_NONBLOCK, 0)) == -1) {
+		log_warn("%s: socket", __func__);
+		return (-1);
+	}
+	if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) {
+		log_warn("%s: setsockopt SO_REUSEADDR", __func__);
+		goto fail;
+	}
+	if (ss->ss_family == AF_INET6 &&
+	    setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &on, sizeof(on)) == -1) {
+		log_warn("%s: setsockopt IPV6_V6ONLY", __func__);
+		goto fail;
+	}
+	if (bind(fd, (struct sockaddr *)ss, ss->ss_len) == -1) {
+		log_warn("%s: bind listener %s port %d", __func__, l->conf.name,
+		    ntohs(l->conf.port));
+		goto fail;
+	}
+	if (listen(fd, SOMAXCONN) == -1) {
+		log_warn("%s: listen", __func__);
+		goto fail;
+	}
+
+	l->fd = fd;
+	return (0);
+fail:
+	close(fd);
+	return (-1);
+}
+
+static void
+listen_listener_launch(struct listener *l)
+{
+	log_debug("%s: listener %s[%u] on fd %d", __func__, l->conf.name,
+	    l->conf.id, l->fd);
+
+	event_set(&l->ev, l->fd, EV_READ | EV_PERSIST, listen_accept, l);
+	if (event_add(&l->ev, NULL) == -1)
+		fatal("%s: event_add", __func__);
+	evtimer_set(&l->pause, listen_accept_paused, l);
+}
+
+/*
+ * A reload removed this listener.  Connections already handed to a
+ * worker live on; their close is still reported with the listener id
+ * and simply finds nothing to decrement.
+ */
+static void
+listen_listener_remove(struct listener *l)
+{
+	struct lconn	*c;
+
+	RB_FOREACH(c, lconntree, &lconns)
+		if (c->l == l)
+			c->l = NULL;
+
+	if (event_initialized(&l->ev))
+		event_del(&l->ev);
+	if (evtimer_initialized(&l->pause))
+		evtimer_del(&l->pause);
+	TAILQ_REMOVE(&env->listeners, l, entry);
+	env->nlisteners--;
+	config_free_listener(l);
+}
+
+static void
+listen_launch(void)
+{
+	struct listener	*l;
+
+	TAILQ_FOREACH(l, &env->listeners, entry)
+		listen_listener_launch(l);
+}
+
+static void
+listen_dispatch_parent(int fd, short event, void *arg)
+{
+	struct imsgev	*iev = arg;
+	struct imsgbuf	*ibuf = &iev->ibuf;
+	struct imsg	 imsg;
+	struct listener	*l;
+	uint32_t	 lid;
+	int		 n, verbose;
+
+	if (event & EV_READ) {
+		if ((n = imsgbuf_read(ibuf)) == -1)
+			fatal("%s: imsgbuf_read", __func__);
+		if (n == 0) {
+			log_warnx("%s: lost parent process", __func__);
+			listen_shutdown();
+		}
+	}
+	if (event & EV_WRITE) {
+		if (imsgbuf_write(ibuf) == -1) {
+			if (errno == EPIPE) {
+				log_warnx("%s: lost parent process", __func__);
+				listen_shutdown();
+			}
+			fatal("%s: imsgbuf_write", __func__);
+		}
+	}
+
+	for (;;) {
+		if ((n = imsgbuf_get(ibuf, &imsg)) == -1)
+			fatal("%s: imsgbuf_get", __func__);
+		if (n == 0)
+			break;
+
+		switch (imsg.hdr.type) {
+		case IMSG_CFG_GLOBAL:
+			config_recv_global(env, &imsg);
+			break;
+		case IMSG_CFG_LISTENER:
+			l = config_recv_listener(env, &imsg);
+			if (l->fd == -1)
+				fatalx("%s: listener %s arrived without a "
+				    "descriptor", __func__, l->conf.name);
+			break;
+		case IMSG_CFG_BRIDGE:
+			listen_bridge_config(&imsg);
+			break;
+		case IMSG_CFG_LISTENER_START:
+			IMSG_SIZE_CHECK(&imsg, &lid);
+			memcpy(&lid, imsg.data, sizeof(lid));
+			if ((l = config_listener_byid(env, lid)) == NULL)
+				break;
+			listen_listener_launch(l);
+			log_info("listener %s added", l->conf.name);
+			break;
+		case IMSG_CFG_LISTENER_DEL:
+			IMSG_SIZE_CHECK(&imsg, &lid);
+			memcpy(&lid, imsg.data, sizeof(lid));
+			if ((l = config_listener_byid(env, lid)) == NULL)
+				break;
+			log_info("listener %s removed", l->conf.name);
+			listen_listener_remove(l);
+			break;
+		case IMSG_CFG_DONE:
+			imsg_compose_event(listen_iev_parent, IMSG_CFG_DONE, 0,
+			    0, -1, NULL, 0);
+			break;
+		case IMSG_CTL_START:
+			listen_launch();
+			break;
+		case IMSG_CTL_VERBOSE:
+			IMSG_SIZE_CHECK(&imsg, &verbose);
+			memcpy(&verbose, imsg.data, sizeof(verbose));
+			log_setverbose(verbose);
+			break;
+		case IMSG_CTL_SHOW_LISTENERS_REQUEST:
+			listen_show_listeners(&imsg);
+			break;
+		default:
+			log_warnx("%s: unexpected imsg %d from parent",
+			    __func__, imsg.hdr.type);
+			break;
+		}
+		imsg_free(&imsg);
+	}
+	imsg_event_add(iev);
+}
+
+static void
+listen_dispatch_control(int fd, short event, void *arg)
+{
+	struct imsgev	*iev = arg;
+	struct imsgbuf	*ibuf = &iev->ibuf;
+	struct imsg	 imsg;
+	int		 n;
+
+	if (event & EV_READ) {
+		if ((n = imsgbuf_read(ibuf)) == -1)
+			fatal("%s: imsgbuf_read", __func__);
+		if (n == 0) {
+			log_warnx("%s: lost control process", __func__);
+			listen_shutdown();
+		}
+	}
+	if (event & EV_WRITE) {
+		if (imsgbuf_write(ibuf) == -1) {
+			if (errno == EPIPE) {
+				log_warnx("%s: lost control process", __func__);
+				listen_shutdown();
+			}
+			fatal("%s: imsgbuf_write", __func__);
+		}
+	}
+
+	/* control only listens to us; its requests arrive via the parent */
+	for (;;) {
+		if ((n = imsgbuf_get(ibuf, &imsg)) == -1)
+			fatal("%s: imsgbuf_get", __func__);
+		if (n == 0)
+			break;
+		log_warnx("%s: unexpected imsg %d from control", __func__,
+		    imsg.hdr.type);
+		imsg_free(&imsg);
+	}
+	imsg_event_add(iev);
+}
+
+static void
+listen_dispatch_broker(int fd, short event, void *arg)
+{
+	struct imsgev	*iev = arg;
+	struct imsgbuf	*ibuf = &iev->ibuf;
+	struct imsg	 imsg;
+	int		 n;
+
+	if (event & EV_READ) {
+		if ((n = imsgbuf_read(ibuf)) == -1)
+			fatal("%s: imsgbuf_read", __func__);
+		if (n == 0) {
+			log_warnx("%s: lost broker process", __func__);
+			listen_shutdown();
+		}
+	}
+	if (event & EV_WRITE) {
+		if (imsgbuf_write(ibuf) == -1) {
+			if (errno == EPIPE) {
+				log_warnx("%s: lost broker process", __func__);
+				listen_shutdown();
+			}
+			fatal("%s: imsgbuf_write", __func__);
+		}
+	}
+
+	for (;;) {
+		if ((n = imsgbuf_get(ibuf, &imsg)) == -1)
+			fatal("%s: imsgbuf_get", __func__);
+		if (n == 0)
+			break;
+
+		switch (imsg.hdr.type) {
+		case IMSG_BRIDGE_CONNECT:
+			listen_bridge_connect(&imsg);
+			break;
+		default:
+			log_warnx("%s: unexpected imsg %d from broker",
+			    __func__, imsg.hdr.type);
+			break;
+		}
+		imsg_free(&imsg);
+	}
+	imsg_event_add(iev);
+}
+
+static void
+listen_dispatch_socks(int fd, short event, void *arg)
+{
+	struct imsgev	*iev = arg;
+	struct imsgbuf	*ibuf = &iev->ibuf;
+	struct imsg	 imsg;
+	uint32_t	 w = (uint32_t)(iev - listen_iev_socks);
+	int		 n;
+
+	if (event & EV_READ) {
+		if ((n = imsgbuf_read(ibuf)) == -1)
+			fatal("%s: imsgbuf_read", __func__);
+		if (n == 0) {
+			log_warnx("%s: lost sockets worker %u", __func__, w);
+			listen_shutdown();
+		}
+	}
+	if (event & EV_WRITE) {
+		if (imsgbuf_write(ibuf) == -1) {
+			if (errno == EPIPE) {
+				log_warnx("%s: lost sockets worker %u",
+				    __func__, w);
+				listen_shutdown();
+			}
+			fatal("%s: imsgbuf_write", __func__);
+		}
+	}
+
+	for (;;) {
+		if ((n = imsgbuf_get(ibuf, &imsg)) == -1)
+			fatal("%s: imsgbuf_get", __func__);
+		if (n == 0)
+			break;
+
+		switch (imsg.hdr.type) {
+		case IMSG_CONN_DONE:
+			listen_conn_done(w, &imsg);
+			break;
+		default:
+			log_warnx("%s: unexpected imsg %d from worker %u",
+			    __func__, imsg.hdr.type, w);
+			break;
+		}
+		imsg_free(&imsg);
+	}
+	imsg_event_add(iev);
+}
+
+/* a worker closed a connection we handed it */
+static void
+listen_conn_done(uint32_t w, struct imsg *imsg)
+{
+	struct conn_done	 cd;
+	struct listener		*l;
+
+	IMSG_SIZE_CHECK(imsg, &cd);
+	memcpy(&cd, imsg->data, sizeof(cd));
+
+	if (worker_load[w] > 0)
+		worker_load[w]--;
+	if (cd.listener_id != 0 &&
+	    (l = config_listener_byid(env, cd.listener_id)) != NULL &&
+	    l->nconns > 0)
+		l->nconns--;
+}
+
+static void
+listen_show_listeners(struct imsg *imsg)
+{
+	struct ctl_listener_info	 li;
+	struct listener			*l;
+
+	TAILQ_FOREACH(l, &env->listeners, entry) {
+		memset(&li, 0, sizeof(li));
+		li.id = l->conf.id;
+		strlcpy(li.name, l->conf.name, sizeof(li.name));
+		sockaddr_ntop(&l->conf.ss, li.addr, sizeof(li.addr));
+		li.port = l->conf.port;
+		li.tls = l->conf.tls;
+		li.websocket = l->conf.websocket;
+		li.proxy = l->conf.proxy;
+		li.fd = l->fd;
+		li.nconns = l->nconns;
+		if (imsg_compose_event(listen_iev_control,
+		    IMSG_CTL_SHOW_LISTENERS_DATA, imsg->hdr.peerid, 0, -1,
+		    &li, sizeof(li)) == -1)
+			return;
+	}
+
+	imsg_compose_event(listen_iev_control, IMSG_CTL_SHOW_LISTENERS_END,
+	    imsg->hdr.peerid, 0, -1, NULL, 0);
+}
+
+/* keep a few descriptors free for the imsg pipes and libevent itself */
+static int
+listen_accept_reserve(int sockfd, struct sockaddr *addr, socklen_t *addrlen)
+{
+	if (getdtablecount() + FD_RESERVE >= getdtablesize()) {
+		log_debug("%s: descriptor table nearly full", __func__);
+		errno = EMFILE;
+		return (-1);
+	}
+
+	return (accept4(sockfd, addr, addrlen, SOCK_NONBLOCK | SOCK_CLOEXEC));
+}
+
+static void
+listen_accept_paused(int fd, short event, void *arg)
+{
+	struct listener	*l = arg;
+
+	event_add(&l->ev, NULL);
+}
+
+static void
+listen_accept(int fd, short event, void *arg)
+{
+	struct listener		*l = arg;
+	struct lconn		*c;
+	struct sockaddr_storage	 ss;
+	struct timeval		 backoff = { 1, 0 };
+	socklen_t		 len = sizeof(ss);
+	int			 cfd;
+
+	cfd = listen_accept_reserve(fd, (struct sockaddr *)&ss, &len);
+	if (cfd == -1) {
+		switch (errno) {
+		case EINTR:
+		case EWOULDBLOCK:
+		case ECONNABORTED:
+			return;
+		case EMFILE:
+		case ENFILE:
+			/* out of descriptors, give the clients a moment */
+			event_del(&l->ev);
+			evtimer_add(&l->pause, &backoff);
+			return;
+		default:
+			log_warn("%s: accept", __func__);
+			return;
+		}
+	}
+
+	if ((c = lconn_new()) == NULL) {
+		log_warn("%s: calloc", __func__);
+		close(cfd);
+		return;
+	}
+	c->fd = cfd;
+	c->l = l;
+	if (ss.ss_family == AF_UNIX)
+		strlcpy(c->peer, "unix", sizeof(c->peer));
+	else
+		sockaddr_ntop(&ss, c->peer, sizeof(c->peer));
+	l->nconns++;
+
+	log_debug("%s: connection %u from %s on listener %s", __func__, c->id,
+	    c->peer, l->conf.name);
+
+	if (l->conf.proxy) {
+		/* the proxy has this long to send its header */
+		lconn_set_timer(c, MQTTD_CONNECT_TIMEOUT);
+		event_set(&c->ev, cfd, EV_READ, lconn_proxy_read, c);
+		event_add(&c->ev, NULL);
+		return;
+	}
+
+	lconn_handoff(c);
+}
+
+static struct lconn *
+lconn_new(void)
+{
+	struct lconn	*c;
+
+	if ((c = calloc(1, sizeof(*c))) == NULL)
+		return (NULL);
+	do {
+		c->id = ++conn_next_id;
+	} while (c->id == 0 || lconn_find(c->id) != NULL);
+	c->fd = -1;
+	evtimer_set(&c->timer, lconn_timeout, c);
+	RB_INSERT(lconntree, &lconns, c);
+	return (c);
+}
+
+static struct lconn *
+lconn_find(uint32_t id)
+{
+	struct lconn	 key;
+
+	key.id = id;
+	return (RB_FIND(lconntree, &lconns, &key));
+}
+
+static void
+lconn_set_timer(struct lconn *c, time_t secs)
+{
+	struct timeval	 tv;
+
+	tv.tv_sec = secs;
+	tv.tv_usec = 0;
+	evtimer_del(&c->timer);
+	evtimer_add(&c->timer, &tv);
+}
+
+/* drop a connection that never reached a worker */
+static void
+lconn_free(struct lconn *c)
+{
+	if (c->bridge != NULL)
+		listen_bridge_failed(c->bridge->conf.id);
+
+	evtimer_del(&c->timer);
+	if (event_initialized(&c->ev))
+		event_del(&c->ev);
+	if (c->fd != -1)
+		close(c->fd);
+	if (c->l != NULL && c->l->nconns > 0)
+		c->l->nconns--;
+	RB_REMOVE(lconntree, &lconns, c);
+	free(c);
+}
+
+/* a worker owns it now; the listener count is released by IMSG_CONN_DONE */
+static void
+lconn_release(struct lconn *c)
+{
+	evtimer_del(&c->timer);
+	if (event_initialized(&c->ev))
+		event_del(&c->ev);
+	RB_REMOVE(lconntree, &lconns, c);
+	free(c);
+}
+
+/* lowest index among the workers with the fewest connections */
+static uint32_t
+listen_pick_worker(void)
+{
+	int	 w, best = 0;
+
+	for (w = 1; w < env->conf.workers; w++)
+		if (worker_load[w] < worker_load[best])
+			best = w;
+	return ((uint32_t)best);
+}
+
+static void
+lconn_handoff(struct lconn *c)
+{
+	struct conn_accept	 ca;
+	uint32_t		 w;
+
+	memset(&ca, 0, sizeof(ca));
+	ca.id = c->id;
+	ca.listener_id = c->l != NULL ? c->l->conf.id : 0;
+	ca.bridge_id = c->bridge != NULL ? c->bridge->conf.id : 0;
+	strlcpy(ca.peer, c->peer, sizeof(ca.peer));
+
+	w = listen_pick_worker();
+
+	/* imsg owns the descriptor from here and closes it once sent */
+	if (imsg_compose_event(&listen_iev_socks[w], IMSG_CONN_ACCEPT, 0, 0,
+	    c->fd, &ca, sizeof(ca)) == -1) {
+		log_warn("%s: imsg_compose_event", __func__);
+		lconn_free(c);
+		return;
+	}
+	c->fd = -1;
+	worker_load[w]++;
+
+	log_debug("%s: connection %u %s %s to worker %u", __func__, c->id,
+	    c->bridge != NULL ? "to" : "from", c->peer, w);
+
+	lconn_release(c);
+}
+
+static void
+lconn_timeout(int fd, short event, void *arg)
+{
+	struct lconn	*c = arg;
+
+	log_debug("%s: connection %u %s %s timed out before hand-off",
+	    __func__, c->id, c->bridge != NULL ? "to" : "from", c->peer);
+	lconn_free(c);
+}
+
+/*
+ * The proxy sends its header in one write before any client byte, so
+ * peek, work out the exact length, then consume exactly that much and
+ * leave the stream untouched for the worker.
+ */
+static void
+lconn_proxy_read(int fd, short event, void *arg)
+{
+	struct lconn	*c = arg;
+	uint8_t		 buf[PROXY_PEEK];
+	ssize_t		 n;
+	int		 used;
+
+	n = recv(fd, buf, sizeof(buf), MSG_PEEK);
+	if (n == -1) {
+		if (errno == EAGAIN || errno == EINTR) {
+			event_add(&c->ev, NULL);
+			return;
+		}
+		lconn_free(c);
+		return;
+	}
+	if (n == 0) {
+		lconn_free(c);
+		return;
+	}
+
+	if (n >= PROXY_V2_MIN &&
+	    memcmp(buf, "\x0d\x0a\x0d\x0a\x00\x0d\x0a\x51\x55\x49\x54\x0a",
+	    12) == 0)
+		used = proxy_parse_v2(c, buf, n);
+	else if (n >= 6 && memcmp(buf, "PROXY ", 6) == 0)
+		used = proxy_parse_v1(c, buf, n);
+	else {
+		log_warnx("%s: no PROXY header from %s", __func__, c->peer);
+		lconn_free(c);
+		return;
+	}
+
+	if (used == 0) {
+		/* incomplete header, wait for the rest */
+		if (n >= (ssize_t)sizeof(buf)) {
+			lconn_free(c);
+			return;
+		}
+		event_add(&c->ev, NULL);
+		return;
+	}
+	if (used == -1) {
+		log_warnx("%s: bad PROXY header from %s", __func__, c->peer);
+		lconn_free(c);
+		return;
+	}
+
+	/* the peeked bytes are still there; this read cannot block */
+	if (recv(fd, buf, used, MSG_WAITALL) != used) {
+		lconn_free(c);
+		return;
+	}
+	event_del(&c->ev);
+	lconn_handoff(c);
+}
+
+/* returns bytes consumed, 0 for more needed, -1 for malformed */
+static int
+proxy_parse_v1(struct lconn *c, const uint8_t *buf, size_t n)
+{
+	char		 line[PROXY_V1_MAX + 1], *p, *end, *fam, *src, *sport;
+	const char	*eol;
+	size_t		 len;
+
+	eol = memmem(buf, n < PROXY_V1_MAX ? n : PROXY_V1_MAX, "\r\n", 2);
+	if (eol == NULL)
+		return (n >= PROXY_V1_MAX ? -1 : 0);
+	len = eol - (const char *)buf;
+	memcpy(line, buf, len);
+	line[len] = '\0';
+
+	/* PROXY TCP4 src dst sport dport, or PROXY UNKNOWN ... */
+	p = line + 6;
+	end = p;
+	fam = strsep(&end, " ");
+	if (fam == NULL)
+		return (-1);
+	if (strcmp(fam, "UNKNOWN") == 0)
+		return (len + 2);
+	if (strcmp(fam, "TCP4") != 0 && strcmp(fam, "TCP6") != 0)
+		return (-1);
+	src = strsep(&end, " ");
+	(void)strsep(&end, " ");
+	sport = strsep(&end, " ");
+	if (src == NULL || sport == NULL || end == NULL)
+		return (-1);
+	if (strchr(src, ':') != NULL)
+		snprintf(c->peer, sizeof(c->peer), "[%s]:%s", src, sport);
+	else
+		snprintf(c->peer, sizeof(c->peer), "%s:%s", src, sport);
+	return (len + 2);
+}
+
+static int
+proxy_parse_v2(struct lconn *c, const uint8_t *buf, size_t n)
+{
+	char		 host[INET6_ADDRSTRLEN];
+	size_t		 len, total;
+	uint8_t		 cmd, fam;
+	uint16_t	 port;
+
+	if ((buf[12] >> 4) != 2)
+		return (-1);
+	cmd = buf[12] & 0x0f;
+	fam = buf[13];
+	len = (size_t)buf[14] << 8 | buf[15];
+	total = PROXY_V2_MIN + len;
+	if (total > PROXY_PEEK)
+		return (-1);
+	if (n < total)
+		return (0);
+
+	/* LOCAL (health checks) keeps the proxy's own address */
+	if (cmd != 0x01)
+		return (total);
+
+	switch (fam) {
+	case 0x11:
+		if (len < 12)
+			return (-1);
+		inet_ntop(AF_INET, buf + 16, host, sizeof(host));
+		port = (uint16_t)buf[24] << 8 | buf[25];
+		snprintf(c->peer, sizeof(c->peer), "%s:%u", host, port);
+		break;
+	case 0x21:
+		if (len < 36)
+			return (-1);
+		inet_ntop(AF_INET6, buf + 16, host, sizeof(host));
+		port = (uint16_t)buf[48] << 8 | buf[49];
+		snprintf(c->peer, sizeof(c->peer), "[%s]:%u", host, port);
+		break;
+	default:
+		break;
+	}
+	return (total);
+}
+
+static struct lbridge *
+lbridge_find(uint32_t id)
+{
+	struct lbridge	*lb;
+
+	TAILQ_FOREACH(lb, &lbridges, entry)
+		if (lb->conf.id == id)
+			return (lb);
+	return (NULL);
+}
+
+static void
+listen_bridge_config(struct imsg *imsg)
+{
+	struct lbridge	*lb;
+
+	if ((lb = calloc(1, sizeof(*lb))) == NULL)
+		fatal("%s: calloc", __func__);
+	IMSG_SIZE_CHECK(imsg, &lb->conf);
+	memcpy(&lb->conf, imsg->data, sizeof(lb->conf));
+	lb->conf.name[sizeof(lb->conf.name) - 1] = '\0';
+	lb->conf.host[sizeof(lb->conf.host) - 1] = '\0';
+	lb->conf.tls_ca_file = NULL;
+	lb->conf.tls_ca = NULL;
+	lb->conf.tls_ca_len = 0;
+	explicit_bzero(lb->conf.password, sizeof(lb->conf.password));
+	TAILQ_INSERT_TAIL(&lbridges, lb, entry);
+}
+
+static void
+listen_bridge_failed(uint32_t id)
+{
+	imsg_compose_event(listen_iev_broker, IMSG_BRIDGE_FAILED, 0, 0, -1,
+	    &id, sizeof(id));
+}
+
+/* the broker wants this bridge up: start a non-blocking connect */
+static void
+listen_bridge_connect(struct imsg *imsg)
+{
+	struct lbridge		*lb;
+	struct lconn		*c;
+	struct sockaddr_storage	 ss;
+	uint32_t		 id;
+	int			 fd;
+
+	IMSG_SIZE_CHECK(imsg, &id);
+	memcpy(&id, imsg->data, sizeof(id));
+	if ((lb = lbridge_find(id)) == NULL) {
+		log_warnx("%s: unknown bridge %u", __func__, id);
+		listen_bridge_failed(id);
+		return;
+	}
+
+	memcpy(&ss, &lb->conf.ss, sizeof(ss));
+	switch (ss.ss_family) {
+	case AF_INET:
+		((struct sockaddr_in *)&ss)->sin_port = lb->conf.port;
+		break;
+	case AF_INET6:
+		((struct sockaddr_in6 *)&ss)->sin6_port = lb->conf.port;
+		break;
+	}
+
+	fd = socket(ss.ss_family, SOCK_STREAM | SOCK_NONBLOCK, 0);
+	if (fd == -1) {
+		log_warn("%s: socket", __func__);
+		listen_bridge_failed(id);
+		return;
+	}
+
+	if ((c = lconn_new()) == NULL) {
+		log_warn("%s: calloc", __func__);
+		close(fd);
+		listen_bridge_failed(id);
+		return;
+	}
+	c->fd = fd;
+	c->bridge = lb;
+	sockaddr_ntop(&ss, c->peer, sizeof(c->peer));
+	lconn_set_timer(c, MQTTD_CONNECT_TIMEOUT);
+
+	if (connect(fd, (struct sockaddr *)&ss, ss.ss_len) == -1 &&
+	    errno != EINPROGRESS) {
+		log_warn("bridge %s: connect to %s", lb->conf.name, c->peer);
+		lconn_free(c);
+		return;
+	}
+
+	event_set(&c->ev, fd, EV_WRITE, listen_bridge_connected, c);
+	event_add(&c->ev, NULL);
+}
+
+static void
+listen_bridge_connected(int fd, short event, void *arg)
+{
+	struct lconn	*c = arg;
+	socklen_t	 len = sizeof(int);
+	int		 err = 0;
+
+	if (getsockopt(fd, SOL_SOCKET, SO_ERROR, &err, &len) == -1)
+		err = errno;
+	if (err != 0) {
+		log_warnx("bridge %s: connect to %s: %s", c->bridge->conf.name,
+		    c->peer, strerror(err));
+		lconn_free(c);
+		return;
+	}
+
+	/* connected; the worker runs TLS and the MQTT client side */
+	lconn_handoff(c);
+}
blob - 412c68b097457a0a5dee2463afc903894b23fff1508e1de12cda91ff6f49e84e
blob + 7978b018372beb9abf030ddad5e13e1d12525ebb6c4eae1b54657567c789a535
--- mqttd/mqttd.8	
+++ mqttd/mqttd.8	
@@ -71,9 +71,19 @@ broker hands it and inserts them over HTTPS, acknowled
 warehouse confirmed.
 The parent resolves names and signs its JWTs so it needs neither DNS nor
 the private key.
+.It listen
+Owns the listening sockets.
+It accepts connections, consumes the PROXY protocol header where a
+listener expects one, dials outbound bridges and hands every connected
+descriptor to the sockets worker with the fewest connections.
 .It sockets
-Accepts connections, runs TLS, splits the byte stream into control
-packets and relays them to the broker.
+A configurable number of worker processes, see
+.Ic workers
+in
+.Xr mqttd.conf 5 ,
+which run TLS, split the byte stream into control packets and relay
+them to the broker.
+A worker never opens a socket itself.
 .El
 .Pp
 Apart from auth, all children run
@@ -141,6 +151,7 @@ the configuration is re-read: limits, users, ACLs, TLS
 listeners take effect immediately without dropping connections; a
 removed listener's existing clients stay connected.
 Changes to
+.Ic workers ,
 .Ic persistence ,
 .Ic auth
 or bridges are reported in the log and need a restart.
blob - 1cc61a93d33ca7110f189e306477af4a169866335eaecfbea136bb9822813120
blob + 0c3eb7fa9ed2ccc51118e6d91519e7254acfe5f6d4ee3b1e0a59735f62b94796
--- mqttd/mqttd.c	
+++ mqttd/mqttd.c	
@@ -18,7 +18,7 @@
 /*
  * The parent keeps root for the privileged setup: reading the
  * configuration and certificates, binding the listeners and forking the
- * control, broker and sockets processes.  Afterwards it only relays
+ * control, broker, listen and sockets processes.  Afterwards it only relays
  * control requests and waits for its children.
  */
 
@@ -66,6 +66,7 @@ static void	 parent_dispatch_auth(int, short, void *);
 static void	 parent_dispatch_snow(int, short, void *);
 static void	 parent_snow_resolve(struct imsg *);
 static void	 parent_snow_jwt(struct imsg *);
+static void	 parent_dispatch_listen(int, short, void *);
 static void	 parent_dispatch_sockets(int, short, void *);
 static void	 parent_sighdlr(int, short, void *);
 static void	 parent_show_info(struct imsg *);
@@ -79,6 +80,7 @@ static struct imsgev	*parent_iev_broker;
 static struct imsgev	*parent_iev_storage;
 static struct imsgev	*parent_iev_auth;
 static struct imsgev	*parent_iev_snow;
+static struct imsgev	*parent_iev_listen;
 static struct imsgev	*parent_iev_socks;
 
 static struct event	 ev_sigint;
@@ -211,7 +213,7 @@ parent_dispatch_control(int fd, short event, void *arg
 	struct imsgev	*iev = arg;
 	struct imsgbuf	*ibuf = &iev->ibuf;
 	struct imsg	 imsg;
-	int		 n, verbose;
+	int		 i, n, verbose;
 
 	if (event & EV_READ) {
 		if ((n = imsgbuf_read(ibuf)) == -1)
@@ -252,14 +254,16 @@ parent_dispatch_control(int fd, short event, void *arg
 			imsg_forward_event(parent_iev_broker, &imsg);
 			break;
 		case IMSG_CTL_SHOW_LISTENERS_REQUEST:
-			imsg_forward_event(parent_iev_socks, &imsg);
+			imsg_forward_event(parent_iev_listen, &imsg);
 			break;
 		case IMSG_CTL_VERBOSE:
 			IMSG_SIZE_CHECK(&imsg, &verbose);
 			memcpy(&verbose, imsg.data, sizeof(verbose));
 			log_setverbose(verbose);
 			imsg_forward_event(parent_iev_broker, &imsg);
-			imsg_forward_event(parent_iev_socks, &imsg);
+			imsg_forward_event(parent_iev_listen, &imsg);
+			for (i = 0; i < env->conf.workers; i++)
+				imsg_forward_event(&parent_iev_socks[i], &imsg);
 			break;
 		default:
 			log_warnx("%s: unexpected imsg %d from control",
@@ -531,6 +535,52 @@ parent_dispatch_snow(int fd, short event, void *arg)
 }
 
 static void
+parent_dispatch_listen(int fd, short event, void *arg)
+{
+	struct imsgev	*iev = arg;
+	struct imsgbuf	*ibuf = &iev->ibuf;
+	struct imsg	 imsg;
+	int		 n;
+
+	if (event & EV_READ) {
+		if ((n = imsgbuf_read(ibuf)) == -1)
+			fatal("%s: imsgbuf_read", __func__);
+		if (n == 0) {
+			log_warnx("%s: lost listen process", __func__);
+			parent_shutdown(env);
+		}
+	}
+	if (event & EV_WRITE) {
+		if (imsgbuf_write(ibuf) == -1) {
+			if (errno == EPIPE) {
+				log_warnx("%s: lost listen process", __func__);
+				parent_shutdown(env);
+			}
+			fatal("%s: imsgbuf_write", __func__);
+		}
+	}
+
+	for (;;) {
+		if ((n = imsgbuf_get(ibuf, &imsg)) == -1)
+			fatal("%s: imsgbuf_get", __func__);
+		if (n == 0)
+			break;
+
+		switch (imsg.hdr.type) {
+		case IMSG_CFG_DONE:
+			parent_configure_done(env);
+			break;
+		default:
+			log_warnx("%s: unexpected imsg %d from listen",
+			    __func__, imsg.hdr.type);
+			break;
+		}
+		imsg_free(&imsg);
+	}
+	imsg_event_add(iev);
+}
+
+static void
 parent_dispatch_sockets(int fd, short event, void *arg)
 {
 	struct imsgev	*iev = arg;
@@ -611,7 +661,7 @@ main(int argc, char **argv)
 	struct mqttd	*e;
 	const char	*conffile = MQTTD_CONF;
 	struct stat	 st;
-	int		 ch;
+	int		 ch, i, n;
 	int		 pipe_control[2];
 	int		 pipe_broker[2];
 	int		 pipe_storage[2];
@@ -621,9 +671,13 @@ main(int argc, char **argv)
 	int		 pipe_broker_auth[2];
 	int		 pipe_snow[2];
 	int		 pipe_broker_snow[2];
-	int		 pipe_socks[2];
-	int		 pipe_control_socks[2];
-	int		 pipe_broker_socks[2];
+	int		 pipe_listen[2];
+	int		 pipe_control_listen[2];
+	int		 pipe_broker_listen[2];
+	int		 pipe_socks[MQTTD_MAX_WORKERS][2];
+	int		 pipe_listen_socks[MQTTD_MAX_WORKERS][2];
+	int		 pipe_broker_socks[MQTTD_MAX_WORKERS][2];
+	int		 fds[MQTTD_MAX_WORKERS];
 
 	if ((e = calloc(1, sizeof(*e))) == NULL)
 		fatal("%s: calloc", __func__);
@@ -722,14 +776,23 @@ main(int argc, char **argv)
 	    socketpair(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, PF_UNSPEC,
 	    pipe_broker_snow) == -1 ||
 	    socketpair(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, PF_UNSPEC,
-	    pipe_socks) == -1 ||
+	    pipe_listen) == -1 ||
 	    socketpair(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, PF_UNSPEC,
-	    pipe_control_socks) == -1 ||
+	    pipe_control_listen) == -1 ||
 	    socketpair(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, PF_UNSPEC,
-	    pipe_broker_socks) == -1)
+	    pipe_broker_listen) == -1)
 		fatal("%s: socketpair", __func__);
+	for (i = 0; i < e->conf.workers; i++) {
+		if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, PF_UNSPEC,
+		    pipe_socks[i]) == -1 ||
+		    socketpair(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, PF_UNSPEC,
+		    pipe_listen_socks[i]) == -1 ||
+		    socketpair(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, PF_UNSPEC,
+		    pipe_broker_socks[i]) == -1)
+			fatal("%s: socketpair", __func__);
+	}
 
-	/* control: parent[1], control_broker[0], control_socks[0] */
+	/* control: parent[1], control_broker[0], control_listen[0] */
 	switch (fork()) {
 	case -1:
 		fatal("%s: fork", __func__);
@@ -750,13 +813,21 @@ main(int argc, char **argv)
 		close(pipe_broker_auth[1]);
 		close(pipe_broker_snow[0]);
 		close(pipe_broker_snow[1]);
-		close(pipe_socks[0]);
-		close(pipe_socks[1]);
-		close(pipe_control_socks[1]);
-		close(pipe_broker_socks[0]);
-		close(pipe_broker_socks[1]);
+		close(pipe_listen[0]);
+		close(pipe_listen[1]);
+		close(pipe_control_listen[1]);
+		close(pipe_broker_listen[0]);
+		close(pipe_broker_listen[1]);
+		for (i = 0; i < e->conf.workers; i++) {
+			close(pipe_socks[i][0]);
+			close(pipe_socks[i][1]);
+			close(pipe_listen_socks[i][0]);
+			close(pipe_listen_socks[i][1]);
+			close(pipe_broker_socks[i][0]);
+			close(pipe_broker_socks[i][1]);
+		}
 		control(pipe_control[1], pipe_control_broker[0],
-		    pipe_control_socks[0]);
+		    pipe_control_listen[0]);
 		/* NOTREACHED */
 	}
 
@@ -781,12 +852,20 @@ main(int argc, char **argv)
 		close(pipe_broker_auth[0]);
 		close(pipe_broker_auth[1]);
 		close(pipe_broker_snow[0]);
-		close(pipe_socks[0]);
-		close(pipe_socks[1]);
-		close(pipe_control_socks[0]);
-		close(pipe_control_socks[1]);
-		close(pipe_broker_socks[0]);
-		close(pipe_broker_socks[1]);
+		close(pipe_listen[0]);
+		close(pipe_listen[1]);
+		close(pipe_control_listen[0]);
+		close(pipe_control_listen[1]);
+		close(pipe_broker_listen[0]);
+		close(pipe_broker_listen[1]);
+		for (i = 0; i < e->conf.workers; i++) {
+			close(pipe_socks[i][0]);
+			close(pipe_socks[i][1]);
+			close(pipe_listen_socks[i][0]);
+			close(pipe_listen_socks[i][1]);
+			close(pipe_broker_socks[i][0]);
+			close(pipe_broker_socks[i][1]);
+		}
 		snowflake(pipe_snow[1], pipe_broker_snow[1]);
 		/* NOTREACHED */
 	}
@@ -812,12 +891,20 @@ main(int argc, char **argv)
 		close(pipe_broker_auth[1]);
 		close(pipe_broker_snow[0]);
 		close(pipe_broker_snow[1]);
-		close(pipe_socks[0]);
-		close(pipe_socks[1]);
-		close(pipe_control_socks[0]);
-		close(pipe_control_socks[1]);
-		close(pipe_broker_socks[0]);
-		close(pipe_broker_socks[1]);
+		close(pipe_listen[0]);
+		close(pipe_listen[1]);
+		close(pipe_control_listen[0]);
+		close(pipe_control_listen[1]);
+		close(pipe_broker_listen[0]);
+		close(pipe_broker_listen[1]);
+		for (i = 0; i < e->conf.workers; i++) {
+			close(pipe_socks[i][0]);
+			close(pipe_socks[i][1]);
+			close(pipe_listen_socks[i][0]);
+			close(pipe_listen_socks[i][1]);
+			close(pipe_broker_socks[i][0]);
+			close(pipe_broker_socks[i][1]);
+		}
 		storage(pipe_storage[1], pipe_broker_storage[1]);
 		/* NOTREACHED */
 	}
@@ -843,12 +930,20 @@ main(int argc, char **argv)
 		close(pipe_broker_auth[0]);
 		close(pipe_broker_snow[0]);
 		close(pipe_broker_snow[1]);
-		close(pipe_socks[0]);
-		close(pipe_socks[1]);
-		close(pipe_control_socks[0]);
-		close(pipe_control_socks[1]);
-		close(pipe_broker_socks[0]);
-		close(pipe_broker_socks[1]);
+		close(pipe_listen[0]);
+		close(pipe_listen[1]);
+		close(pipe_control_listen[0]);
+		close(pipe_control_listen[1]);
+		close(pipe_broker_listen[0]);
+		close(pipe_broker_listen[1]);
+		for (i = 0; i < e->conf.workers; i++) {
+			close(pipe_socks[i][0]);
+			close(pipe_socks[i][1]);
+			close(pipe_listen_socks[i][0]);
+			close(pipe_listen_socks[i][1]);
+			close(pipe_broker_socks[i][0]);
+			close(pipe_broker_socks[i][1]);
+		}
 		auth(pipe_auth[1], pipe_broker_auth[1]);
 		/* NOTREACHED */
 	}
@@ -871,18 +966,29 @@ main(int argc, char **argv)
 		close(pipe_broker_storage[1]);
 		close(pipe_broker_auth[1]);
 		close(pipe_broker_snow[1]);
-		close(pipe_socks[0]);
-		close(pipe_socks[1]);
-		close(pipe_control_socks[0]);
-		close(pipe_control_socks[1]);
-		close(pipe_broker_socks[1]);
+		close(pipe_listen[0]);
+		close(pipe_listen[1]);
+		close(pipe_control_listen[0]);
+		close(pipe_control_listen[1]);
+		close(pipe_broker_listen[1]);
+		for (i = 0; i < e->conf.workers; i++) {
+			close(pipe_socks[i][0]);
+			close(pipe_socks[i][1]);
+			close(pipe_listen_socks[i][0]);
+			close(pipe_listen_socks[i][1]);
+			close(pipe_broker_socks[i][1]);
+			fds[i] = pipe_broker_socks[i][0];
+		}
 		broker(pipe_broker[1], pipe_control_broker[1],
 		    pipe_broker_storage[0], pipe_broker_auth[0],
-		    pipe_broker_snow[0], pipe_broker_socks[0]);
+		    pipe_broker_snow[0], pipe_broker_listen[0], fds);
 		/* NOTREACHED */
 	}
 
-	/* sockets: socks[1], control_socks[1], broker_socks[1] */
+	/*
+	 * listen: listen[1], control_listen[1], broker_listen[1] and
+	 * listen_socks[i][0]
+	 */
 	switch (fork()) {
 	case -1:
 		fatal("%s: fork", __func__);
@@ -905,14 +1011,68 @@ main(int argc, char **argv)
 		close(pipe_broker_auth[1]);
 		close(pipe_broker_snow[0]);
 		close(pipe_broker_snow[1]);
-		close(pipe_socks[0]);
-		close(pipe_control_socks[0]);
-		close(pipe_broker_socks[0]);
-		sockets(pipe_socks[1], pipe_control_socks[1],
-		    pipe_broker_socks[1]);
+		close(pipe_listen[0]);
+		close(pipe_control_listen[0]);
+		close(pipe_broker_listen[0]);
+		for (i = 0; i < e->conf.workers; i++) {
+			close(pipe_socks[i][0]);
+			close(pipe_socks[i][1]);
+			close(pipe_listen_socks[i][1]);
+			close(pipe_broker_socks[i][0]);
+			close(pipe_broker_socks[i][1]);
+			fds[i] = pipe_listen_socks[i][0];
+		}
+		listen_main(pipe_listen[1], pipe_control_listen[1],
+		    pipe_broker_listen[1], fds);
 		/* NOTREACHED */
 	}
 
+	/* sockets n: socks[n][1], listen_socks[n][1], broker_socks[n][1] */
+	for (n = 0; n < e->conf.workers; n++) {
+		switch (fork()) {
+		case -1:
+			fatal("%s: fork", __func__);
+		case 0:
+			close(pipe_control[0]);
+			close(pipe_control[1]);
+			close(pipe_broker[0]);
+			close(pipe_broker[1]);
+			close(pipe_storage[0]);
+			close(pipe_storage[1]);
+			close(pipe_auth[0]);
+			close(pipe_auth[1]);
+			close(pipe_snow[0]);
+			close(pipe_snow[1]);
+			close(pipe_control_broker[0]);
+			close(pipe_control_broker[1]);
+			close(pipe_broker_storage[0]);
+			close(pipe_broker_storage[1]);
+			close(pipe_broker_auth[0]);
+			close(pipe_broker_auth[1]);
+			close(pipe_broker_snow[0]);
+			close(pipe_broker_snow[1]);
+			close(pipe_listen[0]);
+			close(pipe_listen[1]);
+			close(pipe_control_listen[0]);
+			close(pipe_control_listen[1]);
+			close(pipe_broker_listen[0]);
+			close(pipe_broker_listen[1]);
+			for (i = 0; i < e->conf.workers; i++) {
+				close(pipe_socks[i][0]);
+				close(pipe_listen_socks[i][0]);
+				close(pipe_broker_socks[i][0]);
+				if (i == n)
+					continue;
+				close(pipe_socks[i][1]);
+				close(pipe_listen_socks[i][1]);
+				close(pipe_broker_socks[i][1]);
+			}
+			sockets(pipe_socks[n][1], pipe_listen_socks[n][1],
+			    pipe_broker_socks[n][1]);
+			/* NOTREACHED */
+		}
+	}
+
 	/* parent: the [0] end of every pipe to a child */
 	close(pipe_control[1]);
 	close(pipe_broker[1]);
@@ -927,11 +1087,18 @@ main(int argc, char **argv)
 	close(pipe_broker_auth[1]);
 	close(pipe_broker_snow[0]);
 	close(pipe_broker_snow[1]);
-	close(pipe_socks[1]);
-	close(pipe_control_socks[0]);
-	close(pipe_control_socks[1]);
-	close(pipe_broker_socks[0]);
-	close(pipe_broker_socks[1]);
+	close(pipe_listen[1]);
+	close(pipe_control_listen[0]);
+	close(pipe_control_listen[1]);
+	close(pipe_broker_listen[0]);
+	close(pipe_broker_listen[1]);
+	for (i = 0; i < e->conf.workers; i++) {
+		close(pipe_socks[i][1]);
+		close(pipe_listen_socks[i][0]);
+		close(pipe_listen_socks[i][1]);
+		close(pipe_broker_socks[i][0]);
+		close(pipe_broker_socks[i][1]);
+	}
 
 	event_init();
 
@@ -972,18 +1139,25 @@ main(int argc, char **argv)
 		fatal("%s: calloc", __func__);
 	imsg_event_init(parent_iev_snow, pipe_snow[0], parent_dispatch_snow);
 
-	if ((parent_iev_socks = calloc(1, sizeof(*parent_iev_socks))) == NULL)
+	if ((parent_iev_listen = calloc(1, sizeof(*parent_iev_listen))) == NULL)
 		fatal("%s: calloc", __func__);
-	imsg_event_init(parent_iev_socks, pipe_socks[0],
-	    parent_dispatch_sockets);
+	imsg_event_init(parent_iev_listen, pipe_listen[0],
+	    parent_dispatch_listen);
 
+	if ((parent_iev_socks = calloc(e->conf.workers,
+	    sizeof(*parent_iev_socks))) == NULL)
+		fatal("%s: calloc", __func__);
+	for (i = 0; i < e->conf.workers; i++)
+		imsg_event_init(&parent_iev_socks[i], pipe_socks[i][0],
+		    parent_dispatch_sockets);
+
 	if (parent_configure(e) == -1)
 		fatalx("configuration failed");
 
 	/*
 	 * A reload re-parses (rpath dns route), may bind new listeners
 	 * (inet unix cpath fattr) and hands their descriptors to the
-	 * sockets process (sendfd).
+	 * listen process (sendfd).
 	 */
 	if (pledge("stdio rpath inet dns route proc sendfd unix cpath fattr",
 	    NULL) == -1)
@@ -1001,26 +1175,27 @@ parent_configure(struct mqttd *e)
 	struct user		*u;
 	struct acl		*a;
 	struct bridge_def	*b;
-	int			 fd;
+	struct bridge_config	 bc;
+	int			 n;
 
 	TAILQ_FOREACH(l, &e->listeners, entry) {
-		if (sockets_create_listener(l) == -1)
+		if (listen_bind(l) == -1)
 			return (-1);
 		if (listener_tls_load(l) == -1)
 			return (-1);
 
-		if ((fd = dup(l->fd)) == -1) {
-			log_warn("%s: dup", __func__);
+		/* listen takes the descriptor, the workers the TLS material */
+		if (config_send_listener(parent_iev_listen, l, l->fd) == -1)
 			return (-1);
-		}
-		if (config_send_listener(parent_iev_socks, l, fd) == -1)
-			return (-1);
-		if (config_send_tls(parent_iev_socks, l) == -1)
-			return (-1);
-
-		/* the sockets process holds its own copy now */
-		close(l->fd);
 		l->fd = -1;
+		for (n = 0; n < e->conf.workers; n++) {
+			if (config_send_listener(&parent_iev_socks[n], l,
+			    -1) == -1)
+				return (-1);
+			if (config_send_tls(&parent_iev_socks[n], l) == -1)
+				return (-1);
+		}
+
 		freezero(l->conf.tls_key, l->conf.tls_key_len);
 		l->conf.tls_key = NULL;
 		l->conf.tls_key_len = 0;
@@ -1034,8 +1209,11 @@ parent_configure(struct mqttd *e)
 		return (-1);
 	if (config_send_global(parent_iev_snow, e) == -1)
 		return (-1);
-	if (config_send_global(parent_iev_socks, e) == -1)
+	if (config_send_global(parent_iev_listen, e) == -1)
 		return (-1);
+	for (n = 0; n < e->conf.workers; n++)
+		if (config_send_global(&parent_iev_socks[n], e) == -1)
+			return (-1);
 
 	TAILQ_FOREACH(u, &e->users, entry)
 		if (config_send_user(parent_iev_broker, u) == -1)
@@ -1045,8 +1223,9 @@ parent_configure(struct mqttd *e)
 			return (-1);
 
 	/*
-	 * Bridges: the broker gets the whole definition, the sockets
-	 * process that will dial out gets the address and CA bundle.
+	 * Bridges: the broker gets the whole definition, listen the address
+	 * to dial and every worker the address and CA bundle, both without
+	 * the password.
 	 */
 	TAILQ_FOREACH(b, &e->bridges, entry) {
 		if (b->snow != NULL) {
@@ -1089,11 +1268,19 @@ parent_configure(struct mqttd *e)
 		}
 		if (config_send_bridge(parent_iev_broker, b) == -1)
 			return (-1);
-		if (imsg_compose_event(parent_iev_socks, IMSG_CFG_BRIDGE, 0, 0,
-		    -1, &b->conf, sizeof(b->conf)) == -1)
+		bc = b->conf;
+		explicit_bzero(bc.password, sizeof(bc.password));
+		if (imsg_compose_event(parent_iev_listen, IMSG_CFG_BRIDGE, 0, 0,
+		    -1, &bc, sizeof(bc)) == -1)
 			return (-1);
-		if (config_send_bridge_ca(parent_iev_socks, b) == -1)
-			return (-1);
+		for (n = 0; n < e->conf.workers; n++) {
+			if (imsg_compose_event(&parent_iev_socks[n],
+			    IMSG_CFG_BRIDGE, 0, 0, -1, &bc, sizeof(bc)) == -1)
+				return (-1);
+			if (config_send_bridge_ca(&parent_iev_socks[n],
+			    b) == -1)
+				return (-1);
+		}
 		explicit_bzero(b->conf.password, sizeof(b->conf.password));
 	}
 
@@ -1102,13 +1289,16 @@ parent_configure(struct mqttd *e)
 	 * The broker only acknowledges once storage has replayed the last
 	 * snapshot into it, so no client connects to a half restored state.
 	 */
-	e->cfg_pending = 5;
+	e->cfg_pending = 5 + e->conf.workers;
 	imsg_compose_event(parent_iev_storage, IMSG_CFG_DONE, 0, 0, -1, NULL,
 	    0);
 	imsg_compose_event(parent_iev_auth, IMSG_CFG_DONE, 0, 0, -1, NULL, 0);
 	imsg_compose_event(parent_iev_snow, IMSG_CFG_DONE, 0, 0, -1, NULL, 0);
 	imsg_compose_event(parent_iev_broker, IMSG_CFG_DONE, 0, 0, -1, NULL, 0);
-	imsg_compose_event(parent_iev_socks, IMSG_CFG_DONE, 0, 0, -1, NULL, 0);
+	imsg_compose_event(parent_iev_listen, IMSG_CFG_DONE, 0, 0, -1, NULL, 0);
+	for (n = 0; n < e->conf.workers; n++)
+		imsg_compose_event(&parent_iev_socks[n], IMSG_CFG_DONE, 0, 0,
+		    -1, NULL, 0);
 
 	return (0);
 }
@@ -1116,6 +1306,8 @@ parent_configure(struct mqttd *e)
 static void
 parent_configure_done(struct mqttd *e)
 {
+	int	 n;
+
 	if (e->cfg_pending == 0) {
 		log_warnx("%s: configuration already finished", __func__);
 		return;
@@ -1132,9 +1324,14 @@ parent_configure_done(struct mqttd *e)
 	    0);
 	imsg_compose_event(parent_iev_auth, IMSG_CTL_START, 0, 0, -1, NULL, 0);
 	imsg_compose_event(parent_iev_snow, IMSG_CTL_START, 0, 0, -1, NULL, 0);
-	imsg_compose_event(parent_iev_socks, IMSG_CTL_START, 0, 0, -1, NULL, 0);
+	/* workers first, so listen never hands over to an unprepared one */
+	for (n = 0; n < e->conf.workers; n++)
+		imsg_compose_event(&parent_iev_socks[n], IMSG_CTL_START, 0, 0,
+		    -1, NULL, 0);
+	imsg_compose_event(parent_iev_listen, IMSG_CTL_START, 0, 0, -1, NULL, 0);
 
-	log_info("%s ready, %u listener(s)", getprogname(), e->nlisteners);
+	log_info("%s ready, %u listener(s), %u worker(s)", getprogname(),
+	    e->nlisteners, e->conf.workers);
 }
 
 static int
@@ -1178,7 +1375,7 @@ parent_reload(struct mqttd *e)
 	struct user		*u;
 	struct acl		*a;
 	struct bridge_def	*b;
-	int			 fd;
+	int			 i;
 
 	log_info("reloading configuration from %s", e->conffile);
 
@@ -1198,6 +1395,10 @@ parent_reload(struct mqttd *e)
 		return;
 	}
 
+	if (n->conf.workers != e->conf.workers) {
+		log_warnx("workers changed, a restart is needed for that");
+		n->conf.workers = e->conf.workers;
+	}
 	if (n->conf.persist != e->conf.persist ||
 	    strcmp(n->dbdir, e->dbdir) != 0) {
 		log_warnx("persistence settings changed, a restart is needed "
@@ -1244,27 +1445,50 @@ parent_reload(struct mqttd *e)
 				    "material", nl->conf.name);
 				continue;
 			}
-			if (config_send_tls(parent_iev_socks, nl) == 0)
-				imsg_compose_event(parent_iev_socks,
+			for (i = 0; i < e->conf.workers; i++) {
+				if (config_send_tls(&parent_iev_socks[i],
+				    nl) == -1)
+					break;
+				imsg_compose_event(&parent_iev_socks[i],
 				    IMSG_CFG_TLS_DONE, 0, 0, -1, &nl->conf.id,
 				    sizeof(nl->conf.id));
+			}
 		} else {
-			if (sockets_create_listener(nl) == -1 ||
+			if (listen_bind(nl) == -1 ||
 			    listener_tls_load(nl) == -1) {
 				log_warnx("listener %s could not be created, "
 				    "skipped", nl->conf.name);
 				nl->conf.id = 0;
 				continue;
 			}
-			if ((fd = dup(nl->fd)) != -1 &&
-			    config_send_listener(parent_iev_socks, nl,
-			    fd) == 0 &&
-			    config_send_tls(parent_iev_socks, nl) == 0)
-				imsg_compose_event(parent_iev_socks,
+			/*
+			 * The workers hear about the listener first, so a
+			 * connection listen hands over finds its TLS
+			 * material in place.
+			 */
+			for (i = 0; i < e->conf.workers; i++) {
+				if (config_send_listener(&parent_iev_socks[i],
+				    nl, -1) == -1 ||
+				    config_send_tls(&parent_iev_socks[i],
+				    nl) == -1)
+					break;
+				imsg_compose_event(&parent_iev_socks[i],
 				    IMSG_CFG_LISTENER_START, 0, 0, -1,
 				    &nl->conf.id, sizeof(nl->conf.id));
-			close(nl->fd);
+			}
+			if (config_send_listener(parent_iev_listen, nl,
+			    nl->fd) == -1) {
+				log_warnx("listener %s could not be handed "
+				    "over, skipped", nl->conf.name);
+				close(nl->fd);
+				nl->fd = -1;
+				nl->conf.id = 0;
+				continue;
+			}
 			nl->fd = -1;
+			imsg_compose_event(parent_iev_listen,
+			    IMSG_CFG_LISTENER_START, 0, 0, -1, &nl->conf.id,
+			    sizeof(nl->conf.id));
 			log_info("listener %s added", nl->conf.name);
 		}
 		freezero(nl->conf.tls_key, nl->conf.tls_key_len);
@@ -1274,9 +1498,13 @@ parent_reload(struct mqttd *e)
 
 	TAILQ_FOREACH_SAFE(ol, &e->listeners, entry, tl) {
 		if (ol->fd != -2) {
-			imsg_compose_event(parent_iev_socks,
+			imsg_compose_event(parent_iev_listen,
 			    IMSG_CFG_LISTENER_DEL, 0, 0, -1, &ol->conf.id,
 			    sizeof(ol->conf.id));
+			for (i = 0; i < e->conf.workers; i++)
+				imsg_compose_event(&parent_iev_socks[i],
+				    IMSG_CFG_LISTENER_DEL, 0, 0, -1,
+				    &ol->conf.id, sizeof(ol->conf.id));
 			log_info("listener %s removed", ol->conf.name);
 		}
 		ol->fd = -1;
@@ -1311,7 +1539,9 @@ parent_reload(struct mqttd *e)
 	config_send_global(parent_iev_storage, e);
 	config_send_global(parent_iev_auth, e);
 	config_send_global(parent_iev_snow, e);
-	config_send_global(parent_iev_socks, e);
+	config_send_global(parent_iev_listen, e);
+	for (i = 0; i < e->conf.workers; i++)
+		config_send_global(&parent_iev_socks[i], e);
 
 	imsg_compose_event(parent_iev_broker, IMSG_CFG_AUTH_RESET, 0, 0, -1,
 	    NULL, 0);
@@ -1330,6 +1560,7 @@ parent_show_info(struct imsg *imsg)
 
 	memset(&pi, 0, sizeof(pi));
 	pi.verbose = log_getverbose();
+	pi.workers = env->conf.workers;
 	pi.nlisteners = env->nlisteners;
 	pi.started = env->started;
 
@@ -1347,7 +1578,7 @@ parent_shutdown(struct mqttd *e)
 	struct acl		*a;
 	struct bridge_def	*b;
 	pid_t			 pid;
-	int			 status;
+	int			 n, status;
 
 	/* closing the pipes tells the children to exit */
 	imsg_event_shutdown(parent_iev_control);
@@ -1360,7 +1591,10 @@ parent_shutdown(struct mqttd *e)
 	free(parent_iev_auth);
 	imsg_event_shutdown(parent_iev_snow);
 	free(parent_iev_snow);
-	imsg_event_shutdown(parent_iev_socks);
+	imsg_event_shutdown(parent_iev_listen);
+	free(parent_iev_listen);
+	for (n = 0; n < e->conf.workers; n++)
+		imsg_event_shutdown(&parent_iev_socks[n]);
 	free(parent_iev_socks);
 
 	log_debug("waiting for children to terminate");
blob - 7dab33d363d26b173b9e0d1e43dc33514febec4ecba7e1cc34cae888a24bcbeb
blob + aa0076eb4971720889d62060520ab510e1b9ec70030b88de60bf07911957203d
--- mqttd/mqttd.conf.5	
+++ mqttd/mqttd.conf.5	
@@ -238,6 +238,13 @@ The default of 0 means unlimited.
 Publishes per second for this user, overriding
 .Ic max rate .
 .El
+.It Ic workers Ar number
+Run the specified number of sockets worker processes.
+The listen process hands each new connection to the worker with the
+fewest connections.
+The default is 2, the maximum 32.
+.Ic prefork
+is accepted as a deprecated alias.
 .El
 .Sh LISTENERS
 A listener is declared with the
blob - 3ab7ab937a64b0e804f57ed389718f7efae00478bcfb678afbd79f1af0b2f382
blob + 3d800bb72daeaf70f6da6daadbc86ee090d11b05a477cb48f3ccf8dcbd0c6da5
--- mqttd/mqttd.h	
+++ mqttd/mqttd.h	
@@ -77,6 +77,8 @@
 #define MQTTD_MAXPEER		64
 #define MQTTD_MAXHASH		128
 #define MQTTD_MAXIFACE		16
+#define MQTTD_MAX_WORKERS	32
+#define MQTTD_DEFAULT_WORKERS	2
 
 #define MQTT_PORT		1883
 #define MQTTS_PORT		8883
@@ -170,6 +172,10 @@ enum imsg_type {
 	IMSG_CONN_ERROR,
 	IMSG_CONN_KEEPALIVE,
 
+	/* listen <-> sockets */
+	IMSG_CONN_ACCEPT,
+	IMSG_CONN_DONE,
+
 	/* broker <-> storage */
 	IMSG_DB_RECORD,
 	IMSG_DB_RESTORE_END,
@@ -209,6 +215,7 @@ struct imsgev {
 
 /* global broker limits, fixed size so it travels in a single imsg */
 struct mqttd_global {
+	uint16_t		 workers;
 	uint32_t		 max_clients;
 	uint32_t		 max_packet_size;
 	uint16_t		 max_inflight;
@@ -363,8 +370,10 @@ struct pf_ban {
 
 /*
  * Bridges.  The parent resolves the remote address and loads the CA;
- * the sockets process named by proc opens the connection on request and
- * the broker drives the MQTT client side of it.
+ * the listen process dials the remote on request and hands the
+ * connection to a sockets worker, which runs TLS, and the broker drives
+ * the MQTT client side of it.  proc is MQTTD_PROC_SNOW for a sink, whose
+ * remote is the snowflake process instead.
  */
 #define BRIDGE_IN		0x01
 #define BRIDGE_OUT		0x02
@@ -554,6 +563,25 @@ struct conn_keepalive {
 };
 
 /*
+ * listen -> sockets: a connected descriptor travels with this message.
+ * bridge_id is set for an outbound bridge the worker has to run TLS and
+ * MQTT over as a client; the PROXY header, if any, is already consumed
+ * and peer reflects it.
+ */
+struct conn_accept {
+	uint32_t		 id;
+	uint32_t		 listener_id;
+	uint32_t		 bridge_id;
+	char			 peer[MQTTD_MAXPEER];
+};
+
+/* sockets -> listen: the connection is gone, release its load */
+struct conn_done {
+	uint32_t		 id;
+	uint32_t		 listener_id;
+};
+
+/*
  * Snapshot records.  The broker serialises its state into a stream of
  * typed records which the storage process writes out verbatim and plays
  * back at startup; storage never interprets record contents.
@@ -589,6 +617,7 @@ struct ctl_filter {
 
 struct ctl_parent_info {
 	int			 verbose;
+	uint16_t		 workers;
 	uint32_t		 nlisteners;
 	time_t			 started;
 };
@@ -699,14 +728,18 @@ void	 proc_privdrop(const char *);
 void	 proc_privdrop_dir(const char *, const char *);
 int	 sockaddr_ntop(struct sockaddr_storage *, char *, size_t);
 
+/* listen.c */
+__dead void
+	 listen_main(int, int, int, int *);
+int	 listen_bind(struct listener *);
+
 /* sockets.c */
 __dead void
 	 sockets(int, int, int);
-int	 sockets_create_listener(struct listener *);
 
 /* broker.c */
 __dead void
-	 broker(int, int, int, int, int, int);
+	 broker(int, int, int, int, int, int, int *);
 
 /* storage.c */
 __dead void
blob - 9f661dbe61297dd830d546740d9953f2ceeb132d28084dca8d2684872fd8d51a
blob + 66ec6f49d0fb8c02811cef9c217b2eb27fd965a6463b8f09021cade5b040bc79
--- mqttd/parse.y	
+++ mqttd/parse.y	
@@ -133,7 +133,7 @@ typedef struct {
 %token	PACKET PASSWORD PATH PERSISTENCE PF PORT PREFORK PROTOCOLS PROXY
 %token	PUBLISH QOS QUEUED RATE REMOTE RETAIN ROLE SERVERNAME SESSION SIZE
 %token	SNOWFLAKE SPARKPLUG SUBSCRIBE SYS TABLE TIMEOUT TLS TOPIC USER VERSION
-%token	WAREHOUSE WEBSOCKET WITHIN YES
+%token	WAREHOUSE WEBSOCKET WITHIN WORKERS YES
 
 %token	<v.string>	STRING
 %token	<v.number>	NUMBER
@@ -203,9 +203,21 @@ yesno		: YES				{ $$ = 1; }
 		| NO				{ $$ = 0; }
 		;
 
-main		: PREFORK NUMBER {
-			log_warnx("%s:%d: prefork is deprecated and ignored",
+main		: WORKERS NUMBER {
+			if ($2 <= 0 || $2 > MQTTD_MAX_WORKERS) {
+				yyerror("invalid number of workers: %lld", $2);
+				YYERROR;
+			}
+			conf->conf.workers = $2;
+		}
+		| PREFORK NUMBER {
+			log_warnx("%s:%d: prefork is deprecated, use workers",
 			    file->name, yylval.lineno);
+			if ($2 <= 0 || $2 > MQTTD_MAX_WORKERS) {
+				yyerror("invalid number of workers: %lld", $2);
+				YYERROR;
+			}
+			conf->conf.workers = $2;
 		}
 		| MAX CLIENTS NUMBER {
 			if ($3 < 0 || $3 > UINT32_MAX) {
@@ -1326,6 +1338,7 @@ lookup(char *s)
 		{ "warehouse",		WAREHOUSE },
 		{ "websocket",		WEBSOCKET },
 		{ "within",		WITHIN },
+		{ "workers",		WORKERS },
 		{ "yes",		YES }
 	};
 	const struct keywords	*p;
@@ -1699,6 +1712,7 @@ parse_config(const char *filename, struct mqttd *xconf
 	conf = xconf;
 	config_init(conf);
 
+	conf->conf.workers = MQTTD_DEFAULT_WORKERS;
 	conf->conf.max_clients = 0;
 	conf->conf.max_packet_size = MQTTD_DEFAULT_MAX_PACKET;
 	conf->conf.max_inflight = MQTTD_DEFAULT_MAX_INFLIGHT;
blob - a7c5423a0b6ac849c7d3913446cb437090553318e95caabd5eaabd85e1b3a023
blob + ae1924f7ff00a8f96fb2a9e875ab333f2925a00833f19b9d27f506bb673430e4
--- mqttd/sockets.c	
+++ mqttd/sockets.c	
@@ -17,9 +17,11 @@
  */
 
 /*
- * The sockets processes own the network connections.  They accept, run
- * TLS, cut the byte stream into MQTT control packets and relay those to
- * the broker; everything the broker sends back is written out verbatim.
+ * The sockets workers own the client connections.  The listen process
+ * accepts and hands each descriptor over; a worker runs TLS, cuts the
+ * byte stream into MQTT control packets and relays those to the broker.
+ * Everything the broker sends back is written out verbatim.  A worker
+ * never opens a socket itself.
  */
 
 #include <sys/types.h>
@@ -31,16 +33,11 @@
 #include <sys/uio.h>
 
 #include <netinet/in.h>
-#include <arpa/inet.h>
 
-#include <sys/stat.h>
-#include <sys/un.h>
-
 #include <errno.h>
 #include <event.h>
 #include <fcntl.h>
 #include <imsg.h>
-#include <netdb.h>
 #include <resolv.h>
 #include <sha1.h>
 #include <signal.h>
@@ -53,11 +50,7 @@
 #include "mqttd.h"
 #include "mqtt.h"
 
-#define FD_RESERVE		8
 #define TLS_READ_SIZE		16384
-#define PROXY_V1_MAX		107
-#define PROXY_V2_MIN		16
-#define PROXY_PEEK		(PROXY_V2_MIN + 216)
 
 enum ws_state {
 	WS_NONE,
@@ -88,6 +81,7 @@ struct conn {
 	uint32_t		 id;
 	int			 fd;
 	struct listener		*l;
+	uint32_t		 listener_id;
 	struct sbridge		*bridge;
 	struct bufferevent	*bev;
 	struct tls		*tls_ctx;
@@ -116,7 +110,7 @@ void	 bufferevent_read_pressure_cb(struct evbuffer *, 
 	    void *);
 
 static void	 sockets_dispatch_parent(int, short, void *);
-static void	 sockets_dispatch_control(int, short, void *);
+static void	 sockets_dispatch_listen(int, short, void *);
 static void	 sockets_dispatch_broker(int, short, void *);
 static void	 sockets_sighdlr(int, short, void *);
 static __dead void
@@ -125,10 +119,7 @@ static void	 sockets_launch(void);
 static int	 sockets_listener_launch(struct listener *);
 static void	 sockets_listener_remove(struct listener *);
 static void	 sockets_rlimit(void);
-static int	 sockets_accept_reserve(int, struct sockaddr *, socklen_t *);
-static void	 sockets_accept(int, short, void *);
-static void	 sockets_accept_paused(int, short, void *);
-static void	 sockets_show_listeners(struct imsg *);
+static void	 sockets_conn_accept(struct imsg *);
 
 static struct conn
 		*conn_find(uint32_t);
@@ -145,9 +136,6 @@ static void	 conn_write_chunk(struct conn *, const str
 		    const uint8_t *);
 static void	 conn_begin(struct conn *);
 static void	 conn_open_send(struct conn *);
-static void	 conn_proxy_read(int, short, void *);
-static int	 proxy_parse_v1(struct conn *, const uint8_t *, size_t);
-static int	 proxy_parse_v2(struct conn *, const uint8_t *, size_t);
 static int	 ws_handshake(struct conn *);
 static int	 ws_decode(struct conn *);
 static void	 ws_send_frame(struct conn *, uint8_t, const uint8_t *,
@@ -165,19 +153,16 @@ static struct sbridge
 		*sbridge_find(uint32_t);
 static void	 sockets_bridge_config(struct imsg *);
 static void	 sockets_bridge_ca(struct imsg *);
-static void	 sockets_bridge_connect(struct imsg *);
-static void	 sockets_bridge_connected(int, short, void *);
 static void	 sockets_bridge_failed(struct conn *);
 
 static struct imsgev	*socks_iev_parent;
-static struct imsgev	*socks_iev_control;
+static struct imsgev	*socks_iev_listen;
 static struct imsgev	*socks_iev_broker;
 
 static struct conntree	 conns = RB_INITIALIZER(&conns);
 /* connections we stopped reading from while the broker pipe is full */
 static struct connq	 throttled_conns = TAILQ_HEAD_INITIALIZER(throttled_conns);
 static struct sbridgelist sbridges = TAILQ_HEAD_INITIALIZER(sbridges);
-static uint32_t		 conn_next_id;
 
 static struct event	 ev_sigint;
 static struct event	 ev_sigterm;
@@ -194,7 +179,7 @@ conn_cmp(struct conn *a, struct conn *b)
 }
 
 __dead void
-sockets(int fd_parent, int fd_control, int fd_broker)
+sockets(int fd_parent, int fd_listen, int fd_broker)
 {
 	log_procinit("sockets");
 	proc_privdrop("sockets");
@@ -221,17 +206,16 @@ sockets(int fd_parent, int fd_control, int fd_broker)
 		fatal("%s: calloc", __func__);
 	imsg_event_init(socks_iev_parent, fd_parent, sockets_dispatch_parent);
 
-	if ((socks_iev_control = calloc(1, sizeof(*socks_iev_control)))
-	    == NULL)
+	if ((socks_iev_listen = calloc(1, sizeof(*socks_iev_listen))) == NULL)
 		fatal("%s: calloc", __func__);
-	imsg_event_init(socks_iev_control, fd_control,
-	    sockets_dispatch_control);
+	imsg_event_init(socks_iev_listen, fd_listen, sockets_dispatch_listen);
 
 	if ((socks_iev_broker = calloc(1, sizeof(*socks_iev_broker))) == NULL)
 		fatal("%s: calloc", __func__);
 	imsg_event_init(socks_iev_broker, fd_broker, sockets_dispatch_broker);
 
-	if (pledge("stdio inet recvfd", NULL) == -1)
+	/* recvfd for the descriptors listen hands over; no socket calls */
+	if (pledge("stdio recvfd", NULL) == -1)
 		fatal("pledge");
 
 	event_dispatch();
@@ -279,18 +263,14 @@ sockets_shutdown(void)
 			tls_config_free(l->tls_config);
 			tls_free(l->tls_ctx);
 		}
-		if (event_initialized(&l->ev))
-			event_del(&l->ev);
-		if (evtimer_initialized(&l->pause))
-			evtimer_del(&l->pause);
 		TAILQ_REMOVE(&env->listeners, l, entry);
 		config_free_listener(l);
 	}
 
 	imsg_event_shutdown(socks_iev_parent);
 	free(socks_iev_parent);
-	imsg_event_shutdown(socks_iev_control);
-	free(socks_iev_control);
+	imsg_event_shutdown(socks_iev_listen);
+	free(socks_iev_listen);
 	imsg_event_shutdown(socks_iev_broker);
 	free(socks_iev_broker);
 
@@ -314,102 +294,25 @@ sockets_rlimit(void)
 		fatal("%s: setrlimit", __func__);
 }
 
-int
-sockets_create_listener(struct listener *l)
-{
-	struct sockaddr_storage	*ss = &l->conf.ss;
-	struct sockaddr_un	*sun;
-	int			 fd, on = 1;
-
-	if (ss->ss_family == AF_UNIX) {
-		sun = (struct sockaddr_un *)ss;
-		if (unlink(sun->sun_path) == -1 && errno != ENOENT) {
-			log_warn("%s: unlink %s", __func__, sun->sun_path);
-			return (-1);
-		}
-		fd = socket(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, 0);
-		if (fd == -1) {
-			log_warn("%s: socket", __func__);
-			return (-1);
-		}
-		if (bind(fd, (struct sockaddr *)sun, sizeof(*sun)) == -1) {
-			log_warn("%s: bind %s", __func__, sun->sun_path);
-			goto fail;
-		}
-		/* authentication still applies; let local users reach it */
-		if (chmod(sun->sun_path, MQTTD_UNIX_MODE) == -1) {
-			log_warn("%s: chmod %s", __func__, sun->sun_path);
-			goto fail;
-		}
-		if (listen(fd, SOMAXCONN) == -1) {
-			log_warn("%s: listen", __func__);
-			goto fail;
-		}
-		l->fd = fd;
-		return (0);
-	}
-
-	switch (ss->ss_family) {
-	case AF_INET:
-		((struct sockaddr_in *)ss)->sin_port = l->conf.port;
-		((struct sockaddr_in *)ss)->sin_len = sizeof(struct sockaddr_in);
-		break;
-	case AF_INET6:
-		((struct sockaddr_in6 *)ss)->sin6_port = l->conf.port;
-		((struct sockaddr_in6 *)ss)->sin6_len =
-		    sizeof(struct sockaddr_in6);
-		break;
-	default:
-		log_warnx("%s: unsupported address family", __func__);
-		return (-1);
-	}
-
-	if ((fd = socket(ss->ss_family, SOCK_STREAM | SOCK_NONBLOCK, 0)) == -1) {
-		log_warn("%s: socket", __func__);
-		return (-1);
-	}
-	if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) {
-		log_warn("%s: setsockopt SO_REUSEADDR", __func__);
-		goto fail;
-	}
-	if (ss->ss_family == AF_INET6 &&
-	    setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &on, sizeof(on)) == -1) {
-		log_warn("%s: setsockopt IPV6_V6ONLY", __func__);
-		goto fail;
-	}
-	if (bind(fd, (struct sockaddr *)ss, ss->ss_len) == -1) {
-		log_warn("%s: bind listener %s port %d", __func__, l->conf.name,
-		    ntohs(l->conf.port));
-		goto fail;
-	}
-	if (listen(fd, SOMAXCONN) == -1) {
-		log_warn("%s: listen", __func__);
-		goto fail;
-	}
-
-	l->fd = fd;
-	return (0);
-fail:
-	close(fd);
-	return (-1);
-}
-
+/*
+ * The TLS context for a listener; the descriptor itself lives in listen.
+ * Safe to call twice: a connection handed over early sets it up before
+ * the parent's start message arrives.
+ */
 static int
 sockets_listener_launch(struct listener *l)
 {
-	log_debug("%s: configuring listener %s[%u] (fd %d)", __func__,
-	    l->conf.name, l->conf.id, l->fd);
+	if (l->conf.tls && l->tls_ctx != NULL)
+		return (0);
 
+	log_debug("%s: configuring listener %s[%u]", __func__, l->conf.name,
+	    l->conf.id);
+
 	if (listener_tls_init(l) == -1) {
 		log_warnx("%s: tls setup failed for listener %s", __func__,
 		    l->conf.name);
 		return (-1);
 	}
-
-	event_set(&l->ev, l->fd, EV_READ | EV_PERSIST, sockets_accept, l);
-	if (event_add(&l->ev, NULL) == -1)
-		fatal("%s: event_add", __func__);
-	evtimer_set(&l->pause, sockets_accept_paused, l);
 	return (0);
 }
 
@@ -423,10 +326,6 @@ sockets_listener_remove(struct listener *l)
 		if (c->l == l)
 			c->l = NULL;
 
-	if (event_initialized(&l->ev))
-		event_del(&l->ev);
-	if (evtimer_initialized(&l->pause))
-		evtimer_del(&l->pause);
 	if (l->conf.tls) {
 		if (l->tls_config != NULL)
 			tls_config_free(l->tls_config);
@@ -438,7 +337,7 @@ sockets_listener_remove(struct listener *l)
 	config_free_listener(l);
 }
 
-/* listeners that arrive after startup need recvfd, so the pledge stays */
+/* TLS contexts for every listener the parent announced */
 static void
 sockets_launch(void)
 {
@@ -547,9 +446,6 @@ sockets_dispatch_parent(int fd, short event, void *arg
 			memcpy(&verbose, imsg.data, sizeof(verbose));
 			log_setverbose(verbose);
 			break;
-		case IMSG_CTL_SHOW_LISTENERS_REQUEST:
-			sockets_show_listeners(&imsg);
-			break;
 		default:
 			log_warnx("%s: unexpected imsg %d from parent",
 			    __func__, imsg.hdr.type);
@@ -561,7 +457,7 @@ sockets_dispatch_parent(int fd, short event, void *arg
 }
 
 static void
-sockets_dispatch_control(int fd, short event, void *arg)
+sockets_dispatch_listen(int fd, short event, void *arg)
 {
 	struct imsgev	*iev = arg;
 	struct imsgbuf	*ibuf = &iev->ibuf;
@@ -572,28 +468,35 @@ sockets_dispatch_control(int fd, short event, void *ar
 		if ((n = imsgbuf_read(ibuf)) == -1)
 			fatal("%s: imsgbuf_read", __func__);
 		if (n == 0) {
-			log_warnx("%s: lost control process", __func__);
+			log_warnx("%s: lost listen process", __func__);
 			sockets_shutdown();
 		}
 	}
 	if (event & EV_WRITE) {
 		if (imsgbuf_write(ibuf) == -1) {
 			if (errno == EPIPE) {
-				log_warnx("%s: lost control process", __func__);
+				log_warnx("%s: lost listen process", __func__);
 				sockets_shutdown();
 			}
 			fatal("%s: imsgbuf_write", __func__);
 		}
 	}
 
-	/* control only listens to us; its requests arrive via the parent */
 	for (;;) {
 		if ((n = imsgbuf_get(ibuf, &imsg)) == -1)
 			fatal("%s: imsgbuf_get", __func__);
 		if (n == 0)
 			break;
-		log_warnx("%s: unexpected imsg %d from control", __func__,
-		    imsg.hdr.type);
+
+		switch (imsg.hdr.type) {
+		case IMSG_CONN_ACCEPT:
+			sockets_conn_accept(&imsg);
+			break;
+		default:
+			log_warnx("%s: unexpected imsg %d from listen",
+			    __func__, imsg.hdr.type);
+			break;
+		}
 		imsg_free(&imsg);
 	}
 	imsg_event_add(iev);
@@ -680,9 +583,6 @@ sockets_dispatch_broker(int fd, short event, void *arg
 			else
 				conn_set_timer(c, (time_t)ka.keepalive * 3 / 2);
 			break;
-		case IMSG_BRIDGE_CONNECT:
-			sockets_bridge_connect(&imsg);
-			break;
 		default:
 			log_warnx("%s: unexpected imsg %d from broker",
 			    __func__, imsg.hdr.type);
@@ -693,118 +593,121 @@ sockets_dispatch_broker(int fd, short event, void *arg
 	imsg_event_add(iev);
 }
 
+/*
+ * The listen process hands us a connected descriptor.  Whatever happens
+ * to it from here, listen hears IMSG_CONN_DONE once so its counts stay
+ * right; a bridge that fails before it is open is reported to the broker
+ * so it can retry.
+ */
 static void
-sockets_show_listeners(struct imsg *imsg)
+sockets_conn_accept(struct imsg *imsg)
 {
-	struct ctl_listener_info	 li;
-	struct listener			*l;
-
-	TAILQ_FOREACH(l, &env->listeners, entry) {
-		memset(&li, 0, sizeof(li));
-		li.id = l->conf.id;
-		strlcpy(li.name, l->conf.name, sizeof(li.name));
-		sockaddr_ntop(&l->conf.ss, li.addr, sizeof(li.addr));
-		li.port = l->conf.port;
-		li.tls = l->conf.tls;
-		li.websocket = l->conf.websocket;
-		li.proxy = l->conf.proxy;
-		li.fd = l->fd;
-		li.nconns = l->nconns;
-		if (imsg_compose_event(socks_iev_control,
-		    IMSG_CTL_SHOW_LISTENERS_DATA, imsg->hdr.peerid, 0, -1,
-		    &li, sizeof(li)) == -1)
-			return;
-	}
-
-	imsg_compose_event(socks_iev_control, IMSG_CTL_SHOW_LISTENERS_END,
-	    imsg->hdr.peerid, 0, -1, NULL, 0);
-}
-
-/* keep a few descriptors free for the imsg pipes and libevent itself */
-static int
-sockets_accept_reserve(int sockfd, struct sockaddr *addr, socklen_t *addrlen)
-{
-	if (getdtablecount() + FD_RESERVE >= getdtablesize()) {
-		log_debug("%s: descriptor table nearly full", __func__);
-		errno = EMFILE;
-		return (-1);
-	}
-
-	return (accept4(sockfd, addr, addrlen, SOCK_NONBLOCK | SOCK_CLOEXEC));
-}
-
-static void
-sockets_accept_paused(int fd, short event, void *arg)
-{
-	struct listener	*l = arg;
-
-	event_add(&l->ev, NULL);
-}
-
-static void
-sockets_accept(int fd, short event, void *arg)
-{
-	struct listener		*l = arg;
+	struct conn_accept	 ca;
+	struct conn_done	 cd;
 	struct conn		*c;
-	struct sockaddr_storage	 ss;
-	struct timeval		 backoff = { 1, 0 };
-	socklen_t		 len = sizeof(ss);
-	int			 cfd;
+	struct sbridge		*sb = NULL;
+	struct listener		*l = NULL;
+	int			 fd;
 
-	cfd = sockets_accept_reserve(fd, (struct sockaddr *)&ss, &len);
-	if (cfd == -1) {
-		switch (errno) {
-		case EINTR:
-		case EWOULDBLOCK:
-		case ECONNABORTED:
-			return;
-		case EMFILE:
-		case ENFILE:
-			/* out of descriptors, give the clients a moment */
-			event_del(&l->ev);
-			evtimer_add(&l->pause, &backoff);
-			return;
-		default:
-			log_warn("%s: accept", __func__);
-			return;
+	IMSG_SIZE_CHECK(imsg, &ca);
+	memcpy(&ca, imsg->data, sizeof(ca));
+	ca.peer[sizeof(ca.peer) - 1] = '\0';
+	fd = imsg_get_fd(imsg);
+
+	if (fd == -1) {
+		log_warnx("%s: connection %u arrived without a descriptor",
+		    __func__, ca.id);
+		goto fail;
+	}
+	if (ca.bridge_id != 0) {
+		if ((sb = sbridge_find(ca.bridge_id)) == NULL) {
+			log_warnx("%s: unknown bridge %u", __func__,
+			    ca.bridge_id);
+			goto fail;
 		}
+	} else {
+		if ((l = config_listener_byid(env, ca.listener_id)) == NULL) {
+			/* removed by a reload while this one was in flight */
+			log_debug("%s: connection %u from %s: listener %u is "
+			    "gone", __func__, ca.id, ca.peer, ca.listener_id);
+			goto fail;
+		}
+		/*
+		 * A listener added on reload may reach us from listen before
+		 * our own IMSG_CFG_LISTENER_START did; set TLS up now.
+		 */
+		if (l->conf.tls && l->tls_ctx == NULL &&
+		    sockets_listener_launch(l) == -1)
+			goto fail;
 	}
+	if (conn_find(ca.id) != NULL) {
+		log_warnx("%s: duplicate connection %u", __func__, ca.id);
+		goto fail;
+	}
 
 	if ((c = calloc(1, sizeof(*c))) == NULL) {
 		log_warn("%s: calloc", __func__);
-		close(cfd);
-		return;
+		goto fail;
 	}
-
-	do {
-		c->id = ++conn_next_id;
-	} while (c->id == 0 || conn_find(c->id) != NULL);
-	c->fd = cfd;
+	c->id = ca.id;
+	c->fd = fd;
 	c->l = l;
-	if (ss.ss_family == AF_UNIX)
-		strlcpy(c->peer, "unix", sizeof(c->peer));
-	else
-		sockaddr_ntop(&ss, c->peer, sizeof(c->peer));
+	c->listener_id = ca.listener_id;
+	c->bridge = sb;
+	strlcpy(c->peer, ca.peer, sizeof(c->peer));
 	evtimer_set(&c->timer, conn_timeout, c);
 	RB_INSERT(conntree, &conns, c);
-	l->nconns++;
 
-	log_debug("%s: connection %u from %s on listener %s", __func__, c->id,
-	    c->peer, l->conf.name);
+	log_debug("%s: connection %u %s %s", __func__, c->id,
+	    sb != NULL ? "to" : "from", c->peer);
 
-	/* the client has this long to finish PROXY, TLS, WS and CONNECT */
+	/* the peer has this long to finish TLS, WS and CONNECT */
 	conn_set_timer(c, MQTTD_CONNECT_TIMEOUT);
 
-	if (l->conf.proxy) {
-		event_set(&c->ev, cfd, EV_READ, conn_proxy_read, c);
-		event_add(&c->ev, NULL);
+	if (sb == NULL) {
+		conn_begin(c);
 		return;
 	}
 
-	conn_begin(c);
+	/* an outbound bridge: we are the TLS and MQTT client */
+	if (!sb->conf.tls) {
+		conn_start(c);
+		return;
+	}
+	if (sb->tls_config == NULL) {
+		if ((sb->tls_config = tls_config_new()) == NULL)
+			fatal("%s: tls_config_new", __func__);
+		if (sb->conf.tls_ca != NULL &&
+		    tls_config_set_ca_mem(sb->tls_config, sb->conf.tls_ca,
+		    sb->conf.tls_ca_len) == -1)
+			log_warnx("bridge %s: %s", sb->conf.name,
+			    tls_config_error(sb->tls_config));
+	}
+	if ((c->tls_ctx = tls_client()) == NULL ||
+	    tls_configure(c->tls_ctx, sb->tls_config) == -1 ||
+	    tls_connect_socket(c->tls_ctx, fd, sb->conf.host) == -1) {
+		log_warnx("bridge %s: tls setup: %s", sb->conf.name,
+		    c->tls_ctx != NULL ? tls_error(c->tls_ctx) : "no context");
+		conn_free(c, -1);
+		return;
+	}
+	event_set(&c->ev, fd, EV_READ | EV_WRITE, conn_tls_handshake, c);
+	event_add(&c->ev, NULL);
+	return;
+
+fail:
+	if (fd != -1)
+		close(fd);
+	if (ca.bridge_id != 0)
+		imsg_compose_event(socks_iev_broker, IMSG_BRIDGE_FAILED, 0, 0,
+		    -1, &ca.bridge_id, sizeof(ca.bridge_id));
+	cd.id = ca.id;
+	cd.listener_id = ca.listener_id;
+	imsg_compose_event(socks_iev_listen, IMSG_CONN_DONE, 0, 0, -1, &cd,
+	    sizeof(cd));
 }
 
-/* PROXY header consumed if any; now TLS, then the MQTT or WS stream */
+/* TLS first when the listener wants it, then the MQTT or WS stream */
 static void
 conn_begin(struct conn *c)
 {
@@ -825,149 +728,6 @@ conn_begin(struct conn *c)
 	conn_start(c);
 }
 
-/*
- * The proxy sends its header in one write before any client byte, so
- * peek, work out the exact length, then consume exactly that much and
- * leave the stream untouched for TLS or MQTT.
- */
-static void
-conn_proxy_read(int fd, short event, void *arg)
-{
-	struct conn	*c = arg;
-	uint8_t		 buf[PROXY_PEEK];
-	ssize_t		 n;
-	int		 used;
-
-	n = recv(fd, buf, sizeof(buf), MSG_PEEK);
-	if (n == -1) {
-		if (errno == EAGAIN || errno == EINTR) {
-			event_add(&c->ev, NULL);
-			return;
-		}
-		conn_free(c, -1);
-		return;
-	}
-	if (n == 0) {
-		conn_free(c, -1);
-		return;
-	}
-
-	if (n >= PROXY_V2_MIN &&
-	    memcmp(buf, "\x0d\x0a\x0d\x0a\x00\x0d\x0a\x51\x55\x49\x54\x0a",
-	    12) == 0)
-		used = proxy_parse_v2(c, buf, n);
-	else if (n >= 6 && memcmp(buf, "PROXY ", 6) == 0)
-		used = proxy_parse_v1(c, buf, n);
-	else {
-		log_warnx("%s: no PROXY header from %s", __func__, c->peer);
-		conn_free(c, -1);
-		return;
-	}
-
-	if (used == 0) {
-		/* incomplete header, wait for the rest */
-		if (n >= (ssize_t)sizeof(buf)) {
-			conn_free(c, -1);
-			return;
-		}
-		event_add(&c->ev, NULL);
-		return;
-	}
-	if (used == -1) {
-		log_warnx("%s: bad PROXY header from %s", __func__, c->peer);
-		conn_free(c, -1);
-		return;
-	}
-
-	/* the peeked bytes are still there; this read cannot block */
-	if (recv(fd, buf, used, MSG_WAITALL) != used) {
-		conn_free(c, -1);
-		return;
-	}
-	conn_begin(c);
-}
-
-/* returns bytes consumed, 0 for more needed, -1 for malformed */
-static int
-proxy_parse_v1(struct conn *c, const uint8_t *buf, size_t n)
-{
-	char		 line[PROXY_V1_MAX + 1], *p, *end, *fam, *src, *sport;
-	const char	*eol;
-	size_t		 len;
-
-	eol = memmem(buf, n < PROXY_V1_MAX ? n : PROXY_V1_MAX, "\r\n", 2);
-	if (eol == NULL)
-		return (n >= PROXY_V1_MAX ? -1 : 0);
-	len = eol - (const char *)buf;
-	memcpy(line, buf, len);
-	line[len] = '\0';
-
-	/* PROXY TCP4 src dst sport dport, or PROXY UNKNOWN ... */
-	p = line + 6;
-	end = p;
-	fam = strsep(&end, " ");
-	if (fam == NULL)
-		return (-1);
-	if (strcmp(fam, "UNKNOWN") == 0)
-		return (len + 2);
-	if (strcmp(fam, "TCP4") != 0 && strcmp(fam, "TCP6") != 0)
-		return (-1);
-	src = strsep(&end, " ");
-	(void)strsep(&end, " ");
-	sport = strsep(&end, " ");
-	if (src == NULL || sport == NULL || end == NULL)
-		return (-1);
-	if (strchr(src, ':') != NULL)
-		snprintf(c->peer, sizeof(c->peer), "[%s]:%s", src, sport);
-	else
-		snprintf(c->peer, sizeof(c->peer), "%s:%s", src, sport);
-	return (len + 2);
-}
-
-static int
-proxy_parse_v2(struct conn *c, const uint8_t *buf, size_t n)
-{
-	char		 host[INET6_ADDRSTRLEN];
-	size_t		 len, total;
-	uint8_t		 cmd, fam;
-	uint16_t	 port;
-
-	if ((buf[12] >> 4) != 2)
-		return (-1);
-	cmd = buf[12] & 0x0f;
-	fam = buf[13];
-	len = (size_t)buf[14] << 8 | buf[15];
-	total = PROXY_V2_MIN + len;
-	if (total > PROXY_PEEK)
-		return (-1);
-	if (n < total)
-		return (0);
-
-	/* LOCAL (health checks) keeps the proxy's own address */
-	if (cmd != 0x01)
-		return (total);
-
-	switch (fam) {
-	case 0x11:
-		if (len < 12)
-			return (-1);
-		inet_ntop(AF_INET, buf + 16, host, sizeof(host));
-		port = (uint16_t)buf[24] << 8 | buf[25];
-		snprintf(c->peer, sizeof(c->peer), "%s:%u", host, port);
-		break;
-	case 0x21:
-		if (len < 36)
-			return (-1);
-		inet_ntop(AF_INET6, buf + 16, host, sizeof(host));
-		port = (uint16_t)buf[48] << 8 | buf[49];
-		snprintf(c->peer, sizeof(c->peer), "[%s]:%u", host, port);
-		break;
-	default:
-		break;
-	}
-	return (total);
-}
-
 static struct conn *
 conn_find(uint32_t id)
 {
@@ -1052,7 +812,7 @@ conn_open_send(struct conn *c)
 
 	memset(&co, 0, sizeof(co));
 	co.id = c->id;
-	co.listener_id = c->l != NULL ? c->l->conf.id : 0;
+	co.listener_id = c->listener_id;
 	co.bridge_id = c->bridge != NULL ? c->bridge->conf.id : 0;
 	co.tls = c->tls_ctx != NULL;
 	strlcpy(co.peer, c->peer, sizeof(co.peer));
@@ -1072,6 +832,7 @@ static void
 conn_free(struct conn *c, int reason)
 {
 	struct conn_close	 cc;
+	struct conn_done	 cd;
 
 	if (reason >= 0 && c->opened) {
 		cc.id = c->id;
@@ -1083,6 +844,12 @@ conn_free(struct conn *c, int reason)
 	if (c->bridge != NULL && !c->opened)
 		sockets_bridge_failed(c);
 
+	/* listen keeps the per-worker and per-listener counts */
+	cd.id = c->id;
+	cd.listener_id = c->listener_id;
+	imsg_compose_event(socks_iev_listen, IMSG_CONN_DONE, 0, 0, -1, &cd,
+	    sizeof(cd));
+
 	log_debug("%s: closing connection %u %s %s", __func__, c->id,
 	    c->bridge != NULL ? "to" : "from", c->peer);
 
@@ -1102,8 +869,6 @@ conn_free(struct conn *c, int reason)
 	RB_REMOVE(conntree, &conns, c);
 	if (c->throttled)
 		TAILQ_REMOVE(&throttled_conns, c, tentry);
-	if (c->l != NULL)
-		c->l->nconns--;
 	free(c);
 }
 
@@ -1159,113 +924,6 @@ sockets_bridge_failed(struct conn *c)
 	    &id, sizeof(id));
 }
 
-/* the broker wants this bridge up: start a non-blocking connect */
-static void
-sockets_bridge_connect(struct imsg *imsg)
-{
-	struct sbridge		*sb;
-	struct conn		*c;
-	struct sockaddr_storage	 ss;
-	uint32_t		 id;
-	int			 fd;
-
-	IMSG_SIZE_CHECK(imsg, &id);
-	memcpy(&id, imsg->data, sizeof(id));
-	if ((sb = sbridge_find(id)) == NULL) {
-		log_warnx("%s: unknown bridge %u", __func__, id);
-		return;
-	}
-
-	if (sb->conf.tls && sb->tls_config == NULL) {
-		if ((sb->tls_config = tls_config_new()) == NULL)
-			fatal("%s: tls_config_new", __func__);
-		if (sb->conf.tls_ca != NULL &&
-		    tls_config_set_ca_mem(sb->tls_config, sb->conf.tls_ca,
-		    sb->conf.tls_ca_len) == -1)
-			log_warnx("bridge %s: %s", sb->conf.name,
-			    tls_config_error(sb->tls_config));
-	}
-
-	memcpy(&ss, &sb->conf.ss, sizeof(ss));
-	switch (ss.ss_family) {
-	case AF_INET:
-		((struct sockaddr_in *)&ss)->sin_port = sb->conf.port;
-		break;
-	case AF_INET6:
-		((struct sockaddr_in6 *)&ss)->sin6_port = sb->conf.port;
-		break;
-	}
-
-	fd = socket(ss.ss_family, SOCK_STREAM | SOCK_NONBLOCK, 0);
-	if (fd == -1) {
-		log_warn("%s: socket", __func__);
-		imsg_compose_event(socks_iev_broker, IMSG_BRIDGE_FAILED, 0, 0,
-		    -1, &id, sizeof(id));
-		return;
-	}
-
-	if ((c = calloc(1, sizeof(*c))) == NULL) {
-		log_warn("%s: calloc", __func__);
-		close(fd);
-		imsg_compose_event(socks_iev_broker, IMSG_BRIDGE_FAILED, 0, 0,
-		    -1, &id, sizeof(id));
-		return;
-	}
-	do {
-		c->id = ++conn_next_id;
-	} while (c->id == 0 || conn_find(c->id) != NULL);
-	c->fd = fd;
-	c->bridge = sb;
-	sockaddr_ntop(&ss, c->peer, sizeof(c->peer));
-	evtimer_set(&c->timer, conn_timeout, c);
-	RB_INSERT(conntree, &conns, c);
-	conn_set_timer(c, MQTTD_CONNECT_TIMEOUT);
-
-	if (connect(fd, (struct sockaddr *)&ss, ss.ss_len) == -1 &&
-	    errno != EINPROGRESS) {
-		log_warn("bridge %s: connect to %s", sb->conf.name, c->peer);
-		conn_free(c, -1);
-		return;
-	}
-
-	event_set(&c->ev, fd, EV_WRITE, sockets_bridge_connected, c);
-	event_add(&c->ev, NULL);
-}
-
-static void
-sockets_bridge_connected(int fd, short event, void *arg)
-{
-	struct conn	*c = arg;
-	struct sbridge	*sb = c->bridge;
-	socklen_t	 len = sizeof(int);
-	int		 err = 0;
-
-	if (getsockopt(fd, SOL_SOCKET, SO_ERROR, &err, &len) == -1)
-		err = errno;
-	if (err != 0) {
-		log_warnx("bridge %s: connect to %s: %s", sb->conf.name,
-		    c->peer, strerror(err));
-		conn_free(c, -1);
-		return;
-	}
-
-	if (!sb->conf.tls) {
-		conn_start(c);
-		return;
-	}
-
-	if ((c->tls_ctx = tls_client()) == NULL ||
-	    tls_configure(c->tls_ctx, sb->tls_config) == -1 ||
-	    tls_connect_socket(c->tls_ctx, fd, sb->conf.host) == -1) {
-		log_warnx("bridge %s: tls setup: %s", sb->conf.name,
-		    c->tls_ctx != NULL ? tls_error(c->tls_ctx) : "no context");
-		conn_free(c, -1);
-		return;
-	}
-	event_set(&c->ev, fd, EV_READ | EV_WRITE, conn_tls_handshake, c);
-	event_add(&c->ev, NULL);
-}
-
 /* the CN of a verified client certificate, empty when there is none */
 static void
 conn_cert_user(struct tls *ctx, char *buf, size_t len)