commit - 8507d3d5b7832bfe03b50521368f4829ceb782586ac8e40398196e724caec3de
commit + a8f2d00fef2add0808aaaa76f2e20c7662f8d08d3b7aa4e4162b54d6ca6764cc
blob - e8bd3e9700c6d0061d25b3a4b0ddda72267fe3dc9952548d8d8fba7b1b382f2f
blob + 98e930e577b97f059609cedffc4e0acc5b02a469ee71eb2206d691c04e180d7b
--- README.md
+++ README.md
authentication.
The daemon is split into a privileged parent, a control process, a single
-broker process holding all protocol state, and a sockets process that
-owns the network connections and TLS.
+broker process holding all protocol state, a listen process that accepts
+connections and a configurable number of sockets workers that run TLS and
+framing for the connections handed to them.
The layout and plumbing come from the thingsd proxy daemon.
Prerequisites
blob - 024ceec5cb3ff0c4be2245ce6c3448bcc147ea37bee3d844eff7d1fb398ed58d
blob + e5ce5c43a3ed8dd1f584be733b3b244283830366f7bf798756ba057b481e4378
--- mqttctl/mqttctl.c
+++ mqttctl/mqttctl.c
printf(" started: %s\n", fmt_time(pi.started));
printf(" uptime: %s\n",
fmt_duration(time(NULL) - pi.started, dbuf, sizeof(dbuf)));
+ printf(" workers: %u\n", pi.workers);
printf(" listeners: %u\n", pi.nlisteners);
break;
case IMSG_CTL_SHOW_PARENT_END:
blob - 8af8c13e9d413c115c68638a094f11e0317a8c87781fd260589f8e21903d94f8
blob + 8b7dd5c6faf2308982f443ef48745c8eeaffdfd69f07571b9f34394686c9c64b
--- mqttd/Makefile
+++ mqttd/Makefile
PROG = mqttd
SRCS = parse.y
SRCS += mqttd.c config.c control.c log.c
-SRCS += sockets.c tls.c
+SRCS += listen.c sockets.c tls.c
SRCS += broker.c session.c topic.c mqtt.c db.c storage.c auth.c
SRCS += bridge.c sparkplug.c snowflake.c jwt.c
blob - 37147be5cc9d0b8834b998b4d930f048680985cf4bcf06c133b5de8a14df983d
blob + d26f4e17a0c3e03bd3ad94c5495a809eb91684be63756b7349ca8a8bad80b27e
--- mqttd/bridge.c
+++ mqttd/bridge.c
*/
/*
- * Bridges: the broker as an MQTT client of another broker. A sockets
- * process opens the TCP/TLS connection on request; from then on the
- * connection looks like any other, except that we send CONNECT and
+ * Bridges: the broker as an MQTT client of another broker. The listen
+ * process dials the remote on request and a sockets worker runs TLS over
+ * it; from then on the connection looks like any other, except that we
+ * send CONNECT and
* SUBSCRIBE and expect CONNACK, SUBACK and PINGRESP. Local messages for
* the remote flow through the bridge's own session like any delivery;
* remote messages are routed locally with the bridge session as origin
br->attempts++;
log_debug("bridge %s: connecting to %s (attempt %u)", br->conf.name,
br->conf.host, br->attempts);
- broker_socks_send(br->conf.proc, IMSG_BRIDGE_CONNECT, &br->conf.id,
- sizeof(br->conf.id));
+ /* a sink's remote is the snowflake process, everything else dials */
+ if (br->conf.proc == MQTTD_PROC_SNOW)
+ broker_socks_send(br->conf.proc, IMSG_BRIDGE_CONNECT,
+ &br->conf.id, sizeof(br->conf.id));
+ else
+ broker_listen_send(IMSG_BRIDGE_CONNECT, &br->conf.id,
+ sizeof(br->conf.id));
}
static void
bridge_schedule(br);
}
-/* the sockets process has a connection up; we are the client now */
+/* a sockets worker has the connection up; we are the client now */
void
bridge_conn_open(struct bconn *bc, uint32_t id)
{
blob - c47a03beeff7e1b2c59dddc04d810c80e8011009a1b995987ed801adb8a9a708
blob + 9191faedad566553cb5e16d345bc4033d5bc27b820ad8cf18dbad548b23bfd06
--- mqttd/broker.c
+++ mqttd/broker.c
*/
/*
- * The broker process is the protocol authority. The sockets processes
+ * The broker process is the protocol authority. The sockets workers
* hand it complete control packets and it answers with the packets to
* write back; it never touches a network socket itself.
*/
static struct imsgev *broker_iev_parent;
static struct imsgev *broker_iev_control;
+static struct imsgev *broker_iev_listen;
static struct imsgev *broker_iev_socks;
/* CFG_DONE is acknowledged only after storage has replayed its snapshot */
static struct imsgev *broker_iev_snow;
static uint32_t auth_next_id;
-/* the sockets process, or the snowflake process */
+/* sockets workers by index, or the snowflake process */
static struct imsgev *
broker_proc_iev(uint32_t proc)
{
if (proc == MQTTD_PROC_SNOW)
return (broker_iev_snow);
- if (proc != 0)
+ if (proc >= env->conf.workers)
return (NULL);
- return (broker_iev_socks);
+ return (&broker_iev_socks[proc]);
}
struct route_ctx {
static struct imsgev
*broker_proc_iev(uint32_t);
static void broker_dispatch_sockets(int, short, void *);
+static void broker_dispatch_listen(int, short, void *);
static void broker_sighdlr(int, short, void *);
static void broker_sweep(int, short, void *);
static __dead void
__dead void
broker(int fd_parent, int fd_control, int fd_storage, int fd_auth,
- int fd_snow, int fd_socks)
+ int fd_snow, int fd_listen, int *fd_socks)
{
struct timeval tv;
+ int n;
log_procinit("broker");
proc_privdrop("broker");
fatal("%s: calloc", __func__);
imsg_event_init(broker_iev_snow, fd_snow, broker_dispatch_sockets);
- if ((broker_iev_socks = calloc(1, sizeof(*broker_iev_socks))) == NULL)
+ if ((broker_iev_listen = calloc(1, sizeof(*broker_iev_listen))) == NULL)
fatal("%s: calloc", __func__);
- imsg_event_init(broker_iev_socks, fd_socks, broker_dispatch_sockets);
+ imsg_event_init(broker_iev_listen, fd_listen, broker_dispatch_listen);
+ if ((broker_iev_socks = calloc(env->conf.workers,
+ sizeof(*broker_iev_socks))) == NULL)
+ fatal("%s: calloc", __func__);
+ for (n = 0; n < env->conf.workers; n++)
+ imsg_event_init(&broker_iev_socks[n], fd_socks[n],
+ broker_dispatch_sockets);
+
evtimer_set(&ev_sweep, broker_sweep, NULL);
tv.tv_sec = MQTTD_RETAIN_SWEEP;
tv.tv_usec = 0;
static __dead void
broker_shutdown(void)
{
+ int n;
+
/* the last snapshot must reach storage before the pipe closes */
db_snapshot_sync();
imsg_event_shutdown(broker_iev_snow);
free(broker_iev_snow);
}
- imsg_event_shutdown(broker_iev_socks);
+ imsg_event_shutdown(broker_iev_listen);
+ free(broker_iev_listen);
+ for (n = 0; n < env->conf.workers; n++)
+ imsg_event_shutdown(&broker_iev_socks[n]);
free(broker_iev_socks);
log_info("broker exiting, pid %d", getpid());
uint32_t proc;
int n;
- proc = iev == broker_iev_snow ? MQTTD_PROC_SNOW : 0;
+ proc = iev == broker_iev_snow ? MQTTD_PROC_SNOW :
+ (uint32_t)(iev - broker_iev_socks);
if (event & EV_READ) {
if ((n = imsgbuf_read(ibuf)) == -1)
broker_socks_event_add(iev);
}
+/* the listen process only ever reports a bridge it could not dial */
+static void
+broker_dispatch_listen(int fd, short event, void *arg)
+{
+ struct imsgev *iev = arg;
+ struct imsgbuf *ibuf = &iev->ibuf;
+ struct imsg imsg;
+ int n;
+
+ if (event & EV_READ) {
+ if ((n = imsgbuf_read(ibuf)) == -1)
+ fatal("%s: imsgbuf_read", __func__);
+ if (n == 0) {
+ log_warnx("%s: lost listen process", __func__);
+ broker_shutdown();
+ }
+ }
+ if (event & EV_WRITE) {
+ if (imsgbuf_write(ibuf) == -1) {
+ if (errno == EPIPE) {
+ log_warnx("%s: lost listen process", __func__);
+ broker_shutdown();
+ }
+ fatal("%s: imsgbuf_write", __func__);
+ }
+ }
+
+ for (;;) {
+ if ((n = imsgbuf_get(ibuf, &imsg)) == -1)
+ fatal("%s: imsgbuf_get", __func__);
+ if (n == 0)
+ break;
+
+ switch (imsg.hdr.type) {
+ case IMSG_BRIDGE_FAILED:
+ bridge_failed(&imsg);
+ break;
+ default:
+ log_warnx("%s: unexpected imsg %d from listen",
+ __func__, imsg.hdr.type);
+ break;
+ }
+ imsg_free(&imsg);
+ }
+ imsg_event_add(iev);
+}
+
+void
+broker_listen_send(uint32_t type, const void *data, size_t len)
+{
+ if (imsg_compose(&broker_iev_listen->ibuf, type, 0, 0, -1, data,
+ len) == -1) {
+ log_warn("%s: imsg_compose", __func__);
+ return;
+ }
+ imsg_event_add(broker_iev_listen);
+}
+
/* like imsg_event_add, but reading stays off while the broker is paused */
static void
broker_socks_event_add(struct imsgev *iev)
}
/*
- * Backpressure. When the sockets pipe or the snowflake pipe holds more
- * than MQTTD_QUEUE_HIGH unsent messages the broker stops reading from
- * both, so publishers stall instead of the broker growing without bound.
- * Reading resumes once both pipes are below MQTTD_QUEUE_LOW.
+ * Backpressure. When any worker pipe holds more than MQTTD_QUEUE_HIGH
+ * unsent messages the broker stops reading from every worker, so
+ * publishers stall instead of the broker growing without bound.
+ * Reading resumes once every pipe is below MQTTD_QUEUE_LOW.
*/
static void
broker_flow_update(void)
{
size_t q;
- int over = 0, above_low = 0;
+ int n, over = 0, above_low = 0;
- q = imsgbuf_queuelen(&broker_iev_socks->ibuf);
- if (q > MQTTD_QUEUE_HIGH)
- over = 1;
- if (q > MQTTD_QUEUE_LOW)
- above_low = 1;
+ for (n = 0; n < env->conf.workers; n++) {
+ q = imsgbuf_queuelen(&broker_iev_socks[n].ibuf);
+ if (q > MQTTD_QUEUE_HIGH)
+ over = 1;
+ if (q > MQTTD_QUEUE_LOW)
+ above_low = 1;
+ }
if (broker_iev_snow != NULL) {
q = imsgbuf_queuelen(&broker_iev_snow->ibuf);
if (q > MQTTD_QUEUE_HIGH)
} else
return;
- broker_socks_event_add(broker_iev_socks);
+ for (n = 0; n < env->conf.workers; n++)
+ broker_socks_event_add(&broker_iev_socks[n]);
if (broker_iev_snow != NULL)
broker_socks_event_add(broker_iev_snow);
}
blob - ac35b57826a94aa77801850866dacd5d7c34d140d28c625a15c07fdecff6d0b9
blob + ed91869bf43db9b52f5b83d8b114cd7bcafdd6bab73822168695d1002e204244
--- mqttd/broker.h
+++ mqttd/broker.h
void broker_send_ack(struct bconn *, uint8_t, uint16_t, uint8_t);
void broker_close(struct bconn *, int, enum detach_how);
void broker_socks_send(uint32_t, uint32_t, const void *, size_t);
+void broker_listen_send(uint32_t, const void *, size_t);
int broker_filter_match(const char *, const char *, size_t);
int broker_route(struct message *);
#define RC_NONE -1
blob - acbfedcc388b96d19f6feb8f9e0a07e39414e6086c6beb70cc7405a8ff1e5f83
blob + bf462de1116db0c01bef7e27216bd6a26d59e5bdcf73f04ae90edd0f3ba4b075
--- mqttd/config.c
+++ mqttd/config.c
}
/*
- * imsg closes the passed descriptor once sent, so every receiver gets
- * its own dup. The flush is synchronous because the parent pledges
- * without sendfd once configuration is finished.
+ * imsg closes a passed descriptor once it is sent; the listen process
+ * gets the listening socket, the workers get -1 with the configuration
+ * alone. The flush is synchronous so the listener is in place before
+ * anything that depends on it is sent.
*/
int
config_send_listener(struct imsgev *iev, struct listener *l, int fd)
blob - 512395aa375e2bde98bf428ceb550fedaaf686db68bea2c7a85b0dfe00f77c5d
blob + 422026b9f8bbeb9db68ac963a3b72c48b16efb1887fb76181ecf7e30d64fd95a
--- mqttd/control.c
+++ mqttd/control.c
static struct control_sock control_cs;
static struct imsgev *control_iev_parent;
static struct imsgev *control_iev_broker;
-static struct imsgev *control_iev_socks;
+static struct imsgev *control_iev_listen;
static struct event ev_sigint;
static struct event ev_sigterm;
static void control_dispatch_imsg(int, short, void *);
static void control_dispatch_parent(int, short, void *);
static void control_dispatch_broker(int, short, void *);
-static void control_dispatch_sockets(int, short, void *);
+static void control_dispatch_listen(int, short, void *);
static void control_relay(struct imsg *);
static void control_sighdlr(int, short, void *);
static __dead void
control_shutdown(void);
__dead void
-control(int fd_parent, int fd_broker, int fd_socks)
+control(int fd_parent, int fd_broker, int fd_listen)
{
log_procinit("control");
imsg_event_init(control_iev_broker, fd_broker,
control_dispatch_broker);
- if ((control_iev_socks = calloc(1, sizeof(*control_iev_socks)))
+ if ((control_iev_listen = calloc(1, sizeof(*control_iev_listen)))
== NULL)
fatal("%s: calloc", __func__);
- imsg_event_init(control_iev_socks, fd_socks,
- control_dispatch_sockets);
+ imsg_event_init(control_iev_listen, fd_listen,
+ control_dispatch_listen);
TAILQ_INIT(&ctl_conns);
free(control_iev_parent);
imsg_event_shutdown(control_iev_broker);
free(control_iev_broker);
- imsg_event_shutdown(control_iev_socks);
- free(control_iev_socks);
+ imsg_event_shutdown(control_iev_listen);
+ free(control_iev_listen);
log_info("control exiting, pid %d", getpid());
exit(0);
}
static void
-control_dispatch_sockets(int fd, short event, void *arg)
+control_dispatch_listen(int fd, short event, void *arg)
{
struct imsgev *iev = arg;
struct imsgbuf *ibuf = &iev->ibuf;
if ((n = imsgbuf_read(ibuf)) == -1)
fatal("%s: imsgbuf_read", __func__);
if (n == 0) {
- log_warnx("%s: lost sockets process", __func__);
+ log_warnx("%s: lost listen process", __func__);
control_shutdown();
}
}
if (event & EV_WRITE) {
if (imsgbuf_write(ibuf) == -1) {
if (errno == EPIPE) {
- log_warnx("%s: lost sockets process", __func__);
+ log_warnx("%s: lost listen process", __func__);
control_shutdown();
}
fatal("%s: imsgbuf_write", __func__);
control_relay(&imsg);
break;
default:
- log_warnx("%s: unexpected imsg %d from sockets",
+ log_warnx("%s: unexpected imsg %d from listen",
__func__, imsg.hdr.type);
break;
}
blob - /dev/null
blob + 1f4556a3268c9962b4448d1ed5ea5da85fc235c986e7ce493bdcf46f33a31421 (mode 644)
--- /dev/null
+++ mqttd/listen.c
+/*
+ * Copyright (c) 2026 Tracey Emery <tracey@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+/*
+ * The listen process owns the listening sockets. It accepts, consumes
+ * the PROXY protocol header where a listener expects one, dials outbound
+ * bridges, and hands every connected descriptor to the sockets worker
+ * with the fewest connections. It never sees a TLS record or an MQTT
+ * byte, and it is the one place that knows how many connections each
+ * listener and each worker carries.
+ */
+
+#include <sys/types.h>
+#include <sys/queue.h>
+#include <sys/tree.h>
+#include <sys/time.h>
+#include <sys/socket.h>
+#include <sys/resource.h>
+#include <sys/stat.h>
+#include <sys/uio.h>
+#include <sys/un.h>
+
+#include <netinet/in.h>
+#include <arpa/inet.h>
+
+#include <errno.h>
+#include <event.h>
+#include <imsg.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "mqttd.h"
+
+#define FD_RESERVE 8
+#define PROXY_V1_MAX 107
+#define PROXY_V2_MIN 16
+#define PROXY_PEEK (PROXY_V2_MIN + 216)
+
+/* outbound bridge target: the address only, the workers hold the TLS side */
+struct lbridge {
+ TAILQ_ENTRY(lbridge) entry;
+ struct bridge_config conf;
+};
+TAILQ_HEAD(lbridgelist, lbridge);
+
+/* a connection between accept() or connect() and its hand-off */
+struct lconn {
+ RB_ENTRY(lconn) entry;
+ uint32_t id;
+ int fd;
+ struct listener *l;
+ struct lbridge *bridge;
+ struct event ev;
+ struct event timer;
+ char peer[MQTTD_MAXPEER];
+};
+RB_HEAD(lconntree, lconn);
+
+static int lconn_cmp(struct lconn *, struct lconn *);
+RB_PROTOTYPE_STATIC(lconntree, lconn, entry, lconn_cmp);
+RB_GENERATE_STATIC(lconntree, lconn, entry, lconn_cmp);
+
+static void listen_dispatch_parent(int, short, void *);
+static void listen_dispatch_control(int, short, void *);
+static void listen_dispatch_broker(int, short, void *);
+static void listen_dispatch_socks(int, short, void *);
+static void listen_sighdlr(int, short, void *);
+static __dead void
+ listen_shutdown(void);
+static void listen_rlimit(void);
+static void listen_launch(void);
+static void listen_listener_launch(struct listener *);
+static void listen_listener_remove(struct listener *);
+static int listen_accept_reserve(int, struct sockaddr *, socklen_t *);
+static void listen_accept(int, short, void *);
+static void listen_accept_paused(int, short, void *);
+static void listen_show_listeners(struct imsg *);
+static void listen_conn_done(uint32_t, struct imsg *);
+static uint32_t listen_pick_worker(void);
+
+static struct lconn
+ *lconn_new(void);
+static struct lconn
+ *lconn_find(uint32_t);
+static void lconn_free(struct lconn *);
+static void lconn_release(struct lconn *);
+static void lconn_handoff(struct lconn *);
+static void lconn_set_timer(struct lconn *, time_t);
+static void lconn_timeout(int, short, void *);
+static void lconn_proxy_read(int, short, void *);
+static int proxy_parse_v1(struct lconn *, const uint8_t *, size_t);
+static int proxy_parse_v2(struct lconn *, const uint8_t *, size_t);
+
+static struct lbridge
+ *lbridge_find(uint32_t);
+static void listen_bridge_config(struct imsg *);
+static void listen_bridge_connect(struct imsg *);
+static void listen_bridge_connected(int, short, void *);
+static void listen_bridge_failed(uint32_t);
+
+static struct imsgev *listen_iev_parent;
+static struct imsgev *listen_iev_control;
+static struct imsgev *listen_iev_broker;
+static struct imsgev *listen_iev_socks;
+/* live connections per worker, the hand-off decision */
+static uint32_t *worker_load;
+
+static struct lconntree lconns = RB_INITIALIZER(&lconns);
+static struct lbridgelist lbridges = TAILQ_HEAD_INITIALIZER(lbridges);
+static uint32_t conn_next_id;
+
+static struct event ev_sigint;
+static struct event ev_sigterm;
+static struct event ev_sighup;
+static struct event ev_sigpipe;
+static struct event ev_sigusr1;
+
+static int
+lconn_cmp(struct lconn *a, struct lconn *b)
+{
+ if (a->id != b->id)
+ return (a->id < b->id ? -1 : 1);
+ return (0);
+}
+
+__dead void
+listen_main(int fd_parent, int fd_control, int fd_broker, int *fd_socks)
+{
+ int n;
+
+ log_procinit("listen");
+ proc_privdrop("listen");
+
+ /* start from empty lists, the listeners arrive from the parent */
+ config_init(env);
+
+ listen_rlimit();
+
+ event_init();
+
+ signal_set(&ev_sigint, SIGINT, listen_sighdlr, NULL);
+ signal_set(&ev_sigterm, SIGTERM, listen_sighdlr, NULL);
+ signal_set(&ev_sighup, SIGHUP, listen_sighdlr, NULL);
+ signal_set(&ev_sigpipe, SIGPIPE, listen_sighdlr, NULL);
+ signal_set(&ev_sigusr1, SIGUSR1, listen_sighdlr, NULL);
+ signal_add(&ev_sigint, NULL);
+ signal_add(&ev_sigterm, NULL);
+ signal_add(&ev_sighup, NULL);
+ signal_add(&ev_sigpipe, NULL);
+ signal_add(&ev_sigusr1, NULL);
+
+ if ((listen_iev_parent = calloc(1, sizeof(*listen_iev_parent))) == NULL)
+ fatal("%s: calloc", __func__);
+ imsg_event_init(listen_iev_parent, fd_parent, listen_dispatch_parent);
+
+ if ((listen_iev_control = calloc(1, sizeof(*listen_iev_control)))
+ == NULL)
+ fatal("%s: calloc", __func__);
+ imsg_event_init(listen_iev_control, fd_control,
+ listen_dispatch_control);
+
+ if ((listen_iev_broker = calloc(1, sizeof(*listen_iev_broker))) == NULL)
+ fatal("%s: calloc", __func__);
+ imsg_event_init(listen_iev_broker, fd_broker, listen_dispatch_broker);
+
+ if ((listen_iev_socks = calloc(env->conf.workers,
+ sizeof(*listen_iev_socks))) == NULL ||
+ (worker_load = calloc(env->conf.workers,
+ sizeof(*worker_load))) == NULL)
+ fatal("%s: calloc", __func__);
+ for (n = 0; n < env->conf.workers; n++)
+ imsg_event_init(&listen_iev_socks[n], fd_socks[n],
+ listen_dispatch_socks);
+
+ /*
+ * inet and unix for accept(2) and connect(2), sendfd to hand the
+ * connections to the workers, recvfd for listeners added on reload.
+ */
+ if (pledge("stdio inet unix sendfd recvfd", NULL) == -1)
+ fatal("pledge");
+
+ event_dispatch();
+
+ listen_shutdown();
+}
+
+static void
+listen_sighdlr(int sig, short event, void *arg)
+{
+ switch (sig) {
+ case SIGINT:
+ case SIGTERM:
+ listen_shutdown();
+ break;
+ case SIGHUP:
+ case SIGPIPE:
+ case SIGUSR1:
+ break;
+ default:
+ fatalx("%s: unexpected signal", __func__);
+ }
+}
+
+static __dead void
+listen_shutdown(void)
+{
+ struct listener *l, *tl;
+ struct lconn *c, *tc;
+ struct lbridge *lb;
+ int n;
+
+ RB_FOREACH_SAFE(c, lconntree, &lconns, tc)
+ lconn_free(c);
+
+ while ((lb = TAILQ_FIRST(&lbridges)) != NULL) {
+ TAILQ_REMOVE(&lbridges, lb, entry);
+ free(lb);
+ }
+
+ TAILQ_FOREACH_SAFE(l, &env->listeners, entry, tl) {
+ if (event_initialized(&l->ev))
+ event_del(&l->ev);
+ if (evtimer_initialized(&l->pause))
+ evtimer_del(&l->pause);
+ TAILQ_REMOVE(&env->listeners, l, entry);
+ config_free_listener(l);
+ }
+
+ imsg_event_shutdown(listen_iev_parent);
+ free(listen_iev_parent);
+ imsg_event_shutdown(listen_iev_control);
+ free(listen_iev_control);
+ imsg_event_shutdown(listen_iev_broker);
+ free(listen_iev_broker);
+ for (n = 0; n < env->conf.workers; n++)
+ imsg_event_shutdown(&listen_iev_socks[n]);
+ free(listen_iev_socks);
+ free(worker_load);
+
+ log_info("listen exiting, pid %d", getpid());
+ exit(0);
+}
+
+static void
+listen_rlimit(void)
+{
+ struct rlimit rl;
+
+ if (getrlimit(RLIMIT_NOFILE, &rl) == -1)
+ fatal("%s: getrlimit", __func__);
+ log_debug("%s: max open files %llu", __func__,
+ (unsigned long long)rl.rlim_max);
+
+ /*
+ * Descriptors only live here until a worker has them, but a
+ * connect storm can leave many in flight at once.
+ */
+ rl.rlim_cur = rl.rlim_max;
+ if (setrlimit(RLIMIT_NOFILE, &rl) == -1)
+ fatal("%s: setrlimit", __func__);
+}
+
+/* bind and listen; called by the parent, which still has the privileges */
+int
+listen_bind(struct listener *l)
+{
+ struct sockaddr_storage *ss = &l->conf.ss;
+ struct sockaddr_un *sun;
+ int fd, on = 1;
+
+ if (ss->ss_family == AF_UNIX) {
+ sun = (struct sockaddr_un *)ss;
+ if (unlink(sun->sun_path) == -1 && errno != ENOENT) {
+ log_warn("%s: unlink %s", __func__, sun->sun_path);
+ return (-1);
+ }
+ fd = socket(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, 0);
+ if (fd == -1) {
+ log_warn("%s: socket", __func__);
+ return (-1);
+ }
+ if (bind(fd, (struct sockaddr *)sun, sizeof(*sun)) == -1) {
+ log_warn("%s: bind %s", __func__, sun->sun_path);
+ goto fail;
+ }
+ /* authentication still applies; let local users reach it */
+ if (chmod(sun->sun_path, MQTTD_UNIX_MODE) == -1) {
+ log_warn("%s: chmod %s", __func__, sun->sun_path);
+ goto fail;
+ }
+ if (listen(fd, SOMAXCONN) == -1) {
+ log_warn("%s: listen", __func__);
+ goto fail;
+ }
+ l->fd = fd;
+ return (0);
+ }
+
+ switch (ss->ss_family) {
+ case AF_INET:
+ ((struct sockaddr_in *)ss)->sin_port = l->conf.port;
+ ((struct sockaddr_in *)ss)->sin_len = sizeof(struct sockaddr_in);
+ break;
+ case AF_INET6:
+ ((struct sockaddr_in6 *)ss)->sin6_port = l->conf.port;
+ ((struct sockaddr_in6 *)ss)->sin6_len =
+ sizeof(struct sockaddr_in6);
+ break;
+ default:
+ log_warnx("%s: unsupported address family", __func__);
+ return (-1);
+ }
+
+ if ((fd = socket(ss->ss_family, SOCK_STREAM | SOCK_NONBLOCK, 0)) == -1) {
+ log_warn("%s: socket", __func__);
+ return (-1);
+ }
+ if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) {
+ log_warn("%s: setsockopt SO_REUSEADDR", __func__);
+ goto fail;
+ }
+ if (ss->ss_family == AF_INET6 &&
+ setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &on, sizeof(on)) == -1) {
+ log_warn("%s: setsockopt IPV6_V6ONLY", __func__);
+ goto fail;
+ }
+ if (bind(fd, (struct sockaddr *)ss, ss->ss_len) == -1) {
+ log_warn("%s: bind listener %s port %d", __func__, l->conf.name,
+ ntohs(l->conf.port));
+ goto fail;
+ }
+ if (listen(fd, SOMAXCONN) == -1) {
+ log_warn("%s: listen", __func__);
+ goto fail;
+ }
+
+ l->fd = fd;
+ return (0);
+fail:
+ close(fd);
+ return (-1);
+}
+
+static void
+listen_listener_launch(struct listener *l)
+{
+ log_debug("%s: listener %s[%u] on fd %d", __func__, l->conf.name,
+ l->conf.id, l->fd);
+
+ event_set(&l->ev, l->fd, EV_READ | EV_PERSIST, listen_accept, l);
+ if (event_add(&l->ev, NULL) == -1)
+ fatal("%s: event_add", __func__);
+ evtimer_set(&l->pause, listen_accept_paused, l);
+}
+
+/*
+ * A reload removed this listener. Connections already handed to a
+ * worker live on; their close is still reported with the listener id
+ * and simply finds nothing to decrement.
+ */
+static void
+listen_listener_remove(struct listener *l)
+{
+ struct lconn *c;
+
+ RB_FOREACH(c, lconntree, &lconns)
+ if (c->l == l)
+ c->l = NULL;
+
+ if (event_initialized(&l->ev))
+ event_del(&l->ev);
+ if (evtimer_initialized(&l->pause))
+ evtimer_del(&l->pause);
+ TAILQ_REMOVE(&env->listeners, l, entry);
+ env->nlisteners--;
+ config_free_listener(l);
+}
+
+static void
+listen_launch(void)
+{
+ struct listener *l;
+
+ TAILQ_FOREACH(l, &env->listeners, entry)
+ listen_listener_launch(l);
+}
+
+static void
+listen_dispatch_parent(int fd, short event, void *arg)
+{
+ struct imsgev *iev = arg;
+ struct imsgbuf *ibuf = &iev->ibuf;
+ struct imsg imsg;
+ struct listener *l;
+ uint32_t lid;
+ int n, verbose;
+
+ if (event & EV_READ) {
+ if ((n = imsgbuf_read(ibuf)) == -1)
+ fatal("%s: imsgbuf_read", __func__);
+ if (n == 0) {
+ log_warnx("%s: lost parent process", __func__);
+ listen_shutdown();
+ }
+ }
+ if (event & EV_WRITE) {
+ if (imsgbuf_write(ibuf) == -1) {
+ if (errno == EPIPE) {
+ log_warnx("%s: lost parent process", __func__);
+ listen_shutdown();
+ }
+ fatal("%s: imsgbuf_write", __func__);
+ }
+ }
+
+ for (;;) {
+ if ((n = imsgbuf_get(ibuf, &imsg)) == -1)
+ fatal("%s: imsgbuf_get", __func__);
+ if (n == 0)
+ break;
+
+ switch (imsg.hdr.type) {
+ case IMSG_CFG_GLOBAL:
+ config_recv_global(env, &imsg);
+ break;
+ case IMSG_CFG_LISTENER:
+ l = config_recv_listener(env, &imsg);
+ if (l->fd == -1)
+ fatalx("%s: listener %s arrived without a "
+ "descriptor", __func__, l->conf.name);
+ break;
+ case IMSG_CFG_BRIDGE:
+ listen_bridge_config(&imsg);
+ break;
+ case IMSG_CFG_LISTENER_START:
+ IMSG_SIZE_CHECK(&imsg, &lid);
+ memcpy(&lid, imsg.data, sizeof(lid));
+ if ((l = config_listener_byid(env, lid)) == NULL)
+ break;
+ listen_listener_launch(l);
+ log_info("listener %s added", l->conf.name);
+ break;
+ case IMSG_CFG_LISTENER_DEL:
+ IMSG_SIZE_CHECK(&imsg, &lid);
+ memcpy(&lid, imsg.data, sizeof(lid));
+ if ((l = config_listener_byid(env, lid)) == NULL)
+ break;
+ log_info("listener %s removed", l->conf.name);
+ listen_listener_remove(l);
+ break;
+ case IMSG_CFG_DONE:
+ imsg_compose_event(listen_iev_parent, IMSG_CFG_DONE, 0,
+ 0, -1, NULL, 0);
+ break;
+ case IMSG_CTL_START:
+ listen_launch();
+ break;
+ case IMSG_CTL_VERBOSE:
+ IMSG_SIZE_CHECK(&imsg, &verbose);
+ memcpy(&verbose, imsg.data, sizeof(verbose));
+ log_setverbose(verbose);
+ break;
+ case IMSG_CTL_SHOW_LISTENERS_REQUEST:
+ listen_show_listeners(&imsg);
+ break;
+ default:
+ log_warnx("%s: unexpected imsg %d from parent",
+ __func__, imsg.hdr.type);
+ break;
+ }
+ imsg_free(&imsg);
+ }
+ imsg_event_add(iev);
+}
+
+static void
+listen_dispatch_control(int fd, short event, void *arg)
+{
+ struct imsgev *iev = arg;
+ struct imsgbuf *ibuf = &iev->ibuf;
+ struct imsg imsg;
+ int n;
+
+ if (event & EV_READ) {
+ if ((n = imsgbuf_read(ibuf)) == -1)
+ fatal("%s: imsgbuf_read", __func__);
+ if (n == 0) {
+ log_warnx("%s: lost control process", __func__);
+ listen_shutdown();
+ }
+ }
+ if (event & EV_WRITE) {
+ if (imsgbuf_write(ibuf) == -1) {
+ if (errno == EPIPE) {
+ log_warnx("%s: lost control process", __func__);
+ listen_shutdown();
+ }
+ fatal("%s: imsgbuf_write", __func__);
+ }
+ }
+
+ /* control only listens to us; its requests arrive via the parent */
+ for (;;) {
+ if ((n = imsgbuf_get(ibuf, &imsg)) == -1)
+ fatal("%s: imsgbuf_get", __func__);
+ if (n == 0)
+ break;
+ log_warnx("%s: unexpected imsg %d from control", __func__,
+ imsg.hdr.type);
+ imsg_free(&imsg);
+ }
+ imsg_event_add(iev);
+}
+
+static void
+listen_dispatch_broker(int fd, short event, void *arg)
+{
+ struct imsgev *iev = arg;
+ struct imsgbuf *ibuf = &iev->ibuf;
+ struct imsg imsg;
+ int n;
+
+ if (event & EV_READ) {
+ if ((n = imsgbuf_read(ibuf)) == -1)
+ fatal("%s: imsgbuf_read", __func__);
+ if (n == 0) {
+ log_warnx("%s: lost broker process", __func__);
+ listen_shutdown();
+ }
+ }
+ if (event & EV_WRITE) {
+ if (imsgbuf_write(ibuf) == -1) {
+ if (errno == EPIPE) {
+ log_warnx("%s: lost broker process", __func__);
+ listen_shutdown();
+ }
+ fatal("%s: imsgbuf_write", __func__);
+ }
+ }
+
+ for (;;) {
+ if ((n = imsgbuf_get(ibuf, &imsg)) == -1)
+ fatal("%s: imsgbuf_get", __func__);
+ if (n == 0)
+ break;
+
+ switch (imsg.hdr.type) {
+ case IMSG_BRIDGE_CONNECT:
+ listen_bridge_connect(&imsg);
+ break;
+ default:
+ log_warnx("%s: unexpected imsg %d from broker",
+ __func__, imsg.hdr.type);
+ break;
+ }
+ imsg_free(&imsg);
+ }
+ imsg_event_add(iev);
+}
+
+static void
+listen_dispatch_socks(int fd, short event, void *arg)
+{
+ struct imsgev *iev = arg;
+ struct imsgbuf *ibuf = &iev->ibuf;
+ struct imsg imsg;
+ uint32_t w = (uint32_t)(iev - listen_iev_socks);
+ int n;
+
+ if (event & EV_READ) {
+ if ((n = imsgbuf_read(ibuf)) == -1)
+ fatal("%s: imsgbuf_read", __func__);
+ if (n == 0) {
+ log_warnx("%s: lost sockets worker %u", __func__, w);
+ listen_shutdown();
+ }
+ }
+ if (event & EV_WRITE) {
+ if (imsgbuf_write(ibuf) == -1) {
+ if (errno == EPIPE) {
+ log_warnx("%s: lost sockets worker %u",
+ __func__, w);
+ listen_shutdown();
+ }
+ fatal("%s: imsgbuf_write", __func__);
+ }
+ }
+
+ for (;;) {
+ if ((n = imsgbuf_get(ibuf, &imsg)) == -1)
+ fatal("%s: imsgbuf_get", __func__);
+ if (n == 0)
+ break;
+
+ switch (imsg.hdr.type) {
+ case IMSG_CONN_DONE:
+ listen_conn_done(w, &imsg);
+ break;
+ default:
+ log_warnx("%s: unexpected imsg %d from worker %u",
+ __func__, imsg.hdr.type, w);
+ break;
+ }
+ imsg_free(&imsg);
+ }
+ imsg_event_add(iev);
+}
+
+/* a worker closed a connection we handed it */
+static void
+listen_conn_done(uint32_t w, struct imsg *imsg)
+{
+ struct conn_done cd;
+ struct listener *l;
+
+ IMSG_SIZE_CHECK(imsg, &cd);
+ memcpy(&cd, imsg->data, sizeof(cd));
+
+ if (worker_load[w] > 0)
+ worker_load[w]--;
+ if (cd.listener_id != 0 &&
+ (l = config_listener_byid(env, cd.listener_id)) != NULL &&
+ l->nconns > 0)
+ l->nconns--;
+}
+
+static void
+listen_show_listeners(struct imsg *imsg)
+{
+ struct ctl_listener_info li;
+ struct listener *l;
+
+ TAILQ_FOREACH(l, &env->listeners, entry) {
+ memset(&li, 0, sizeof(li));
+ li.id = l->conf.id;
+ strlcpy(li.name, l->conf.name, sizeof(li.name));
+ sockaddr_ntop(&l->conf.ss, li.addr, sizeof(li.addr));
+ li.port = l->conf.port;
+ li.tls = l->conf.tls;
+ li.websocket = l->conf.websocket;
+ li.proxy = l->conf.proxy;
+ li.fd = l->fd;
+ li.nconns = l->nconns;
+ if (imsg_compose_event(listen_iev_control,
+ IMSG_CTL_SHOW_LISTENERS_DATA, imsg->hdr.peerid, 0, -1,
+ &li, sizeof(li)) == -1)
+ return;
+ }
+
+ imsg_compose_event(listen_iev_control, IMSG_CTL_SHOW_LISTENERS_END,
+ imsg->hdr.peerid, 0, -1, NULL, 0);
+}
+
+/* keep a few descriptors free for the imsg pipes and libevent itself */
+static int
+listen_accept_reserve(int sockfd, struct sockaddr *addr, socklen_t *addrlen)
+{
+ if (getdtablecount() + FD_RESERVE >= getdtablesize()) {
+ log_debug("%s: descriptor table nearly full", __func__);
+ errno = EMFILE;
+ return (-1);
+ }
+
+ return (accept4(sockfd, addr, addrlen, SOCK_NONBLOCK | SOCK_CLOEXEC));
+}
+
+static void
+listen_accept_paused(int fd, short event, void *arg)
+{
+ struct listener *l = arg;
+
+ event_add(&l->ev, NULL);
+}
+
+static void
+listen_accept(int fd, short event, void *arg)
+{
+ struct listener *l = arg;
+ struct lconn *c;
+ struct sockaddr_storage ss;
+ struct timeval backoff = { 1, 0 };
+ socklen_t len = sizeof(ss);
+ int cfd;
+
+ cfd = listen_accept_reserve(fd, (struct sockaddr *)&ss, &len);
+ if (cfd == -1) {
+ switch (errno) {
+ case EINTR:
+ case EWOULDBLOCK:
+ case ECONNABORTED:
+ return;
+ case EMFILE:
+ case ENFILE:
+ /* out of descriptors, give the clients a moment */
+ event_del(&l->ev);
+ evtimer_add(&l->pause, &backoff);
+ return;
+ default:
+ log_warn("%s: accept", __func__);
+ return;
+ }
+ }
+
+ if ((c = lconn_new()) == NULL) {
+ log_warn("%s: calloc", __func__);
+ close(cfd);
+ return;
+ }
+ c->fd = cfd;
+ c->l = l;
+ if (ss.ss_family == AF_UNIX)
+ strlcpy(c->peer, "unix", sizeof(c->peer));
+ else
+ sockaddr_ntop(&ss, c->peer, sizeof(c->peer));
+ l->nconns++;
+
+ log_debug("%s: connection %u from %s on listener %s", __func__, c->id,
+ c->peer, l->conf.name);
+
+ if (l->conf.proxy) {
+ /* the proxy has this long to send its header */
+ lconn_set_timer(c, MQTTD_CONNECT_TIMEOUT);
+ event_set(&c->ev, cfd, EV_READ, lconn_proxy_read, c);
+ event_add(&c->ev, NULL);
+ return;
+ }
+
+ lconn_handoff(c);
+}
+
+static struct lconn *
+lconn_new(void)
+{
+ struct lconn *c;
+
+ if ((c = calloc(1, sizeof(*c))) == NULL)
+ return (NULL);
+ do {
+ c->id = ++conn_next_id;
+ } while (c->id == 0 || lconn_find(c->id) != NULL);
+ c->fd = -1;
+ evtimer_set(&c->timer, lconn_timeout, c);
+ RB_INSERT(lconntree, &lconns, c);
+ return (c);
+}
+
+static struct lconn *
+lconn_find(uint32_t id)
+{
+ struct lconn key;
+
+ key.id = id;
+ return (RB_FIND(lconntree, &lconns, &key));
+}
+
+static void
+lconn_set_timer(struct lconn *c, time_t secs)
+{
+ struct timeval tv;
+
+ tv.tv_sec = secs;
+ tv.tv_usec = 0;
+ evtimer_del(&c->timer);
+ evtimer_add(&c->timer, &tv);
+}
+
+/* drop a connection that never reached a worker */
+static void
+lconn_free(struct lconn *c)
+{
+ if (c->bridge != NULL)
+ listen_bridge_failed(c->bridge->conf.id);
+
+ evtimer_del(&c->timer);
+ if (event_initialized(&c->ev))
+ event_del(&c->ev);
+ if (c->fd != -1)
+ close(c->fd);
+ if (c->l != NULL && c->l->nconns > 0)
+ c->l->nconns--;
+ RB_REMOVE(lconntree, &lconns, c);
+ free(c);
+}
+
+/* a worker owns it now; the listener count is released by IMSG_CONN_DONE */
+static void
+lconn_release(struct lconn *c)
+{
+ evtimer_del(&c->timer);
+ if (event_initialized(&c->ev))
+ event_del(&c->ev);
+ RB_REMOVE(lconntree, &lconns, c);
+ free(c);
+}
+
+/* lowest index among the workers with the fewest connections */
+static uint32_t
+listen_pick_worker(void)
+{
+ int w, best = 0;
+
+ for (w = 1; w < env->conf.workers; w++)
+ if (worker_load[w] < worker_load[best])
+ best = w;
+ return ((uint32_t)best);
+}
+
+static void
+lconn_handoff(struct lconn *c)
+{
+ struct conn_accept ca;
+ uint32_t w;
+
+ memset(&ca, 0, sizeof(ca));
+ ca.id = c->id;
+ ca.listener_id = c->l != NULL ? c->l->conf.id : 0;
+ ca.bridge_id = c->bridge != NULL ? c->bridge->conf.id : 0;
+ strlcpy(ca.peer, c->peer, sizeof(ca.peer));
+
+ w = listen_pick_worker();
+
+ /* imsg owns the descriptor from here and closes it once sent */
+ if (imsg_compose_event(&listen_iev_socks[w], IMSG_CONN_ACCEPT, 0, 0,
+ c->fd, &ca, sizeof(ca)) == -1) {
+ log_warn("%s: imsg_compose_event", __func__);
+ lconn_free(c);
+ return;
+ }
+ c->fd = -1;
+ worker_load[w]++;
+
+ log_debug("%s: connection %u %s %s to worker %u", __func__, c->id,
+ c->bridge != NULL ? "to" : "from", c->peer, w);
+
+ lconn_release(c);
+}
+
+static void
+lconn_timeout(int fd, short event, void *arg)
+{
+ struct lconn *c = arg;
+
+ log_debug("%s: connection %u %s %s timed out before hand-off",
+ __func__, c->id, c->bridge != NULL ? "to" : "from", c->peer);
+ lconn_free(c);
+}
+
+/*
+ * The proxy sends its header in one write before any client byte, so
+ * peek, work out the exact length, then consume exactly that much and
+ * leave the stream untouched for the worker.
+ */
+static void
+lconn_proxy_read(int fd, short event, void *arg)
+{
+ struct lconn *c = arg;
+ uint8_t buf[PROXY_PEEK];
+ ssize_t n;
+ int used;
+
+ n = recv(fd, buf, sizeof(buf), MSG_PEEK);
+ if (n == -1) {
+ if (errno == EAGAIN || errno == EINTR) {
+ event_add(&c->ev, NULL);
+ return;
+ }
+ lconn_free(c);
+ return;
+ }
+ if (n == 0) {
+ lconn_free(c);
+ return;
+ }
+
+ if (n >= PROXY_V2_MIN &&
+ memcmp(buf, "\x0d\x0a\x0d\x0a\x00\x0d\x0a\x51\x55\x49\x54\x0a",
+ 12) == 0)
+ used = proxy_parse_v2(c, buf, n);
+ else if (n >= 6 && memcmp(buf, "PROXY ", 6) == 0)
+ used = proxy_parse_v1(c, buf, n);
+ else {
+ log_warnx("%s: no PROXY header from %s", __func__, c->peer);
+ lconn_free(c);
+ return;
+ }
+
+ if (used == 0) {
+ /* incomplete header, wait for the rest */
+ if (n >= (ssize_t)sizeof(buf)) {
+ lconn_free(c);
+ return;
+ }
+ event_add(&c->ev, NULL);
+ return;
+ }
+ if (used == -1) {
+ log_warnx("%s: bad PROXY header from %s", __func__, c->peer);
+ lconn_free(c);
+ return;
+ }
+
+ /* the peeked bytes are still there; this read cannot block */
+ if (recv(fd, buf, used, MSG_WAITALL) != used) {
+ lconn_free(c);
+ return;
+ }
+ event_del(&c->ev);
+ lconn_handoff(c);
+}
+
+/* returns bytes consumed, 0 for more needed, -1 for malformed */
+static int
+proxy_parse_v1(struct lconn *c, const uint8_t *buf, size_t n)
+{
+ char line[PROXY_V1_MAX + 1], *p, *end, *fam, *src, *sport;
+ const char *eol;
+ size_t len;
+
+ eol = memmem(buf, n < PROXY_V1_MAX ? n : PROXY_V1_MAX, "\r\n", 2);
+ if (eol == NULL)
+ return (n >= PROXY_V1_MAX ? -1 : 0);
+ len = eol - (const char *)buf;
+ memcpy(line, buf, len);
+ line[len] = '\0';
+
+ /* PROXY TCP4 src dst sport dport, or PROXY UNKNOWN ... */
+ p = line + 6;
+ end = p;
+ fam = strsep(&end, " ");
+ if (fam == NULL)
+ return (-1);
+ if (strcmp(fam, "UNKNOWN") == 0)
+ return (len + 2);
+ if (strcmp(fam, "TCP4") != 0 && strcmp(fam, "TCP6") != 0)
+ return (-1);
+ src = strsep(&end, " ");
+ (void)strsep(&end, " ");
+ sport = strsep(&end, " ");
+ if (src == NULL || sport == NULL || end == NULL)
+ return (-1);
+ if (strchr(src, ':') != NULL)
+ snprintf(c->peer, sizeof(c->peer), "[%s]:%s", src, sport);
+ else
+ snprintf(c->peer, sizeof(c->peer), "%s:%s", src, sport);
+ return (len + 2);
+}
+
+static int
+proxy_parse_v2(struct lconn *c, const uint8_t *buf, size_t n)
+{
+ char host[INET6_ADDRSTRLEN];
+ size_t len, total;
+ uint8_t cmd, fam;
+ uint16_t port;
+
+ if ((buf[12] >> 4) != 2)
+ return (-1);
+ cmd = buf[12] & 0x0f;
+ fam = buf[13];
+ len = (size_t)buf[14] << 8 | buf[15];
+ total = PROXY_V2_MIN + len;
+ if (total > PROXY_PEEK)
+ return (-1);
+ if (n < total)
+ return (0);
+
+ /* LOCAL (health checks) keeps the proxy's own address */
+ if (cmd != 0x01)
+ return (total);
+
+ switch (fam) {
+ case 0x11:
+ if (len < 12)
+ return (-1);
+ inet_ntop(AF_INET, buf + 16, host, sizeof(host));
+ port = (uint16_t)buf[24] << 8 | buf[25];
+ snprintf(c->peer, sizeof(c->peer), "%s:%u", host, port);
+ break;
+ case 0x21:
+ if (len < 36)
+ return (-1);
+ inet_ntop(AF_INET6, buf + 16, host, sizeof(host));
+ port = (uint16_t)buf[48] << 8 | buf[49];
+ snprintf(c->peer, sizeof(c->peer), "[%s]:%u", host, port);
+ break;
+ default:
+ break;
+ }
+ return (total);
+}
+
+static struct lbridge *
+lbridge_find(uint32_t id)
+{
+ struct lbridge *lb;
+
+ TAILQ_FOREACH(lb, &lbridges, entry)
+ if (lb->conf.id == id)
+ return (lb);
+ return (NULL);
+}
+
+static void
+listen_bridge_config(struct imsg *imsg)
+{
+ struct lbridge *lb;
+
+ if ((lb = calloc(1, sizeof(*lb))) == NULL)
+ fatal("%s: calloc", __func__);
+ IMSG_SIZE_CHECK(imsg, &lb->conf);
+ memcpy(&lb->conf, imsg->data, sizeof(lb->conf));
+ lb->conf.name[sizeof(lb->conf.name) - 1] = '\0';
+ lb->conf.host[sizeof(lb->conf.host) - 1] = '\0';
+ lb->conf.tls_ca_file = NULL;
+ lb->conf.tls_ca = NULL;
+ lb->conf.tls_ca_len = 0;
+ explicit_bzero(lb->conf.password, sizeof(lb->conf.password));
+ TAILQ_INSERT_TAIL(&lbridges, lb, entry);
+}
+
+static void
+listen_bridge_failed(uint32_t id)
+{
+ imsg_compose_event(listen_iev_broker, IMSG_BRIDGE_FAILED, 0, 0, -1,
+ &id, sizeof(id));
+}
+
+/* the broker wants this bridge up: start a non-blocking connect */
+static void
+listen_bridge_connect(struct imsg *imsg)
+{
+ struct lbridge *lb;
+ struct lconn *c;
+ struct sockaddr_storage ss;
+ uint32_t id;
+ int fd;
+
+ IMSG_SIZE_CHECK(imsg, &id);
+ memcpy(&id, imsg->data, sizeof(id));
+ if ((lb = lbridge_find(id)) == NULL) {
+ log_warnx("%s: unknown bridge %u", __func__, id);
+ listen_bridge_failed(id);
+ return;
+ }
+
+ memcpy(&ss, &lb->conf.ss, sizeof(ss));
+ switch (ss.ss_family) {
+ case AF_INET:
+ ((struct sockaddr_in *)&ss)->sin_port = lb->conf.port;
+ break;
+ case AF_INET6:
+ ((struct sockaddr_in6 *)&ss)->sin6_port = lb->conf.port;
+ break;
+ }
+
+ fd = socket(ss.ss_family, SOCK_STREAM | SOCK_NONBLOCK, 0);
+ if (fd == -1) {
+ log_warn("%s: socket", __func__);
+ listen_bridge_failed(id);
+ return;
+ }
+
+ if ((c = lconn_new()) == NULL) {
+ log_warn("%s: calloc", __func__);
+ close(fd);
+ listen_bridge_failed(id);
+ return;
+ }
+ c->fd = fd;
+ c->bridge = lb;
+ sockaddr_ntop(&ss, c->peer, sizeof(c->peer));
+ lconn_set_timer(c, MQTTD_CONNECT_TIMEOUT);
+
+ if (connect(fd, (struct sockaddr *)&ss, ss.ss_len) == -1 &&
+ errno != EINPROGRESS) {
+ log_warn("bridge %s: connect to %s", lb->conf.name, c->peer);
+ lconn_free(c);
+ return;
+ }
+
+ event_set(&c->ev, fd, EV_WRITE, listen_bridge_connected, c);
+ event_add(&c->ev, NULL);
+}
+
+static void
+listen_bridge_connected(int fd, short event, void *arg)
+{
+ struct lconn *c = arg;
+ socklen_t len = sizeof(int);
+ int err = 0;
+
+ if (getsockopt(fd, SOL_SOCKET, SO_ERROR, &err, &len) == -1)
+ err = errno;
+ if (err != 0) {
+ log_warnx("bridge %s: connect to %s: %s", c->bridge->conf.name,
+ c->peer, strerror(err));
+ lconn_free(c);
+ return;
+ }
+
+ /* connected; the worker runs TLS and the MQTT client side */
+ lconn_handoff(c);
+}
blob - 412c68b097457a0a5dee2463afc903894b23fff1508e1de12cda91ff6f49e84e
blob + 7978b018372beb9abf030ddad5e13e1d12525ebb6c4eae1b54657567c789a535
--- mqttd/mqttd.8
+++ mqttd/mqttd.8
warehouse confirmed.
The parent resolves names and signs its JWTs so it needs neither DNS nor
the private key.
+.It listen
+Owns the listening sockets.
+It accepts connections, consumes the PROXY protocol header where a
+listener expects one, dials outbound bridges and hands every connected
+descriptor to the sockets worker with the fewest connections.
.It sockets
-Accepts connections, runs TLS, splits the byte stream into control
-packets and relays them to the broker.
+A configurable number of worker processes, see
+.Ic workers
+in
+.Xr mqttd.conf 5 ,
+which run TLS, split the byte stream into control packets and relay
+them to the broker.
+A worker never opens a socket itself.
.El
.Pp
Apart from auth, all children run
listeners take effect immediately without dropping connections; a
removed listener's existing clients stay connected.
Changes to
+.Ic workers ,
.Ic persistence ,
.Ic auth
or bridges are reported in the log and need a restart.
blob - 1cc61a93d33ca7110f189e306477af4a169866335eaecfbea136bb9822813120
blob + 0c3eb7fa9ed2ccc51118e6d91519e7254acfe5f6d4ee3b1e0a59735f62b94796
--- mqttd/mqttd.c
+++ mqttd/mqttd.c
/*
* The parent keeps root for the privileged setup: reading the
* configuration and certificates, binding the listeners and forking the
- * control, broker and sockets processes. Afterwards it only relays
+ * control, broker, listen and sockets processes. Afterwards it only relays
* control requests and waits for its children.
*/
static void parent_dispatch_snow(int, short, void *);
static void parent_snow_resolve(struct imsg *);
static void parent_snow_jwt(struct imsg *);
+static void parent_dispatch_listen(int, short, void *);
static void parent_dispatch_sockets(int, short, void *);
static void parent_sighdlr(int, short, void *);
static void parent_show_info(struct imsg *);
static struct imsgev *parent_iev_storage;
static struct imsgev *parent_iev_auth;
static struct imsgev *parent_iev_snow;
+static struct imsgev *parent_iev_listen;
static struct imsgev *parent_iev_socks;
static struct event ev_sigint;
struct imsgev *iev = arg;
struct imsgbuf *ibuf = &iev->ibuf;
struct imsg imsg;
- int n, verbose;
+ int i, n, verbose;
if (event & EV_READ) {
if ((n = imsgbuf_read(ibuf)) == -1)
imsg_forward_event(parent_iev_broker, &imsg);
break;
case IMSG_CTL_SHOW_LISTENERS_REQUEST:
- imsg_forward_event(parent_iev_socks, &imsg);
+ imsg_forward_event(parent_iev_listen, &imsg);
break;
case IMSG_CTL_VERBOSE:
IMSG_SIZE_CHECK(&imsg, &verbose);
memcpy(&verbose, imsg.data, sizeof(verbose));
log_setverbose(verbose);
imsg_forward_event(parent_iev_broker, &imsg);
- imsg_forward_event(parent_iev_socks, &imsg);
+ imsg_forward_event(parent_iev_listen, &imsg);
+ for (i = 0; i < env->conf.workers; i++)
+ imsg_forward_event(&parent_iev_socks[i], &imsg);
break;
default:
log_warnx("%s: unexpected imsg %d from control",
}
static void
+parent_dispatch_listen(int fd, short event, void *arg)
+{
+ struct imsgev *iev = arg;
+ struct imsgbuf *ibuf = &iev->ibuf;
+ struct imsg imsg;
+ int n;
+
+ if (event & EV_READ) {
+ if ((n = imsgbuf_read(ibuf)) == -1)
+ fatal("%s: imsgbuf_read", __func__);
+ if (n == 0) {
+ log_warnx("%s: lost listen process", __func__);
+ parent_shutdown(env);
+ }
+ }
+ if (event & EV_WRITE) {
+ if (imsgbuf_write(ibuf) == -1) {
+ if (errno == EPIPE) {
+ log_warnx("%s: lost listen process", __func__);
+ parent_shutdown(env);
+ }
+ fatal("%s: imsgbuf_write", __func__);
+ }
+ }
+
+ for (;;) {
+ if ((n = imsgbuf_get(ibuf, &imsg)) == -1)
+ fatal("%s: imsgbuf_get", __func__);
+ if (n == 0)
+ break;
+
+ switch (imsg.hdr.type) {
+ case IMSG_CFG_DONE:
+ parent_configure_done(env);
+ break;
+ default:
+ log_warnx("%s: unexpected imsg %d from listen",
+ __func__, imsg.hdr.type);
+ break;
+ }
+ imsg_free(&imsg);
+ }
+ imsg_event_add(iev);
+}
+
+static void
parent_dispatch_sockets(int fd, short event, void *arg)
{
struct imsgev *iev = arg;
struct mqttd *e;
const char *conffile = MQTTD_CONF;
struct stat st;
- int ch;
+ int ch, i, n;
int pipe_control[2];
int pipe_broker[2];
int pipe_storage[2];
int pipe_broker_auth[2];
int pipe_snow[2];
int pipe_broker_snow[2];
- int pipe_socks[2];
- int pipe_control_socks[2];
- int pipe_broker_socks[2];
+ int pipe_listen[2];
+ int pipe_control_listen[2];
+ int pipe_broker_listen[2];
+ int pipe_socks[MQTTD_MAX_WORKERS][2];
+ int pipe_listen_socks[MQTTD_MAX_WORKERS][2];
+ int pipe_broker_socks[MQTTD_MAX_WORKERS][2];
+ int fds[MQTTD_MAX_WORKERS];
if ((e = calloc(1, sizeof(*e))) == NULL)
fatal("%s: calloc", __func__);
socketpair(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, PF_UNSPEC,
pipe_broker_snow) == -1 ||
socketpair(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, PF_UNSPEC,
- pipe_socks) == -1 ||
+ pipe_listen) == -1 ||
socketpair(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, PF_UNSPEC,
- pipe_control_socks) == -1 ||
+ pipe_control_listen) == -1 ||
socketpair(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, PF_UNSPEC,
- pipe_broker_socks) == -1)
+ pipe_broker_listen) == -1)
fatal("%s: socketpair", __func__);
+ for (i = 0; i < e->conf.workers; i++) {
+ if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, PF_UNSPEC,
+ pipe_socks[i]) == -1 ||
+ socketpair(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, PF_UNSPEC,
+ pipe_listen_socks[i]) == -1 ||
+ socketpair(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, PF_UNSPEC,
+ pipe_broker_socks[i]) == -1)
+ fatal("%s: socketpair", __func__);
+ }
- /* control: parent[1], control_broker[0], control_socks[0] */
+ /* control: parent[1], control_broker[0], control_listen[0] */
switch (fork()) {
case -1:
fatal("%s: fork", __func__);
close(pipe_broker_auth[1]);
close(pipe_broker_snow[0]);
close(pipe_broker_snow[1]);
- close(pipe_socks[0]);
- close(pipe_socks[1]);
- close(pipe_control_socks[1]);
- close(pipe_broker_socks[0]);
- close(pipe_broker_socks[1]);
+ close(pipe_listen[0]);
+ close(pipe_listen[1]);
+ close(pipe_control_listen[1]);
+ close(pipe_broker_listen[0]);
+ close(pipe_broker_listen[1]);
+ for (i = 0; i < e->conf.workers; i++) {
+ close(pipe_socks[i][0]);
+ close(pipe_socks[i][1]);
+ close(pipe_listen_socks[i][0]);
+ close(pipe_listen_socks[i][1]);
+ close(pipe_broker_socks[i][0]);
+ close(pipe_broker_socks[i][1]);
+ }
control(pipe_control[1], pipe_control_broker[0],
- pipe_control_socks[0]);
+ pipe_control_listen[0]);
/* NOTREACHED */
}
close(pipe_broker_auth[0]);
close(pipe_broker_auth[1]);
close(pipe_broker_snow[0]);
- close(pipe_socks[0]);
- close(pipe_socks[1]);
- close(pipe_control_socks[0]);
- close(pipe_control_socks[1]);
- close(pipe_broker_socks[0]);
- close(pipe_broker_socks[1]);
+ close(pipe_listen[0]);
+ close(pipe_listen[1]);
+ close(pipe_control_listen[0]);
+ close(pipe_control_listen[1]);
+ close(pipe_broker_listen[0]);
+ close(pipe_broker_listen[1]);
+ for (i = 0; i < e->conf.workers; i++) {
+ close(pipe_socks[i][0]);
+ close(pipe_socks[i][1]);
+ close(pipe_listen_socks[i][0]);
+ close(pipe_listen_socks[i][1]);
+ close(pipe_broker_socks[i][0]);
+ close(pipe_broker_socks[i][1]);
+ }
snowflake(pipe_snow[1], pipe_broker_snow[1]);
/* NOTREACHED */
}
close(pipe_broker_auth[1]);
close(pipe_broker_snow[0]);
close(pipe_broker_snow[1]);
- close(pipe_socks[0]);
- close(pipe_socks[1]);
- close(pipe_control_socks[0]);
- close(pipe_control_socks[1]);
- close(pipe_broker_socks[0]);
- close(pipe_broker_socks[1]);
+ close(pipe_listen[0]);
+ close(pipe_listen[1]);
+ close(pipe_control_listen[0]);
+ close(pipe_control_listen[1]);
+ close(pipe_broker_listen[0]);
+ close(pipe_broker_listen[1]);
+ for (i = 0; i < e->conf.workers; i++) {
+ close(pipe_socks[i][0]);
+ close(pipe_socks[i][1]);
+ close(pipe_listen_socks[i][0]);
+ close(pipe_listen_socks[i][1]);
+ close(pipe_broker_socks[i][0]);
+ close(pipe_broker_socks[i][1]);
+ }
storage(pipe_storage[1], pipe_broker_storage[1]);
/* NOTREACHED */
}
close(pipe_broker_auth[0]);
close(pipe_broker_snow[0]);
close(pipe_broker_snow[1]);
- close(pipe_socks[0]);
- close(pipe_socks[1]);
- close(pipe_control_socks[0]);
- close(pipe_control_socks[1]);
- close(pipe_broker_socks[0]);
- close(pipe_broker_socks[1]);
+ close(pipe_listen[0]);
+ close(pipe_listen[1]);
+ close(pipe_control_listen[0]);
+ close(pipe_control_listen[1]);
+ close(pipe_broker_listen[0]);
+ close(pipe_broker_listen[1]);
+ for (i = 0; i < e->conf.workers; i++) {
+ close(pipe_socks[i][0]);
+ close(pipe_socks[i][1]);
+ close(pipe_listen_socks[i][0]);
+ close(pipe_listen_socks[i][1]);
+ close(pipe_broker_socks[i][0]);
+ close(pipe_broker_socks[i][1]);
+ }
auth(pipe_auth[1], pipe_broker_auth[1]);
/* NOTREACHED */
}
close(pipe_broker_storage[1]);
close(pipe_broker_auth[1]);
close(pipe_broker_snow[1]);
- close(pipe_socks[0]);
- close(pipe_socks[1]);
- close(pipe_control_socks[0]);
- close(pipe_control_socks[1]);
- close(pipe_broker_socks[1]);
+ close(pipe_listen[0]);
+ close(pipe_listen[1]);
+ close(pipe_control_listen[0]);
+ close(pipe_control_listen[1]);
+ close(pipe_broker_listen[1]);
+ for (i = 0; i < e->conf.workers; i++) {
+ close(pipe_socks[i][0]);
+ close(pipe_socks[i][1]);
+ close(pipe_listen_socks[i][0]);
+ close(pipe_listen_socks[i][1]);
+ close(pipe_broker_socks[i][1]);
+ fds[i] = pipe_broker_socks[i][0];
+ }
broker(pipe_broker[1], pipe_control_broker[1],
pipe_broker_storage[0], pipe_broker_auth[0],
- pipe_broker_snow[0], pipe_broker_socks[0]);
+ pipe_broker_snow[0], pipe_broker_listen[0], fds);
/* NOTREACHED */
}
- /* sockets: socks[1], control_socks[1], broker_socks[1] */
+ /*
+ * listen: listen[1], control_listen[1], broker_listen[1] and
+ * listen_socks[i][0]
+ */
switch (fork()) {
case -1:
fatal("%s: fork", __func__);
close(pipe_broker_auth[1]);
close(pipe_broker_snow[0]);
close(pipe_broker_snow[1]);
- close(pipe_socks[0]);
- close(pipe_control_socks[0]);
- close(pipe_broker_socks[0]);
- sockets(pipe_socks[1], pipe_control_socks[1],
- pipe_broker_socks[1]);
+ close(pipe_listen[0]);
+ close(pipe_control_listen[0]);
+ close(pipe_broker_listen[0]);
+ for (i = 0; i < e->conf.workers; i++) {
+ close(pipe_socks[i][0]);
+ close(pipe_socks[i][1]);
+ close(pipe_listen_socks[i][1]);
+ close(pipe_broker_socks[i][0]);
+ close(pipe_broker_socks[i][1]);
+ fds[i] = pipe_listen_socks[i][0];
+ }
+ listen_main(pipe_listen[1], pipe_control_listen[1],
+ pipe_broker_listen[1], fds);
/* NOTREACHED */
}
+ /* sockets n: socks[n][1], listen_socks[n][1], broker_socks[n][1] */
+ for (n = 0; n < e->conf.workers; n++) {
+ switch (fork()) {
+ case -1:
+ fatal("%s: fork", __func__);
+ case 0:
+ close(pipe_control[0]);
+ close(pipe_control[1]);
+ close(pipe_broker[0]);
+ close(pipe_broker[1]);
+ close(pipe_storage[0]);
+ close(pipe_storage[1]);
+ close(pipe_auth[0]);
+ close(pipe_auth[1]);
+ close(pipe_snow[0]);
+ close(pipe_snow[1]);
+ close(pipe_control_broker[0]);
+ close(pipe_control_broker[1]);
+ close(pipe_broker_storage[0]);
+ close(pipe_broker_storage[1]);
+ close(pipe_broker_auth[0]);
+ close(pipe_broker_auth[1]);
+ close(pipe_broker_snow[0]);
+ close(pipe_broker_snow[1]);
+ close(pipe_listen[0]);
+ close(pipe_listen[1]);
+ close(pipe_control_listen[0]);
+ close(pipe_control_listen[1]);
+ close(pipe_broker_listen[0]);
+ close(pipe_broker_listen[1]);
+ for (i = 0; i < e->conf.workers; i++) {
+ close(pipe_socks[i][0]);
+ close(pipe_listen_socks[i][0]);
+ close(pipe_broker_socks[i][0]);
+ if (i == n)
+ continue;
+ close(pipe_socks[i][1]);
+ close(pipe_listen_socks[i][1]);
+ close(pipe_broker_socks[i][1]);
+ }
+ sockets(pipe_socks[n][1], pipe_listen_socks[n][1],
+ pipe_broker_socks[n][1]);
+ /* NOTREACHED */
+ }
+ }
+
/* parent: the [0] end of every pipe to a child */
close(pipe_control[1]);
close(pipe_broker[1]);
close(pipe_broker_auth[1]);
close(pipe_broker_snow[0]);
close(pipe_broker_snow[1]);
- close(pipe_socks[1]);
- close(pipe_control_socks[0]);
- close(pipe_control_socks[1]);
- close(pipe_broker_socks[0]);
- close(pipe_broker_socks[1]);
+ close(pipe_listen[1]);
+ close(pipe_control_listen[0]);
+ close(pipe_control_listen[1]);
+ close(pipe_broker_listen[0]);
+ close(pipe_broker_listen[1]);
+ for (i = 0; i < e->conf.workers; i++) {
+ close(pipe_socks[i][1]);
+ close(pipe_listen_socks[i][0]);
+ close(pipe_listen_socks[i][1]);
+ close(pipe_broker_socks[i][0]);
+ close(pipe_broker_socks[i][1]);
+ }
event_init();
fatal("%s: calloc", __func__);
imsg_event_init(parent_iev_snow, pipe_snow[0], parent_dispatch_snow);
- if ((parent_iev_socks = calloc(1, sizeof(*parent_iev_socks))) == NULL)
+ if ((parent_iev_listen = calloc(1, sizeof(*parent_iev_listen))) == NULL)
fatal("%s: calloc", __func__);
- imsg_event_init(parent_iev_socks, pipe_socks[0],
- parent_dispatch_sockets);
+ imsg_event_init(parent_iev_listen, pipe_listen[0],
+ parent_dispatch_listen);
+ if ((parent_iev_socks = calloc(e->conf.workers,
+ sizeof(*parent_iev_socks))) == NULL)
+ fatal("%s: calloc", __func__);
+ for (i = 0; i < e->conf.workers; i++)
+ imsg_event_init(&parent_iev_socks[i], pipe_socks[i][0],
+ parent_dispatch_sockets);
+
if (parent_configure(e) == -1)
fatalx("configuration failed");
/*
* A reload re-parses (rpath dns route), may bind new listeners
* (inet unix cpath fattr) and hands their descriptors to the
- * sockets process (sendfd).
+ * listen process (sendfd).
*/
if (pledge("stdio rpath inet dns route proc sendfd unix cpath fattr",
NULL) == -1)
struct user *u;
struct acl *a;
struct bridge_def *b;
- int fd;
+ struct bridge_config bc;
+ int n;
TAILQ_FOREACH(l, &e->listeners, entry) {
- if (sockets_create_listener(l) == -1)
+ if (listen_bind(l) == -1)
return (-1);
if (listener_tls_load(l) == -1)
return (-1);
- if ((fd = dup(l->fd)) == -1) {
- log_warn("%s: dup", __func__);
+ /* listen takes the descriptor, the workers the TLS material */
+ if (config_send_listener(parent_iev_listen, l, l->fd) == -1)
return (-1);
- }
- if (config_send_listener(parent_iev_socks, l, fd) == -1)
- return (-1);
- if (config_send_tls(parent_iev_socks, l) == -1)
- return (-1);
-
- /* the sockets process holds its own copy now */
- close(l->fd);
l->fd = -1;
+ for (n = 0; n < e->conf.workers; n++) {
+ if (config_send_listener(&parent_iev_socks[n], l,
+ -1) == -1)
+ return (-1);
+ if (config_send_tls(&parent_iev_socks[n], l) == -1)
+ return (-1);
+ }
+
freezero(l->conf.tls_key, l->conf.tls_key_len);
l->conf.tls_key = NULL;
l->conf.tls_key_len = 0;
return (-1);
if (config_send_global(parent_iev_snow, e) == -1)
return (-1);
- if (config_send_global(parent_iev_socks, e) == -1)
+ if (config_send_global(parent_iev_listen, e) == -1)
return (-1);
+ for (n = 0; n < e->conf.workers; n++)
+ if (config_send_global(&parent_iev_socks[n], e) == -1)
+ return (-1);
TAILQ_FOREACH(u, &e->users, entry)
if (config_send_user(parent_iev_broker, u) == -1)
return (-1);
/*
- * Bridges: the broker gets the whole definition, the sockets
- * process that will dial out gets the address and CA bundle.
+ * Bridges: the broker gets the whole definition, listen the address
+ * to dial and every worker the address and CA bundle, both without
+ * the password.
*/
TAILQ_FOREACH(b, &e->bridges, entry) {
if (b->snow != NULL) {
}
if (config_send_bridge(parent_iev_broker, b) == -1)
return (-1);
- if (imsg_compose_event(parent_iev_socks, IMSG_CFG_BRIDGE, 0, 0,
- -1, &b->conf, sizeof(b->conf)) == -1)
+ bc = b->conf;
+ explicit_bzero(bc.password, sizeof(bc.password));
+ if (imsg_compose_event(parent_iev_listen, IMSG_CFG_BRIDGE, 0, 0,
+ -1, &bc, sizeof(bc)) == -1)
return (-1);
- if (config_send_bridge_ca(parent_iev_socks, b) == -1)
- return (-1);
+ for (n = 0; n < e->conf.workers; n++) {
+ if (imsg_compose_event(&parent_iev_socks[n],
+ IMSG_CFG_BRIDGE, 0, 0, -1, &bc, sizeof(bc)) == -1)
+ return (-1);
+ if (config_send_bridge_ca(&parent_iev_socks[n],
+ b) == -1)
+ return (-1);
+ }
explicit_bzero(b->conf.password, sizeof(b->conf.password));
}
* The broker only acknowledges once storage has replayed the last
* snapshot into it, so no client connects to a half restored state.
*/
- e->cfg_pending = 5;
+ e->cfg_pending = 5 + e->conf.workers;
imsg_compose_event(parent_iev_storage, IMSG_CFG_DONE, 0, 0, -1, NULL,
0);
imsg_compose_event(parent_iev_auth, IMSG_CFG_DONE, 0, 0, -1, NULL, 0);
imsg_compose_event(parent_iev_snow, IMSG_CFG_DONE, 0, 0, -1, NULL, 0);
imsg_compose_event(parent_iev_broker, IMSG_CFG_DONE, 0, 0, -1, NULL, 0);
- imsg_compose_event(parent_iev_socks, IMSG_CFG_DONE, 0, 0, -1, NULL, 0);
+ imsg_compose_event(parent_iev_listen, IMSG_CFG_DONE, 0, 0, -1, NULL, 0);
+ for (n = 0; n < e->conf.workers; n++)
+ imsg_compose_event(&parent_iev_socks[n], IMSG_CFG_DONE, 0, 0,
+ -1, NULL, 0);
return (0);
}
static void
parent_configure_done(struct mqttd *e)
{
+ int n;
+
if (e->cfg_pending == 0) {
log_warnx("%s: configuration already finished", __func__);
return;
0);
imsg_compose_event(parent_iev_auth, IMSG_CTL_START, 0, 0, -1, NULL, 0);
imsg_compose_event(parent_iev_snow, IMSG_CTL_START, 0, 0, -1, NULL, 0);
- imsg_compose_event(parent_iev_socks, IMSG_CTL_START, 0, 0, -1, NULL, 0);
+ /* workers first, so listen never hands over to an unprepared one */
+ for (n = 0; n < e->conf.workers; n++)
+ imsg_compose_event(&parent_iev_socks[n], IMSG_CTL_START, 0, 0,
+ -1, NULL, 0);
+ imsg_compose_event(parent_iev_listen, IMSG_CTL_START, 0, 0, -1, NULL, 0);
- log_info("%s ready, %u listener(s)", getprogname(), e->nlisteners);
+ log_info("%s ready, %u listener(s), %u worker(s)", getprogname(),
+ e->nlisteners, e->conf.workers);
}
static int
struct user *u;
struct acl *a;
struct bridge_def *b;
- int fd;
+ int i;
log_info("reloading configuration from %s", e->conffile);
return;
}
+ if (n->conf.workers != e->conf.workers) {
+ log_warnx("workers changed, a restart is needed for that");
+ n->conf.workers = e->conf.workers;
+ }
if (n->conf.persist != e->conf.persist ||
strcmp(n->dbdir, e->dbdir) != 0) {
log_warnx("persistence settings changed, a restart is needed "
"material", nl->conf.name);
continue;
}
- if (config_send_tls(parent_iev_socks, nl) == 0)
- imsg_compose_event(parent_iev_socks,
+ for (i = 0; i < e->conf.workers; i++) {
+ if (config_send_tls(&parent_iev_socks[i],
+ nl) == -1)
+ break;
+ imsg_compose_event(&parent_iev_socks[i],
IMSG_CFG_TLS_DONE, 0, 0, -1, &nl->conf.id,
sizeof(nl->conf.id));
+ }
} else {
- if (sockets_create_listener(nl) == -1 ||
+ if (listen_bind(nl) == -1 ||
listener_tls_load(nl) == -1) {
log_warnx("listener %s could not be created, "
"skipped", nl->conf.name);
nl->conf.id = 0;
continue;
}
- if ((fd = dup(nl->fd)) != -1 &&
- config_send_listener(parent_iev_socks, nl,
- fd) == 0 &&
- config_send_tls(parent_iev_socks, nl) == 0)
- imsg_compose_event(parent_iev_socks,
+ /*
+ * The workers hear about the listener first, so a
+ * connection listen hands over finds its TLS
+ * material in place.
+ */
+ for (i = 0; i < e->conf.workers; i++) {
+ if (config_send_listener(&parent_iev_socks[i],
+ nl, -1) == -1 ||
+ config_send_tls(&parent_iev_socks[i],
+ nl) == -1)
+ break;
+ imsg_compose_event(&parent_iev_socks[i],
IMSG_CFG_LISTENER_START, 0, 0, -1,
&nl->conf.id, sizeof(nl->conf.id));
- close(nl->fd);
+ }
+ if (config_send_listener(parent_iev_listen, nl,
+ nl->fd) == -1) {
+ log_warnx("listener %s could not be handed "
+ "over, skipped", nl->conf.name);
+ close(nl->fd);
+ nl->fd = -1;
+ nl->conf.id = 0;
+ continue;
+ }
nl->fd = -1;
+ imsg_compose_event(parent_iev_listen,
+ IMSG_CFG_LISTENER_START, 0, 0, -1, &nl->conf.id,
+ sizeof(nl->conf.id));
log_info("listener %s added", nl->conf.name);
}
freezero(nl->conf.tls_key, nl->conf.tls_key_len);
TAILQ_FOREACH_SAFE(ol, &e->listeners, entry, tl) {
if (ol->fd != -2) {
- imsg_compose_event(parent_iev_socks,
+ imsg_compose_event(parent_iev_listen,
IMSG_CFG_LISTENER_DEL, 0, 0, -1, &ol->conf.id,
sizeof(ol->conf.id));
+ for (i = 0; i < e->conf.workers; i++)
+ imsg_compose_event(&parent_iev_socks[i],
+ IMSG_CFG_LISTENER_DEL, 0, 0, -1,
+ &ol->conf.id, sizeof(ol->conf.id));
log_info("listener %s removed", ol->conf.name);
}
ol->fd = -1;
config_send_global(parent_iev_storage, e);
config_send_global(parent_iev_auth, e);
config_send_global(parent_iev_snow, e);
- config_send_global(parent_iev_socks, e);
+ config_send_global(parent_iev_listen, e);
+ for (i = 0; i < e->conf.workers; i++)
+ config_send_global(&parent_iev_socks[i], e);
imsg_compose_event(parent_iev_broker, IMSG_CFG_AUTH_RESET, 0, 0, -1,
NULL, 0);
memset(&pi, 0, sizeof(pi));
pi.verbose = log_getverbose();
+ pi.workers = env->conf.workers;
pi.nlisteners = env->nlisteners;
pi.started = env->started;
struct acl *a;
struct bridge_def *b;
pid_t pid;
- int status;
+ int n, status;
/* closing the pipes tells the children to exit */
imsg_event_shutdown(parent_iev_control);
free(parent_iev_auth);
imsg_event_shutdown(parent_iev_snow);
free(parent_iev_snow);
- imsg_event_shutdown(parent_iev_socks);
+ imsg_event_shutdown(parent_iev_listen);
+ free(parent_iev_listen);
+ for (n = 0; n < e->conf.workers; n++)
+ imsg_event_shutdown(&parent_iev_socks[n]);
free(parent_iev_socks);
log_debug("waiting for children to terminate");
blob - 7dab33d363d26b173b9e0d1e43dc33514febec4ecba7e1cc34cae888a24bcbeb
blob + aa0076eb4971720889d62060520ab510e1b9ec70030b88de60bf07911957203d
--- mqttd/mqttd.conf.5
+++ mqttd/mqttd.conf.5
Publishes per second for this user, overriding
.Ic max rate .
.El
+.It Ic workers Ar number
+Run the specified number of sockets worker processes.
+The listen process hands each new connection to the worker with the
+fewest connections.
+The default is 2, the maximum 32.
+.Ic prefork
+is accepted as a deprecated alias.
.El
.Sh LISTENERS
A listener is declared with the
blob - 3ab7ab937a64b0e804f57ed389718f7efae00478bcfb678afbd79f1af0b2f382
blob + 3d800bb72daeaf70f6da6daadbc86ee090d11b05a477cb48f3ccf8dcbd0c6da5
--- mqttd/mqttd.h
+++ mqttd/mqttd.h
#define MQTTD_MAXPEER 64
#define MQTTD_MAXHASH 128
#define MQTTD_MAXIFACE 16
+#define MQTTD_MAX_WORKERS 32
+#define MQTTD_DEFAULT_WORKERS 2
#define MQTT_PORT 1883
#define MQTTS_PORT 8883
IMSG_CONN_ERROR,
IMSG_CONN_KEEPALIVE,
+ /* listen <-> sockets */
+ IMSG_CONN_ACCEPT,
+ IMSG_CONN_DONE,
+
/* broker <-> storage */
IMSG_DB_RECORD,
IMSG_DB_RESTORE_END,
/* global broker limits, fixed size so it travels in a single imsg */
struct mqttd_global {
+ uint16_t workers;
uint32_t max_clients;
uint32_t max_packet_size;
uint16_t max_inflight;
/*
* Bridges. The parent resolves the remote address and loads the CA;
- * the sockets process named by proc opens the connection on request and
- * the broker drives the MQTT client side of it.
+ * the listen process dials the remote on request and hands the
+ * connection to a sockets worker, which runs TLS, and the broker drives
+ * the MQTT client side of it. proc is MQTTD_PROC_SNOW for a sink, whose
+ * remote is the snowflake process instead.
*/
#define BRIDGE_IN 0x01
#define BRIDGE_OUT 0x02
};
/*
+ * listen -> sockets: a connected descriptor travels with this message.
+ * bridge_id is set for an outbound bridge the worker has to run TLS and
+ * MQTT over as a client; the PROXY header, if any, is already consumed
+ * and peer reflects it.
+ */
+struct conn_accept {
+ uint32_t id;
+ uint32_t listener_id;
+ uint32_t bridge_id;
+ char peer[MQTTD_MAXPEER];
+};
+
+/* sockets -> listen: the connection is gone, release its load */
+struct conn_done {
+ uint32_t id;
+ uint32_t listener_id;
+};
+
+/*
* Snapshot records. The broker serialises its state into a stream of
* typed records which the storage process writes out verbatim and plays
* back at startup; storage never interprets record contents.
struct ctl_parent_info {
int verbose;
+ uint16_t workers;
uint32_t nlisteners;
time_t started;
};
void proc_privdrop_dir(const char *, const char *);
int sockaddr_ntop(struct sockaddr_storage *, char *, size_t);
+/* listen.c */
+__dead void
+ listen_main(int, int, int, int *);
+int listen_bind(struct listener *);
+
/* sockets.c */
__dead void
sockets(int, int, int);
-int sockets_create_listener(struct listener *);
/* broker.c */
__dead void
- broker(int, int, int, int, int, int);
+ broker(int, int, int, int, int, int, int *);
/* storage.c */
__dead void
blob - 9f661dbe61297dd830d546740d9953f2ceeb132d28084dca8d2684872fd8d51a
blob + 66ec6f49d0fb8c02811cef9c217b2eb27fd965a6463b8f09021cade5b040bc79
--- mqttd/parse.y
+++ mqttd/parse.y
%token PACKET PASSWORD PATH PERSISTENCE PF PORT PREFORK PROTOCOLS PROXY
%token PUBLISH QOS QUEUED RATE REMOTE RETAIN ROLE SERVERNAME SESSION SIZE
%token SNOWFLAKE SPARKPLUG SUBSCRIBE SYS TABLE TIMEOUT TLS TOPIC USER VERSION
-%token WAREHOUSE WEBSOCKET WITHIN YES
+%token WAREHOUSE WEBSOCKET WITHIN WORKERS YES
%token <v.string> STRING
%token <v.number> NUMBER
| NO { $$ = 0; }
;
-main : PREFORK NUMBER {
- log_warnx("%s:%d: prefork is deprecated and ignored",
+main : WORKERS NUMBER {
+ if ($2 <= 0 || $2 > MQTTD_MAX_WORKERS) {
+ yyerror("invalid number of workers: %lld", $2);
+ YYERROR;
+ }
+ conf->conf.workers = $2;
+ }
+ | PREFORK NUMBER {
+ log_warnx("%s:%d: prefork is deprecated, use workers",
file->name, yylval.lineno);
+ if ($2 <= 0 || $2 > MQTTD_MAX_WORKERS) {
+ yyerror("invalid number of workers: %lld", $2);
+ YYERROR;
+ }
+ conf->conf.workers = $2;
}
| MAX CLIENTS NUMBER {
if ($3 < 0 || $3 > UINT32_MAX) {
{ "warehouse", WAREHOUSE },
{ "websocket", WEBSOCKET },
{ "within", WITHIN },
+ { "workers", WORKERS },
{ "yes", YES }
};
const struct keywords *p;
conf = xconf;
config_init(conf);
+ conf->conf.workers = MQTTD_DEFAULT_WORKERS;
conf->conf.max_clients = 0;
conf->conf.max_packet_size = MQTTD_DEFAULT_MAX_PACKET;
conf->conf.max_inflight = MQTTD_DEFAULT_MAX_INFLIGHT;
blob - a7c5423a0b6ac849c7d3913446cb437090553318e95caabd5eaabd85e1b3a023
blob + ae1924f7ff00a8f96fb2a9e875ab333f2925a00833f19b9d27f506bb673430e4
--- mqttd/sockets.c
+++ mqttd/sockets.c
*/
/*
- * The sockets processes own the network connections. They accept, run
- * TLS, cut the byte stream into MQTT control packets and relay those to
- * the broker; everything the broker sends back is written out verbatim.
+ * The sockets workers own the client connections. The listen process
+ * accepts and hands each descriptor over; a worker runs TLS, cuts the
+ * byte stream into MQTT control packets and relays those to the broker.
+ * Everything the broker sends back is written out verbatim. A worker
+ * never opens a socket itself.
*/
#include <sys/types.h>
#include <sys/uio.h>
#include <netinet/in.h>
-#include <arpa/inet.h>
-#include <sys/stat.h>
-#include <sys/un.h>
-
#include <errno.h>
#include <event.h>
#include <fcntl.h>
#include <imsg.h>
-#include <netdb.h>
#include <resolv.h>
#include <sha1.h>
#include <signal.h>
#include "mqttd.h"
#include "mqtt.h"
-#define FD_RESERVE 8
#define TLS_READ_SIZE 16384
-#define PROXY_V1_MAX 107
-#define PROXY_V2_MIN 16
-#define PROXY_PEEK (PROXY_V2_MIN + 216)
enum ws_state {
WS_NONE,
uint32_t id;
int fd;
struct listener *l;
+ uint32_t listener_id;
struct sbridge *bridge;
struct bufferevent *bev;
struct tls *tls_ctx;
void *);
static void sockets_dispatch_parent(int, short, void *);
-static void sockets_dispatch_control(int, short, void *);
+static void sockets_dispatch_listen(int, short, void *);
static void sockets_dispatch_broker(int, short, void *);
static void sockets_sighdlr(int, short, void *);
static __dead void
static int sockets_listener_launch(struct listener *);
static void sockets_listener_remove(struct listener *);
static void sockets_rlimit(void);
-static int sockets_accept_reserve(int, struct sockaddr *, socklen_t *);
-static void sockets_accept(int, short, void *);
-static void sockets_accept_paused(int, short, void *);
-static void sockets_show_listeners(struct imsg *);
+static void sockets_conn_accept(struct imsg *);
static struct conn
*conn_find(uint32_t);
const uint8_t *);
static void conn_begin(struct conn *);
static void conn_open_send(struct conn *);
-static void conn_proxy_read(int, short, void *);
-static int proxy_parse_v1(struct conn *, const uint8_t *, size_t);
-static int proxy_parse_v2(struct conn *, const uint8_t *, size_t);
static int ws_handshake(struct conn *);
static int ws_decode(struct conn *);
static void ws_send_frame(struct conn *, uint8_t, const uint8_t *,
*sbridge_find(uint32_t);
static void sockets_bridge_config(struct imsg *);
static void sockets_bridge_ca(struct imsg *);
-static void sockets_bridge_connect(struct imsg *);
-static void sockets_bridge_connected(int, short, void *);
static void sockets_bridge_failed(struct conn *);
static struct imsgev *socks_iev_parent;
-static struct imsgev *socks_iev_control;
+static struct imsgev *socks_iev_listen;
static struct imsgev *socks_iev_broker;
static struct conntree conns = RB_INITIALIZER(&conns);
/* connections we stopped reading from while the broker pipe is full */
static struct connq throttled_conns = TAILQ_HEAD_INITIALIZER(throttled_conns);
static struct sbridgelist sbridges = TAILQ_HEAD_INITIALIZER(sbridges);
-static uint32_t conn_next_id;
static struct event ev_sigint;
static struct event ev_sigterm;
}
__dead void
-sockets(int fd_parent, int fd_control, int fd_broker)
+sockets(int fd_parent, int fd_listen, int fd_broker)
{
log_procinit("sockets");
proc_privdrop("sockets");
fatal("%s: calloc", __func__);
imsg_event_init(socks_iev_parent, fd_parent, sockets_dispatch_parent);
- if ((socks_iev_control = calloc(1, sizeof(*socks_iev_control)))
- == NULL)
+ if ((socks_iev_listen = calloc(1, sizeof(*socks_iev_listen))) == NULL)
fatal("%s: calloc", __func__);
- imsg_event_init(socks_iev_control, fd_control,
- sockets_dispatch_control);
+ imsg_event_init(socks_iev_listen, fd_listen, sockets_dispatch_listen);
if ((socks_iev_broker = calloc(1, sizeof(*socks_iev_broker))) == NULL)
fatal("%s: calloc", __func__);
imsg_event_init(socks_iev_broker, fd_broker, sockets_dispatch_broker);
- if (pledge("stdio inet recvfd", NULL) == -1)
+ /* recvfd for the descriptors listen hands over; no socket calls */
+ if (pledge("stdio recvfd", NULL) == -1)
fatal("pledge");
event_dispatch();
tls_config_free(l->tls_config);
tls_free(l->tls_ctx);
}
- if (event_initialized(&l->ev))
- event_del(&l->ev);
- if (evtimer_initialized(&l->pause))
- evtimer_del(&l->pause);
TAILQ_REMOVE(&env->listeners, l, entry);
config_free_listener(l);
}
imsg_event_shutdown(socks_iev_parent);
free(socks_iev_parent);
- imsg_event_shutdown(socks_iev_control);
- free(socks_iev_control);
+ imsg_event_shutdown(socks_iev_listen);
+ free(socks_iev_listen);
imsg_event_shutdown(socks_iev_broker);
free(socks_iev_broker);
fatal("%s: setrlimit", __func__);
}
-int
-sockets_create_listener(struct listener *l)
-{
- struct sockaddr_storage *ss = &l->conf.ss;
- struct sockaddr_un *sun;
- int fd, on = 1;
-
- if (ss->ss_family == AF_UNIX) {
- sun = (struct sockaddr_un *)ss;
- if (unlink(sun->sun_path) == -1 && errno != ENOENT) {
- log_warn("%s: unlink %s", __func__, sun->sun_path);
- return (-1);
- }
- fd = socket(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, 0);
- if (fd == -1) {
- log_warn("%s: socket", __func__);
- return (-1);
- }
- if (bind(fd, (struct sockaddr *)sun, sizeof(*sun)) == -1) {
- log_warn("%s: bind %s", __func__, sun->sun_path);
- goto fail;
- }
- /* authentication still applies; let local users reach it */
- if (chmod(sun->sun_path, MQTTD_UNIX_MODE) == -1) {
- log_warn("%s: chmod %s", __func__, sun->sun_path);
- goto fail;
- }
- if (listen(fd, SOMAXCONN) == -1) {
- log_warn("%s: listen", __func__);
- goto fail;
- }
- l->fd = fd;
- return (0);
- }
-
- switch (ss->ss_family) {
- case AF_INET:
- ((struct sockaddr_in *)ss)->sin_port = l->conf.port;
- ((struct sockaddr_in *)ss)->sin_len = sizeof(struct sockaddr_in);
- break;
- case AF_INET6:
- ((struct sockaddr_in6 *)ss)->sin6_port = l->conf.port;
- ((struct sockaddr_in6 *)ss)->sin6_len =
- sizeof(struct sockaddr_in6);
- break;
- default:
- log_warnx("%s: unsupported address family", __func__);
- return (-1);
- }
-
- if ((fd = socket(ss->ss_family, SOCK_STREAM | SOCK_NONBLOCK, 0)) == -1) {
- log_warn("%s: socket", __func__);
- return (-1);
- }
- if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) {
- log_warn("%s: setsockopt SO_REUSEADDR", __func__);
- goto fail;
- }
- if (ss->ss_family == AF_INET6 &&
- setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &on, sizeof(on)) == -1) {
- log_warn("%s: setsockopt IPV6_V6ONLY", __func__);
- goto fail;
- }
- if (bind(fd, (struct sockaddr *)ss, ss->ss_len) == -1) {
- log_warn("%s: bind listener %s port %d", __func__, l->conf.name,
- ntohs(l->conf.port));
- goto fail;
- }
- if (listen(fd, SOMAXCONN) == -1) {
- log_warn("%s: listen", __func__);
- goto fail;
- }
-
- l->fd = fd;
- return (0);
-fail:
- close(fd);
- return (-1);
-}
-
+/*
+ * The TLS context for a listener; the descriptor itself lives in listen.
+ * Safe to call twice: a connection handed over early sets it up before
+ * the parent's start message arrives.
+ */
static int
sockets_listener_launch(struct listener *l)
{
- log_debug("%s: configuring listener %s[%u] (fd %d)", __func__,
- l->conf.name, l->conf.id, l->fd);
+ if (l->conf.tls && l->tls_ctx != NULL)
+ return (0);
+ log_debug("%s: configuring listener %s[%u]", __func__, l->conf.name,
+ l->conf.id);
+
if (listener_tls_init(l) == -1) {
log_warnx("%s: tls setup failed for listener %s", __func__,
l->conf.name);
return (-1);
}
-
- event_set(&l->ev, l->fd, EV_READ | EV_PERSIST, sockets_accept, l);
- if (event_add(&l->ev, NULL) == -1)
- fatal("%s: event_add", __func__);
- evtimer_set(&l->pause, sockets_accept_paused, l);
return (0);
}
if (c->l == l)
c->l = NULL;
- if (event_initialized(&l->ev))
- event_del(&l->ev);
- if (evtimer_initialized(&l->pause))
- evtimer_del(&l->pause);
if (l->conf.tls) {
if (l->tls_config != NULL)
tls_config_free(l->tls_config);
config_free_listener(l);
}
-/* listeners that arrive after startup need recvfd, so the pledge stays */
+/* TLS contexts for every listener the parent announced */
static void
sockets_launch(void)
{
memcpy(&verbose, imsg.data, sizeof(verbose));
log_setverbose(verbose);
break;
- case IMSG_CTL_SHOW_LISTENERS_REQUEST:
- sockets_show_listeners(&imsg);
- break;
default:
log_warnx("%s: unexpected imsg %d from parent",
__func__, imsg.hdr.type);
}
static void
-sockets_dispatch_control(int fd, short event, void *arg)
+sockets_dispatch_listen(int fd, short event, void *arg)
{
struct imsgev *iev = arg;
struct imsgbuf *ibuf = &iev->ibuf;
if ((n = imsgbuf_read(ibuf)) == -1)
fatal("%s: imsgbuf_read", __func__);
if (n == 0) {
- log_warnx("%s: lost control process", __func__);
+ log_warnx("%s: lost listen process", __func__);
sockets_shutdown();
}
}
if (event & EV_WRITE) {
if (imsgbuf_write(ibuf) == -1) {
if (errno == EPIPE) {
- log_warnx("%s: lost control process", __func__);
+ log_warnx("%s: lost listen process", __func__);
sockets_shutdown();
}
fatal("%s: imsgbuf_write", __func__);
}
}
- /* control only listens to us; its requests arrive via the parent */
for (;;) {
if ((n = imsgbuf_get(ibuf, &imsg)) == -1)
fatal("%s: imsgbuf_get", __func__);
if (n == 0)
break;
- log_warnx("%s: unexpected imsg %d from control", __func__,
- imsg.hdr.type);
+
+ switch (imsg.hdr.type) {
+ case IMSG_CONN_ACCEPT:
+ sockets_conn_accept(&imsg);
+ break;
+ default:
+ log_warnx("%s: unexpected imsg %d from listen",
+ __func__, imsg.hdr.type);
+ break;
+ }
imsg_free(&imsg);
}
imsg_event_add(iev);
else
conn_set_timer(c, (time_t)ka.keepalive * 3 / 2);
break;
- case IMSG_BRIDGE_CONNECT:
- sockets_bridge_connect(&imsg);
- break;
default:
log_warnx("%s: unexpected imsg %d from broker",
__func__, imsg.hdr.type);
imsg_event_add(iev);
}
+/*
+ * The listen process hands us a connected descriptor. Whatever happens
+ * to it from here, listen hears IMSG_CONN_DONE once so its counts stay
+ * right; a bridge that fails before it is open is reported to the broker
+ * so it can retry.
+ */
static void
-sockets_show_listeners(struct imsg *imsg)
+sockets_conn_accept(struct imsg *imsg)
{
- struct ctl_listener_info li;
- struct listener *l;
-
- TAILQ_FOREACH(l, &env->listeners, entry) {
- memset(&li, 0, sizeof(li));
- li.id = l->conf.id;
- strlcpy(li.name, l->conf.name, sizeof(li.name));
- sockaddr_ntop(&l->conf.ss, li.addr, sizeof(li.addr));
- li.port = l->conf.port;
- li.tls = l->conf.tls;
- li.websocket = l->conf.websocket;
- li.proxy = l->conf.proxy;
- li.fd = l->fd;
- li.nconns = l->nconns;
- if (imsg_compose_event(socks_iev_control,
- IMSG_CTL_SHOW_LISTENERS_DATA, imsg->hdr.peerid, 0, -1,
- &li, sizeof(li)) == -1)
- return;
- }
-
- imsg_compose_event(socks_iev_control, IMSG_CTL_SHOW_LISTENERS_END,
- imsg->hdr.peerid, 0, -1, NULL, 0);
-}
-
-/* keep a few descriptors free for the imsg pipes and libevent itself */
-static int
-sockets_accept_reserve(int sockfd, struct sockaddr *addr, socklen_t *addrlen)
-{
- if (getdtablecount() + FD_RESERVE >= getdtablesize()) {
- log_debug("%s: descriptor table nearly full", __func__);
- errno = EMFILE;
- return (-1);
- }
-
- return (accept4(sockfd, addr, addrlen, SOCK_NONBLOCK | SOCK_CLOEXEC));
-}
-
-static void
-sockets_accept_paused(int fd, short event, void *arg)
-{
- struct listener *l = arg;
-
- event_add(&l->ev, NULL);
-}
-
-static void
-sockets_accept(int fd, short event, void *arg)
-{
- struct listener *l = arg;
+ struct conn_accept ca;
+ struct conn_done cd;
struct conn *c;
- struct sockaddr_storage ss;
- struct timeval backoff = { 1, 0 };
- socklen_t len = sizeof(ss);
- int cfd;
+ struct sbridge *sb = NULL;
+ struct listener *l = NULL;
+ int fd;
- cfd = sockets_accept_reserve(fd, (struct sockaddr *)&ss, &len);
- if (cfd == -1) {
- switch (errno) {
- case EINTR:
- case EWOULDBLOCK:
- case ECONNABORTED:
- return;
- case EMFILE:
- case ENFILE:
- /* out of descriptors, give the clients a moment */
- event_del(&l->ev);
- evtimer_add(&l->pause, &backoff);
- return;
- default:
- log_warn("%s: accept", __func__);
- return;
+ IMSG_SIZE_CHECK(imsg, &ca);
+ memcpy(&ca, imsg->data, sizeof(ca));
+ ca.peer[sizeof(ca.peer) - 1] = '\0';
+ fd = imsg_get_fd(imsg);
+
+ if (fd == -1) {
+ log_warnx("%s: connection %u arrived without a descriptor",
+ __func__, ca.id);
+ goto fail;
+ }
+ if (ca.bridge_id != 0) {
+ if ((sb = sbridge_find(ca.bridge_id)) == NULL) {
+ log_warnx("%s: unknown bridge %u", __func__,
+ ca.bridge_id);
+ goto fail;
}
+ } else {
+ if ((l = config_listener_byid(env, ca.listener_id)) == NULL) {
+ /* removed by a reload while this one was in flight */
+ log_debug("%s: connection %u from %s: listener %u is "
+ "gone", __func__, ca.id, ca.peer, ca.listener_id);
+ goto fail;
+ }
+ /*
+ * A listener added on reload may reach us from listen before
+ * our own IMSG_CFG_LISTENER_START did; set TLS up now.
+ */
+ if (l->conf.tls && l->tls_ctx == NULL &&
+ sockets_listener_launch(l) == -1)
+ goto fail;
}
+ if (conn_find(ca.id) != NULL) {
+ log_warnx("%s: duplicate connection %u", __func__, ca.id);
+ goto fail;
+ }
if ((c = calloc(1, sizeof(*c))) == NULL) {
log_warn("%s: calloc", __func__);
- close(cfd);
- return;
+ goto fail;
}
-
- do {
- c->id = ++conn_next_id;
- } while (c->id == 0 || conn_find(c->id) != NULL);
- c->fd = cfd;
+ c->id = ca.id;
+ c->fd = fd;
c->l = l;
- if (ss.ss_family == AF_UNIX)
- strlcpy(c->peer, "unix", sizeof(c->peer));
- else
- sockaddr_ntop(&ss, c->peer, sizeof(c->peer));
+ c->listener_id = ca.listener_id;
+ c->bridge = sb;
+ strlcpy(c->peer, ca.peer, sizeof(c->peer));
evtimer_set(&c->timer, conn_timeout, c);
RB_INSERT(conntree, &conns, c);
- l->nconns++;
- log_debug("%s: connection %u from %s on listener %s", __func__, c->id,
- c->peer, l->conf.name);
+ log_debug("%s: connection %u %s %s", __func__, c->id,
+ sb != NULL ? "to" : "from", c->peer);
- /* the client has this long to finish PROXY, TLS, WS and CONNECT */
+ /* the peer has this long to finish TLS, WS and CONNECT */
conn_set_timer(c, MQTTD_CONNECT_TIMEOUT);
- if (l->conf.proxy) {
- event_set(&c->ev, cfd, EV_READ, conn_proxy_read, c);
- event_add(&c->ev, NULL);
+ if (sb == NULL) {
+ conn_begin(c);
return;
}
- conn_begin(c);
+ /* an outbound bridge: we are the TLS and MQTT client */
+ if (!sb->conf.tls) {
+ conn_start(c);
+ return;
+ }
+ if (sb->tls_config == NULL) {
+ if ((sb->tls_config = tls_config_new()) == NULL)
+ fatal("%s: tls_config_new", __func__);
+ if (sb->conf.tls_ca != NULL &&
+ tls_config_set_ca_mem(sb->tls_config, sb->conf.tls_ca,
+ sb->conf.tls_ca_len) == -1)
+ log_warnx("bridge %s: %s", sb->conf.name,
+ tls_config_error(sb->tls_config));
+ }
+ if ((c->tls_ctx = tls_client()) == NULL ||
+ tls_configure(c->tls_ctx, sb->tls_config) == -1 ||
+ tls_connect_socket(c->tls_ctx, fd, sb->conf.host) == -1) {
+ log_warnx("bridge %s: tls setup: %s", sb->conf.name,
+ c->tls_ctx != NULL ? tls_error(c->tls_ctx) : "no context");
+ conn_free(c, -1);
+ return;
+ }
+ event_set(&c->ev, fd, EV_READ | EV_WRITE, conn_tls_handshake, c);
+ event_add(&c->ev, NULL);
+ return;
+
+fail:
+ if (fd != -1)
+ close(fd);
+ if (ca.bridge_id != 0)
+ imsg_compose_event(socks_iev_broker, IMSG_BRIDGE_FAILED, 0, 0,
+ -1, &ca.bridge_id, sizeof(ca.bridge_id));
+ cd.id = ca.id;
+ cd.listener_id = ca.listener_id;
+ imsg_compose_event(socks_iev_listen, IMSG_CONN_DONE, 0, 0, -1, &cd,
+ sizeof(cd));
}
-/* PROXY header consumed if any; now TLS, then the MQTT or WS stream */
+/* TLS first when the listener wants it, then the MQTT or WS stream */
static void
conn_begin(struct conn *c)
{
conn_start(c);
}
-/*
- * The proxy sends its header in one write before any client byte, so
- * peek, work out the exact length, then consume exactly that much and
- * leave the stream untouched for TLS or MQTT.
- */
-static void
-conn_proxy_read(int fd, short event, void *arg)
-{
- struct conn *c = arg;
- uint8_t buf[PROXY_PEEK];
- ssize_t n;
- int used;
-
- n = recv(fd, buf, sizeof(buf), MSG_PEEK);
- if (n == -1) {
- if (errno == EAGAIN || errno == EINTR) {
- event_add(&c->ev, NULL);
- return;
- }
- conn_free(c, -1);
- return;
- }
- if (n == 0) {
- conn_free(c, -1);
- return;
- }
-
- if (n >= PROXY_V2_MIN &&
- memcmp(buf, "\x0d\x0a\x0d\x0a\x00\x0d\x0a\x51\x55\x49\x54\x0a",
- 12) == 0)
- used = proxy_parse_v2(c, buf, n);
- else if (n >= 6 && memcmp(buf, "PROXY ", 6) == 0)
- used = proxy_parse_v1(c, buf, n);
- else {
- log_warnx("%s: no PROXY header from %s", __func__, c->peer);
- conn_free(c, -1);
- return;
- }
-
- if (used == 0) {
- /* incomplete header, wait for the rest */
- if (n >= (ssize_t)sizeof(buf)) {
- conn_free(c, -1);
- return;
- }
- event_add(&c->ev, NULL);
- return;
- }
- if (used == -1) {
- log_warnx("%s: bad PROXY header from %s", __func__, c->peer);
- conn_free(c, -1);
- return;
- }
-
- /* the peeked bytes are still there; this read cannot block */
- if (recv(fd, buf, used, MSG_WAITALL) != used) {
- conn_free(c, -1);
- return;
- }
- conn_begin(c);
-}
-
-/* returns bytes consumed, 0 for more needed, -1 for malformed */
-static int
-proxy_parse_v1(struct conn *c, const uint8_t *buf, size_t n)
-{
- char line[PROXY_V1_MAX + 1], *p, *end, *fam, *src, *sport;
- const char *eol;
- size_t len;
-
- eol = memmem(buf, n < PROXY_V1_MAX ? n : PROXY_V1_MAX, "\r\n", 2);
- if (eol == NULL)
- return (n >= PROXY_V1_MAX ? -1 : 0);
- len = eol - (const char *)buf;
- memcpy(line, buf, len);
- line[len] = '\0';
-
- /* PROXY TCP4 src dst sport dport, or PROXY UNKNOWN ... */
- p = line + 6;
- end = p;
- fam = strsep(&end, " ");
- if (fam == NULL)
- return (-1);
- if (strcmp(fam, "UNKNOWN") == 0)
- return (len + 2);
- if (strcmp(fam, "TCP4") != 0 && strcmp(fam, "TCP6") != 0)
- return (-1);
- src = strsep(&end, " ");
- (void)strsep(&end, " ");
- sport = strsep(&end, " ");
- if (src == NULL || sport == NULL || end == NULL)
- return (-1);
- if (strchr(src, ':') != NULL)
- snprintf(c->peer, sizeof(c->peer), "[%s]:%s", src, sport);
- else
- snprintf(c->peer, sizeof(c->peer), "%s:%s", src, sport);
- return (len + 2);
-}
-
-static int
-proxy_parse_v2(struct conn *c, const uint8_t *buf, size_t n)
-{
- char host[INET6_ADDRSTRLEN];
- size_t len, total;
- uint8_t cmd, fam;
- uint16_t port;
-
- if ((buf[12] >> 4) != 2)
- return (-1);
- cmd = buf[12] & 0x0f;
- fam = buf[13];
- len = (size_t)buf[14] << 8 | buf[15];
- total = PROXY_V2_MIN + len;
- if (total > PROXY_PEEK)
- return (-1);
- if (n < total)
- return (0);
-
- /* LOCAL (health checks) keeps the proxy's own address */
- if (cmd != 0x01)
- return (total);
-
- switch (fam) {
- case 0x11:
- if (len < 12)
- return (-1);
- inet_ntop(AF_INET, buf + 16, host, sizeof(host));
- port = (uint16_t)buf[24] << 8 | buf[25];
- snprintf(c->peer, sizeof(c->peer), "%s:%u", host, port);
- break;
- case 0x21:
- if (len < 36)
- return (-1);
- inet_ntop(AF_INET6, buf + 16, host, sizeof(host));
- port = (uint16_t)buf[48] << 8 | buf[49];
- snprintf(c->peer, sizeof(c->peer), "[%s]:%u", host, port);
- break;
- default:
- break;
- }
- return (total);
-}
-
static struct conn *
conn_find(uint32_t id)
{
memset(&co, 0, sizeof(co));
co.id = c->id;
- co.listener_id = c->l != NULL ? c->l->conf.id : 0;
+ co.listener_id = c->listener_id;
co.bridge_id = c->bridge != NULL ? c->bridge->conf.id : 0;
co.tls = c->tls_ctx != NULL;
strlcpy(co.peer, c->peer, sizeof(co.peer));
conn_free(struct conn *c, int reason)
{
struct conn_close cc;
+ struct conn_done cd;
if (reason >= 0 && c->opened) {
cc.id = c->id;
if (c->bridge != NULL && !c->opened)
sockets_bridge_failed(c);
+ /* listen keeps the per-worker and per-listener counts */
+ cd.id = c->id;
+ cd.listener_id = c->listener_id;
+ imsg_compose_event(socks_iev_listen, IMSG_CONN_DONE, 0, 0, -1, &cd,
+ sizeof(cd));
+
log_debug("%s: closing connection %u %s %s", __func__, c->id,
c->bridge != NULL ? "to" : "from", c->peer);
RB_REMOVE(conntree, &conns, c);
if (c->throttled)
TAILQ_REMOVE(&throttled_conns, c, tentry);
- if (c->l != NULL)
- c->l->nconns--;
free(c);
}
&id, sizeof(id));
}
-/* the broker wants this bridge up: start a non-blocking connect */
-static void
-sockets_bridge_connect(struct imsg *imsg)
-{
- struct sbridge *sb;
- struct conn *c;
- struct sockaddr_storage ss;
- uint32_t id;
- int fd;
-
- IMSG_SIZE_CHECK(imsg, &id);
- memcpy(&id, imsg->data, sizeof(id));
- if ((sb = sbridge_find(id)) == NULL) {
- log_warnx("%s: unknown bridge %u", __func__, id);
- return;
- }
-
- if (sb->conf.tls && sb->tls_config == NULL) {
- if ((sb->tls_config = tls_config_new()) == NULL)
- fatal("%s: tls_config_new", __func__);
- if (sb->conf.tls_ca != NULL &&
- tls_config_set_ca_mem(sb->tls_config, sb->conf.tls_ca,
- sb->conf.tls_ca_len) == -1)
- log_warnx("bridge %s: %s", sb->conf.name,
- tls_config_error(sb->tls_config));
- }
-
- memcpy(&ss, &sb->conf.ss, sizeof(ss));
- switch (ss.ss_family) {
- case AF_INET:
- ((struct sockaddr_in *)&ss)->sin_port = sb->conf.port;
- break;
- case AF_INET6:
- ((struct sockaddr_in6 *)&ss)->sin6_port = sb->conf.port;
- break;
- }
-
- fd = socket(ss.ss_family, SOCK_STREAM | SOCK_NONBLOCK, 0);
- if (fd == -1) {
- log_warn("%s: socket", __func__);
- imsg_compose_event(socks_iev_broker, IMSG_BRIDGE_FAILED, 0, 0,
- -1, &id, sizeof(id));
- return;
- }
-
- if ((c = calloc(1, sizeof(*c))) == NULL) {
- log_warn("%s: calloc", __func__);
- close(fd);
- imsg_compose_event(socks_iev_broker, IMSG_BRIDGE_FAILED, 0, 0,
- -1, &id, sizeof(id));
- return;
- }
- do {
- c->id = ++conn_next_id;
- } while (c->id == 0 || conn_find(c->id) != NULL);
- c->fd = fd;
- c->bridge = sb;
- sockaddr_ntop(&ss, c->peer, sizeof(c->peer));
- evtimer_set(&c->timer, conn_timeout, c);
- RB_INSERT(conntree, &conns, c);
- conn_set_timer(c, MQTTD_CONNECT_TIMEOUT);
-
- if (connect(fd, (struct sockaddr *)&ss, ss.ss_len) == -1 &&
- errno != EINPROGRESS) {
- log_warn("bridge %s: connect to %s", sb->conf.name, c->peer);
- conn_free(c, -1);
- return;
- }
-
- event_set(&c->ev, fd, EV_WRITE, sockets_bridge_connected, c);
- event_add(&c->ev, NULL);
-}
-
-static void
-sockets_bridge_connected(int fd, short event, void *arg)
-{
- struct conn *c = arg;
- struct sbridge *sb = c->bridge;
- socklen_t len = sizeof(int);
- int err = 0;
-
- if (getsockopt(fd, SOL_SOCKET, SO_ERROR, &err, &len) == -1)
- err = errno;
- if (err != 0) {
- log_warnx("bridge %s: connect to %s: %s", sb->conf.name,
- c->peer, strerror(err));
- conn_free(c, -1);
- return;
- }
-
- if (!sb->conf.tls) {
- conn_start(c);
- return;
- }
-
- if ((c->tls_ctx = tls_client()) == NULL ||
- tls_configure(c->tls_ctx, sb->tls_config) == -1 ||
- tls_connect_socket(c->tls_ctx, fd, sb->conf.host) == -1) {
- log_warnx("bridge %s: tls setup: %s", sb->conf.name,
- c->tls_ctx != NULL ? tls_error(c->tls_ctx) : "no context");
- conn_free(c, -1);
- return;
- }
- event_set(&c->ev, fd, EV_READ | EV_WRITE, conn_tls_handshake, c);
- event_add(&c->ev, NULL);
-}
-
/* the CN of a verified client certificate, empty when there is none */
static void
conn_cert_user(struct tls *ctx, char *buf, size_t len)