commit fcb5ee1b57307679caae901031c6fe8a6316f3b4 from: Tracey Emery date: Tue Nov 10 16:23:32 2026 UTC thingsd: run a single sockets process Drop the prefork pool of sockets processes. thingsd serves a handful of clients, every thing packet was broadcast to all N processes, and the duplicate client name and max clients checks in parse.y only ever saw one process, as the XXX comment there admitted. With one sockets process those checks are global, control no longer counts replies from N processes, and the pipe arrays in main() go away. "prefork" in thingsd.conf is still parsed and logs a deprecation warning. The listener descriptor is handed to the sockets process instead of dup'd, so the parent stops holding a copy of every listener. This should help prepare mqttd changes. commit - c74c6691dc039dde5b1e331ff94da6e8ea2bbf41 commit + fcb5ee1b57307679caae901031c6fe8a6316f3b4 blob - 9e795c88908f44330fea214c5c8cc7b10b01a90d blob + e98be80dd08b8373855bfdb405e097f043120734 --- thingsd/config.c +++ thingsd/config.c @@ -143,7 +143,7 @@ int config_setsocks(struct thingsd *env, struct socket *sock) { struct socket_config u; - int fd, n, ret; + int ret; /* setup socket in priv process */ if (sockets_privinit(sock) == -1) @@ -152,39 +152,28 @@ config_setsocks(struct thingsd *env, struct socket *so memcpy(&u, &sock->conf, sizeof(u)); /* - * imsg closes the passed descriptor once sent, so every sockets - * process gets its own dup. The flush is synchronous because the - * parent pledges without sendfd once configuration is finished. + * imsg closes the passed descriptor once it is sent, which also + * keeps listener descriptors from piling up in the parent. The + * flush is synchronous because the parent pledges without sendfd + * once configuration is finished. */ - for (n = 0; n < env->prefork_socks; n++) { - if (sock->fd == -1) - fd = -1; - else if ((fd = dup(sock->fd)) == -1) - return (-1); - if (imsg_compose(&parent_iev_socks[n].ibuf, IMSG_CFG_SOCKS, 0, - 0, fd, &u, sizeof(u)) == -1) { - log_warn("%s: failed to compose IMSG_CFG_SOCKS imsg", - __func__); - return (-1); - } - do - ret = imsgbuf_flush(&parent_iev_socks[n].ibuf); - while (ret == -1 && errno == EAGAIN); - if (ret == -1) { - log_warn("%s: failed to flush IMSG_CFG_SOCKS imsg", - __func__); - return (-1); - } + if (imsg_compose(&parent_iev_socks->ibuf, IMSG_CFG_SOCKS, 0, 0, + sock->fd, &u, sizeof(u)) == -1) { + log_warn("%s: failed to compose IMSG_CFG_SOCKS imsg", __func__); + return (-1); } + sock->fd = -1; + do + ret = imsgbuf_flush(&parent_iev_socks->ibuf); + while (ret == -1 && errno == EAGAIN); + if (ret == -1) { + log_warn("%s: failed to flush IMSG_CFG_SOCKS imsg", __func__); + return (-1); + } /* Configure TLS if necessary. */ config_setsocks_tls(env, sock); - /* Close thing socket early to prevent fd exhaustion in thingsd. */ - if (sock->fd != -1) { - close(sock->fd); - sock->fd = -1; - } return (0); } @@ -196,13 +185,12 @@ config_settls(struct thingsd *env, struct socket *sock struct tls_config tls; struct iovec iov[2]; size_t c; - int n; if (data == NULL || len == 0) return (0); - log_debug("%s: sending tls %s for \"%s[%u]\" to sockets fd %d", - __func__, label, sock_conf->name, sock_conf->id, sock->fd); + log_debug("%s: sending tls %s for \"%s[%u]\"", __func__, label, + sock_conf->name, sock_conf->id); memset(&tls, 0, sizeof(tls)); tls.id = sock_conf->id; @@ -221,14 +209,11 @@ config_settls(struct thingsd *env, struct socket *sock iov[c].iov_base = data; iov[c++].iov_len = tls.tls_chunk_len; - for (n = 0; n < env->prefork_socks; n++) { - if (imsg_composev_event(&parent_iev_socks[n], - IMSG_CFG_TLS, 0, 0, -1, iov, c) == -1) { - log_warn("%s: failed to compose IMSG_CFG_TLS " - "imsg for `%s'", __func__, - sock_conf->name); - return (-1); - } + if (imsg_composev_event(parent_iev_socks, IMSG_CFG_TLS, 0, 0, + -1, iov, c) == -1) { + log_warn("%s: failed to compose IMSG_CFG_TLS imsg for " + "`%s'", __func__, sock_conf->name); + return (-1); } tls.tls_chunk_offset += tls.tls_chunk_len; blob - b787c8badc63e179b4331ace198c4de19d658f54 blob + 5da45e1aff89c489f673de3e847cb6e8ce89e65d --- thingsd/control.c +++ thingsd/control.c @@ -54,10 +54,6 @@ struct ctl_conn { #define CTL_CONN_NOTIFY 0x01 struct imsgev iev; struct sockpeercred peercred; - - /* sockets processes still to answer this conn's list request */ - int pending_sockets; - int pending_clients; }; TAILQ_HEAD(ctl_connlist, ctl_conn); @@ -86,10 +82,9 @@ __dead void control_shutdown(void); __dead void -control(int fd_parent, int *fd_socks) +control(int fd_parent, int fd_socks) { struct passwd *pw; - int n; log_procinit("control"); @@ -132,12 +127,11 @@ control(int fd_parent, int *fd_socks) imsg_event_init(control_iev_parent, fd_parent, control_dispatch_thingsd); - if ((control_iev_socks = calloc(thingsd_env->prefork_socks, - sizeof(*control_iev_socks))) == NULL) + if ((control_iev_socks = calloc(1, sizeof(*control_iev_socks))) + == NULL) fatal("%s: calloc", __func__); - for (n = 0; n < thingsd_env->prefork_socks; n++) - imsg_event_init(&control_iev_socks[n], fd_socks[n], - control_dispatch_sockets); + imsg_event_init(control_iev_socks, fd_socks, + control_dispatch_sockets); TAILQ_INIT(&ctl_conns); if (control_listen(&control_cs) == -1) @@ -173,7 +167,6 @@ __dead void control_shutdown(void) { struct ctl_conn *c; - int n; control_cleanup(&control_cs); @@ -189,11 +182,9 @@ control_shutdown(void) imsgbuf_clear(&control_iev_parent->ibuf); close(control_iev_parent->ibuf.fd); free(control_iev_parent); - for (n = 0; n < thingsd_env->prefork_socks; n++) { - event_del(&control_iev_socks[n].ev); - imsgbuf_clear(&control_iev_socks[n].ibuf); - close(control_iev_socks[n].ibuf.fd); - } + event_del(&control_iev_socks->ev); + imsgbuf_clear(&control_iev_socks->ibuf); + close(control_iev_socks->ibuf.fd); free(control_iev_socks); log_info("control exiting, pid %d", getpid()); @@ -245,17 +236,6 @@ control_dispatch_sockets(int fd, short event, void *ar break; } - /* - * Every sockets process answers the request, so the - * end marker is only passed on once all have replied. - */ - if (imsg.hdr.type == IMSG_GET_INFO_SOCKETS_END_DATA && - --c->pending_sockets > 0) - break; - if (imsg.hdr.type == IMSG_GET_INFO_CLIENTS_END_DATA && - --c->pending_clients > 0) - break; - imsg_compose_event(&c->iev, imsg.hdr.type, 0, 0, -1, imsg.data, IMSG_DATA_SIZE(&imsg)); break; @@ -648,11 +628,6 @@ control_dispatch_imsg(int fd, short event, void *arg) imsg.hdr.type == IMSG_GET_INFO_THINGS_REQUEST) imsg.hdr.type = IMSG_GET_INFO_THINGS_REQUEST_ROOT; - if (imsg.hdr.type == IMSG_GET_INFO_SOCKETS_REQUEST) - c->pending_sockets = thingsd_env->prefork_socks; - if (imsg.hdr.type == IMSG_GET_INFO_CLIENTS_REQUEST) - c->pending_clients = thingsd_env->prefork_socks; - if (imsg_forward_event(control_iev_parent, &imsg) == -1) { imsg_free(&imsg); blob - fca5f21b6415bca631faf17c607243db62428de0 blob + c0cf12b04a2cc1ea2c9f71a9c4fdfb70b98385dc --- thingsd/parse.y +++ thingsd/parse.y @@ -153,12 +153,8 @@ config : /* empty */ conf_main : bindopts1 | maxclients | PREFORK NUMBER { - if ($2 <= 0 || $2 > THINGSD_MAX_PREFORK) { - yyerror("invalid number of preforked " - "sockets: %lld", $2); - YYERROR; - } - thingsd->prefork_socks = $2; + log_warnx("%s:%d: prefork is deprecated and ignored", + file->name, yylval.lineno); } | thingretry ; @@ -533,11 +529,6 @@ subthings : THING '{' STRING optcomma STRING '}' { strlen($5)) != 0) goto done; - /* - * XXX: the duplicate name and max clients checks - * only see this sockets process, so with prefork - * greater than one they are per process limits. - */ TAILQ_FOREACH(tsock, thingsd_env->sockets, entry) { TAILQ_FOREACH(clt, tsock->clients, entry) { if (clt->subscribed && strcmp(clt->name, @@ -1334,7 +1325,6 @@ parse_config(const char *filename, struct thingsd *env thingsd = env; thingsd->conn_retry = CONN_RETRY; - thingsd->prefork_socks = SOCKS_NUMPROC; thingsd->run = 1; parsing_pkt = 0; blob - 0a2f08cf8ae42a3f3b9fd2484c6b17552ce2812f blob + 1812b491b65d47715a903dc2c74d1cb7003582e3 --- thingsd/thingsd.c +++ thingsd/thingsd.c @@ -179,7 +179,7 @@ thingsd_dispatch_control(int fd, short event, void *ar struct imsgbuf *ibuf = &iev->ibuf; struct imsg imsg; struct thing *thing; - int i, n, res = 0, verbose, exists; + int n, res = 0, verbose, exists; unsigned int v; char thing_name[THINGSD_MAXNAME]; char client_name[THINGSD_MAXNAME]; @@ -238,8 +238,7 @@ thingsd_dispatch_control(int fd, short event, void *ar /* FALLTHROUGH */ case IMSG_GET_INFO_CLIENTS_REQUEST: case IMSG_GET_INFO_SOCKETS_REQUEST: - for (i = 0; i < thingsd_env->prefork_socks; i++) - imsg_forward_event(&parent_iev_socks[i], &imsg); + imsg_forward_event(parent_iev_socks, &imsg); break; case IMSG_GET_INFO_THINGS_REQUEST: case IMSG_GET_INFO_THINGS_REQUEST_ROOT: @@ -252,8 +251,7 @@ thingsd_dispatch_control(int fd, short event, void *ar IMSG_SIZE_CHECK(&imsg, &verbose); memcpy(&verbose, imsg.data, sizeof(verbose)); log_setverbose(verbose); - for (i = 0; i < thingsd_env->prefork_socks; i++) - imsg_forward_event(&parent_iev_socks[i], &imsg); + imsg_forward_event(parent_iev_socks, &imsg); break; default: log_warnx("%s: unexpected imsg %d from control", @@ -358,11 +356,10 @@ main(int argc, char **argv) struct thingsd *env; struct timeval eb_timeout; const char *conffile = THINGSD_CONF; - int ch, i, n; + int ch; int pipe_control[2]; - int pipe_socks[THINGSD_MAX_PREFORK][2]; - int pipe_control_socks[THINGSD_MAX_PREFORK][2]; - int fd_socks[THINGSD_MAX_PREFORK]; + int pipe_socks[2]; + int pipe_control_socks[2]; if ((env = calloc(1, sizeof(*env))) == NULL) fatal("%s: calloc", __func__); @@ -437,58 +434,44 @@ main(int argc, char **argv) * together. Each child closes the ends it does not own. */ if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, PF_UNSPEC, - pipe_control) == -1) + pipe_control) == -1 || + socketpair(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, PF_UNSPEC, + pipe_socks) == -1 || + socketpair(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, PF_UNSPEC, + pipe_control_socks) == -1) fatal("%s: socketpair", __func__); - for (i = 0; i < env->prefork_socks; i++) { - if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, PF_UNSPEC, - pipe_socks[i]) == -1) - fatal("%s: socketpair", __func__); - if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, PF_UNSPEC, - pipe_control_socks[i]) == -1) - fatal("%s: socketpair", __func__); - } + /* control: parent[1], control_socks[0] */ switch (fork()) { case -1: fatal("%s: fork", __func__); case 0: close(pipe_control[0]); - for (i = 0; i < env->prefork_socks; i++) { - close(pipe_socks[i][0]); - close(pipe_socks[i][1]); - close(pipe_control_socks[i][1]); - fd_socks[i] = pipe_control_socks[i][0]; - } - control(pipe_control[1], fd_socks); + close(pipe_socks[0]); + close(pipe_socks[1]); + close(pipe_control_socks[1]); + control(pipe_control[1], pipe_control_socks[0]); /* NOTREACHED */ } - for (n = 0; n < env->prefork_socks; n++) { - switch (fork()) { - case -1: - fatal("%s: fork", __func__); - case 0: - close(pipe_control[0]); - close(pipe_control[1]); - for (i = 0; i < env->prefork_socks; i++) { - close(pipe_socks[i][0]); - close(pipe_control_socks[i][0]); - if (i == n) - continue; - close(pipe_socks[i][1]); - close(pipe_control_socks[i][1]); - } - sockets(pipe_socks[n][1], pipe_control_socks[n][1]); - /* NOTREACHED */ - } + /* sockets: socks[1], control_socks[1] */ + switch (fork()) { + case -1: + fatal("%s: fork", __func__); + case 0: + close(pipe_control[0]); + close(pipe_control[1]); + close(pipe_socks[0]); + close(pipe_control_socks[0]); + sockets(pipe_socks[1], pipe_control_socks[1]); + /* NOTREACHED */ } + /* parent: the [0] end of every pipe to a child */ close(pipe_control[1]); - for (i = 0; i < env->prefork_socks; i++) { - close(pipe_socks[i][1]); - close(pipe_control_socks[i][0]); - close(pipe_control_socks[i][1]); - } + close(pipe_socks[1]); + close(pipe_control_socks[0]); + close(pipe_control_socks[1]); env->thingsd_eb = event_init(); @@ -510,12 +493,10 @@ main(int argc, char **argv) imsg_event_init(parent_iev_control, pipe_control[0], thingsd_dispatch_control); - if ((parent_iev_socks = calloc(env->prefork_socks, - sizeof(*parent_iev_socks))) == NULL) + if ((parent_iev_socks = calloc(1, sizeof(*parent_iev_socks))) == NULL) fatal("%s: calloc", __func__); - for (i = 0; i < env->prefork_socks; i++) - imsg_event_init(&parent_iev_socks[i], pipe_socks[i][0], - thingsd_dispatch_sockets); + imsg_event_init(parent_iev_socks, pipe_socks[0], + thingsd_dispatch_sockets); if (thingsd_configure(env) == -1) fatalx("configuration failed"); @@ -551,9 +532,8 @@ thingsd_configure(struct thingsd *env) { struct thing *thing; struct socket *sock; - int n; - env->socks_reload = env->prefork_socks; + env->socks_reload = 1; /* setup our things, which run privileged */ TAILQ_FOREACH(thing, env->things, entry) @@ -567,9 +547,7 @@ thingsd_configure(struct thingsd *env) imsg_compose_event(parent_iev_control, IMSG_CFG_DONE, 0, 0, -1, NULL, 0); - for (n = 0; n < env->prefork_socks; n++) - imsg_compose_event(&parent_iev_socks[n], IMSG_CFG_DONE, 0, 0, - -1, NULL, 0); + imsg_compose_event(parent_iev_socks, IMSG_CFG_DONE, 0, 0, -1, NULL, 0); return (0); } @@ -577,8 +555,6 @@ thingsd_configure(struct thingsd *env) void thingsd_configure_done(struct thingsd *env) { - int n; - if (env->socks_reload == 0) { log_warnx("%s: configuration already finished", __func__); return; @@ -588,9 +564,8 @@ thingsd_configure_done(struct thingsd *env) if (env->socks_reload == 0) { imsg_compose_event(parent_iev_control, IMSG_CTL_START, 0, 0, -1, NULL, 0); - for (n = 0; n < env->prefork_socks; n++) - imsg_compose_event(&parent_iev_socks[n], - IMSG_CTL_START, 0, 0, -1, NULL, 0); + imsg_compose_event(parent_iev_socks, IMSG_CTL_START, 0, 0, -1, + NULL, 0); } } @@ -598,17 +573,15 @@ __dead void thingsd_shutdown(struct thingsd *env) { pid_t pid; - int n, status; + int status; /* closing the pipes tells the children to exit */ event_del(&parent_iev_control->ev); imsgbuf_clear(&parent_iev_control->ibuf); close(parent_iev_control->ibuf.fd); - for (n = 0; n < env->prefork_socks; n++) { - event_del(&parent_iev_socks[n].ev); - imsgbuf_clear(&parent_iev_socks[n].ibuf); - close(parent_iev_socks[n].ibuf.fd); - } + event_del(&parent_iev_socks->ev); + imsgbuf_clear(&parent_iev_socks->ibuf); + close(parent_iev_socks->ibuf.fd); free(parent_iev_control); free(parent_iev_socks); @@ -1101,19 +1074,15 @@ thingsd_write_to_socks(struct packages *packages) struct package *package, *tpkg, p; struct packet_client *packet_client; struct thing *thing = NULL; - int n; TAILQ_FOREACH_SAFE(package, packages, entry, tpkg) { memcpy(&p, package, sizeof(p)); - for (n = 0; n < thingsd_env->prefork_socks; n++) { - if (imsg_compose_event(&parent_iev_socks[n], - IMSG_DIST_THING_PACKAGE, 0, 0, -1, &p, - sizeof(p)) == -1) { - log_warn("%s: failed to compose " - "IMSG_DIST_THING_PACKAGE imsg", __func__); - return; - } + if (imsg_compose_event(parent_iev_socks, + IMSG_DIST_THING_PACKAGE, 0, 0, -1, &p, sizeof(p)) == -1) { + log_warn("%s: failed to compose " + "IMSG_DIST_THING_PACKAGE imsg", __func__); + return; } TAILQ_FOREACH(thing, thingsd_env->things, entry) { blob - 57ccb9e6b192312cb14e5de02adaec541ced1636 blob + 2451fae6177834fa3d19fe92a774055bf5779ca9 --- thingsd/thingsd.conf.5 +++ thingsd/thingsd.conf.5 @@ -67,11 +67,6 @@ Range is 30-600 seconds. .It Ic max clients Ar number Set the default maximum number of clients allowed to connect to each .Xr thingsd 8 thing. -.It Ic prefork Ar number -Run the specified number of things processes. -This increases the performance and prevents delays when connecting to a thing. -.Xr thingsd 8 -runs 3 things processes by default. .El .Sh THINGS things represent a serial, TCP/IP, or UDP listener thing for @@ -278,8 +273,6 @@ This configuration file would create a serial thing an # Global Options # -prefork 5 - max clients 10 max suscriptions 5 blob - c23ff1460f49a68f37e772087bca8431ab54536d blob + 5190432795a1b3d63604b7ee9b17399b8a11ef2b --- thingsd/thingsd.h +++ thingsd/thingsd.h @@ -57,12 +57,8 @@ #define THINGSD_MAXPORT 6 #define THINGSD_MAXIFACE 16 -#define SOCKS_NUMPROC 3 - #define PKT_BUFF 2048 -#define THINGSD_MAX_PREFORK 32 - #define MAX_IMSG_DATA_SIZE (MAX_IMSGSIZE - IMSG_HEADER_SIZE) #define IMSG_DATA_SIZE(imsg) ((imsg)->hdr.len - IMSG_HEADER_SIZE) #define IMSG_SIZE_CHECK(imsg, p) do { \ @@ -388,7 +384,6 @@ struct thingsd { size_t max_clients; size_t conn_retry; - uint16_t prefork_socks; int run; struct deadthinglist *dead_things; @@ -407,7 +402,7 @@ struct thingsd { extern struct thingsd *thingsd_env; extern int thing_id; -/* parent process pipes, indexed by prefork instance for sockets */ +/* parent process pipes to the control and sockets processes */ extern struct imsgev *parent_iev_control; extern struct imsgev *parent_iev_socks; @@ -446,7 +441,7 @@ void imsg_get_string(struct imsg *, char *, size_t); /* control.c */ __dead void - control(int, int *); + control(int, int); /* config.c */ int config_init(struct thingsd *);