From 0d6824d2853dd50bdb882429c3528f6e0fd8d4de Mon Sep 17 00:00:00 2001 From: Rory& Date: Thu, 28 May 2026 10:38:39 +0200 Subject: [PATCH 01/11] Process lifecycle scripts --- package-lock.json | 16 ++++- src/api/Server.ts | 13 ++-- src/api/routes/stop.ts | 5 +- src/bundle/Server.ts | 20 +++--- src/cdn/Server.ts | 11 +++- src/gateway/Server.ts | 34 ++++++---- src/gateway/events/Close.ts | 60 +++++++++--------- src/gateway/events/Connection.ts | 41 +++++++++--- src/util/util/Database.ts | 7 ++- src/util/util/ProcessLifecycle.ts | 62 +++++++++++++++++++ .../ipc/listener/RabbitMqSingleListener.ts | 26 ++++++-- .../util/ipc/listener/UnixSocketListener.ts | 43 +++++++++++-- .../util/ipc/writer/RabbitMqSingleWriter.ts | 6 +- src/util/util/ipc/writer/UnixSocketWriter.ts | 24 +++++-- src/webrtc/Server.ts | 24 ++++--- 15 files changed, 294 insertions(+), 98 deletions(-) create mode 100644 src/util/util/ProcessLifecycle.ts diff --git a/package-lock.json b/package-lock.json index 0cd867fdbe..05ebdfacd4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -86,7 +86,8 @@ "pretty-quick": "^4.2.2", "ts-node": "^10.9.2", "typescript": "^6.0.3", - "typescript-json-schema": "^0.67.4" + "typescript-json-schema": "^0.67.4", + "why-is-node-running": "^3.2.2" }, "optionalDependencies": { "@sendgrid/mail": "^8.1.6", @@ -7387,6 +7388,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/why-is-node-running": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-3.2.2.tgz", + "integrity": "sha512-NKUzAelcoCXhXL4dJzKIwXeR8iEVqsA0Lq6Vnd0UXvgaKbzVo4ZTHROF2Jidrv+SgxOQ03fMinnNhzZATxOD3A==", + "dev": true, + "license": "MIT", + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=20.11" + } + }, "node_modules/word-wrap": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", diff --git a/src/api/Server.ts b/src/api/Server.ts index 1d9fe2ab22..f9f1c708f6 100644 --- a/src/api/Server.ts +++ b/src/api/Server.ts @@ -16,15 +16,17 @@ along with this program. If not, see . */ -import { Config, ConnectionConfig, ConnectionLoader, Email, JSONReplacer, WebAuthn, initDatabase, initEvent, registerRoutes, getDatabase, getRevInfoOrFail } from "@spacebar/util"; -import { Authentication, CORS, ImageProxy, BodyParser, ErrorHandler, initRateLimits, initTranslation } from "./middlewares"; +import path from "node:path"; import { Request, Response, Router } from "express"; -import { Server, ServerOptions } from "lambert-server"; import morgan from "morgan"; -import path from "node:path"; +import { Server, ServerOptions } from "lambert-server"; import { red } from "picocolors"; +import { Config, ConnectionConfig, ConnectionLoader, Email, JSONReplacer, WebAuthn, initDatabase, initEvent, registerRoutes, getDatabase, getRevInfoOrFail } from "@spacebar/util"; +import { Authentication, CORS, ImageProxy, BodyParser, ErrorHandler, initRateLimits, initTranslation } from "./middlewares"; import { initInstance } from "./util/handlers/Instance"; import { route } from "./util"; +import { ProcessLifecycle } from "../util/util/ProcessLifecycle"; +import { Monitoring } from "../util/monitoring/Monitoring"; const ASSETS_FOLDER = path.join(__dirname, "..", "..", "assets"); const PUBLIC_ASSETS_FOLDER = path.join(ASSETS_FOLDER, "public"); @@ -50,6 +52,8 @@ export class SpacebarServer extends Server { } async start() { + await Monitoring.init(); + Monitoring.attach(this.app); await initDatabase(); await Config.init(); await initEvent(); @@ -196,6 +200,7 @@ export class SpacebarServer extends Server { if (logRequests) console.log(red(`Warning: Request logging is enabled! This will spam your console!\nTo disable this, unset the 'LOG_REQUESTS' environment variable!`)); + await ProcessLifecycle.Ready(); return super.start(); } } diff --git a/src/api/routes/stop.ts b/src/api/routes/stop.ts index 4f86973508..b6ecd04ce5 100644 --- a/src/api/routes/stop.ts +++ b/src/api/routes/stop.ts @@ -18,6 +18,7 @@ import { route } from "@spacebar/api"; import { Request, Response, Router } from "express"; +import { ProcessLifecycle } from "../../util/util/ProcessLifecycle"; const router: Router = Router({ mergeParams: true }); @@ -35,7 +36,9 @@ router.post( (req: Request, res: Response) => { console.log(`/stop was called by ${req.user_id} at ${new Date()}`); res.sendStatus(200); - process.kill(process.pid, "SIGTERM"); + ProcessLifecycle.Shutdown().catch((e) => { + console.error("Failed to shut down:", e); + }); }, ); diff --git a/src/bundle/Server.ts b/src/bundle/Server.ts index 91c3dcf423..c396d10006 100644 --- a/src/bundle/Server.ts +++ b/src/bundle/Server.ts @@ -16,21 +16,19 @@ along with this program. If not, see . */ -import morgan from "morgan"; - -process.on("unhandledRejection", console.error); -process.on("uncaughtException", console.error); - import http from "node:http"; +import fs from "node:fs"; +import cluster from "node:cluster"; +import morgan from "morgan"; +import express from "express"; +import { green, bold } from "picocolors"; import * as Api from "@spacebar/api"; import * as Gateway from "@spacebar/gateway"; import * as Webrtc from "@spacebar/webrtc"; import { CDNServer } from "@spacebar/cdn"; -import express from "express"; -import { green, bold } from "picocolors"; import { Config, initDatabase } from "@spacebar/util"; -import fs from "node:fs"; -import cluster from "node:cluster"; +import { ProcessLifecycle } from "../util/util/ProcessLifecycle"; +import { Monitoring } from "../util/monitoring/Monitoring"; const app = express(); const server = http.createServer(); @@ -48,8 +46,7 @@ const webrtc = new Webrtc.Server({ production, }); -process.on("SIGTERM", async () => { - console.log("Shutting down due to SIGTERM"); +ProcessLifecycle.eventEmitter.on("stopping", async () => { await gateway.stop(); await cdn.stop(); await api.stop(); @@ -58,6 +55,7 @@ process.on("SIGTERM", async () => { }); async function main() { + await Monitoring.init(); await initDatabase(); await Config.init(); diff --git a/src/cdn/Server.ts b/src/cdn/Server.ts index 8260407ec3..8964161dcc 100644 --- a/src/cdn/Server.ts +++ b/src/cdn/Server.ts @@ -16,13 +16,15 @@ along with this program. If not, see . */ +import path from "node:path"; +import morgan from "morgan"; import { Server, ServerOptions } from "lambert-server"; import { Attachment, Config, initDatabase, registerRoutes } from "@spacebar/util"; import { CORS, BodyParser } from "@spacebar/api"; -import path from "node:path"; import guildProfilesRoute from "./routes/guild-profiles"; -import morgan from "morgan"; import { storage } from "./util"; +import { ProcessLifecycle } from "../util/util/ProcessLifecycle"; +import { Monitoring } from "../util/monitoring/Monitoring"; export type CDNServerOptions = ServerOptions; @@ -34,6 +36,8 @@ export class CDNServer extends Server { } async start() { + await Monitoring.init(); + Monitoring.attach(this.app); await initDatabase(); await Config.init(); @@ -71,6 +75,7 @@ export class CDNServer extends Server { this.app.use("/guilds/:guild_id/users/:user_id/banners", guildProfilesRoute); if (process.env.LOG_ROUTES !== "false") console.log("[Server] Route /guilds/:guild_id/users/:user_id/banners registered"); + await ProcessLifecycle.Ready(); return super.start(); } @@ -89,6 +94,8 @@ export class CDNServer extends Server { } async stop() { + await ProcessLifecycle.Shutdown(); + await ProcessLifecycle.Finalize(); return super.stop(); } } diff --git a/src/gateway/Server.ts b/src/gateway/Server.ts index 632fbc5295..430419752e 100644 --- a/src/gateway/Server.ts +++ b/src/gateway/Server.ts @@ -16,21 +16,22 @@ along with this program. If not, see . */ -import dotenv from "dotenv"; -dotenv.config({ quiet: true }); -import { checkToken, closeDatabase, Config, initDatabase, initEvent, Rights } from "@spacebar/util"; +import http from "node:http"; +import { setInterval } from "node:timers"; import ws from "ws"; +import { checkToken, Config, initDatabase, initEvent, Rights } from "@spacebar/util"; +import { randomString } from "@spacebar/api"; // TODO: move to util import { Connection, openConnections } from "./events/Connection"; -import http from "node:http"; import { cleanupOnStartup } from "./util"; -import { randomString } from "@spacebar/api"; -import { setInterval } from "node:timers"; +import { ProcessLifecycle } from "../util/util/ProcessLifecycle"; +import { Monitoring } from "../util/monitoring/Monitoring"; export class Server { public ws: ws.Server; public port: number; public server: http.Server; public production: boolean; + private monitoringLoop: NodeJS.Timeout; constructor({ port, server, production }: { port: number; server?: http.Server; production?: boolean }) { this.port = port; @@ -42,7 +43,7 @@ export class Server { const eluP = [1, 5, 15].map(() => performance.eventLoopUtilization()); const cpu = [1, 5, 15].map(() => process.cpuUsage()); let sec = 0; - setInterval(() => { + const monitoringLoop = setInterval(() => { sec += 1; // for some reason this behaves differently from cpuUsage, so we need an absolute reference as "previous" const eluC = performance.eventLoopUtilization(); @@ -67,7 +68,9 @@ export class Server { res.setHeader("Set-Cookie", `__sb_sessid=${randomString(32)}; Secure; HttpOnly; SameSite=None; Path=/`); } const requestUrl = new URL(`http://${req.headers.host}${req.url}`); - if (requestUrl.pathname === "/_spacebar/gateway/admin/introspect") { + if (requestUrl.pathname === "/metrics") { + return await Monitoring.handleRawRequest(req, res); + } else if (requestUrl.pathname === "/_spacebar/gateway/admin/introspect") { if (!req.headers.authorization) { return res.writeHead(401).end("Unauthorized"); } else { @@ -150,6 +153,8 @@ export class Server { res.writeHead(200).end("Online"); }); + + ProcessLifecycle.eventEmitter.on("stopping", () => clearTimeout(monitoringLoop)); } this.server.on("upgrade", (request, socket, head) => { @@ -167,6 +172,7 @@ export class Server { } async start(): Promise { + await Monitoring.init(); await initDatabase(); await Config.init(); await initEvent(); @@ -177,14 +183,16 @@ export class Server { this.server.listen(this.port); console.log(`[Gateway] online on 0.0.0.0:${this.port}`); } + + await ProcessLifecycle.Ready(); } async stop() { + await ProcessLifecycle.Shutdown(); + clearInterval(this.monitoringLoop); this.ws.clients.forEach((x) => x.close()); - this.ws.close(() => { - this.server.close(() => { - closeDatabase(); - }); - }); + this.ws.close(); + this.server.close(); + await ProcessLifecycle.Finalize(); } } diff --git a/src/gateway/events/Close.ts b/src/gateway/events/Close.ts index 594149057c..6a3c6cb324 100644 --- a/src/gateway/events/Close.ts +++ b/src/gateway/events/Close.ts @@ -19,6 +19,7 @@ import { WebSocket } from "@spacebar/gateway"; import { emitEvent, Member, PresenceUpdateEvent, Session, SessionsReplace, User, VoiceState, VoiceStateUpdateEvent, distributePresenceUpdate } from "@spacebar/util"; import { randomString } from "@spacebar/api"; +import { ProcessLifecycle } from "../../util/util/ProcessLifecycle"; export async function Close(this: WebSocket, code: number, reason: Buffer) { console.log("[WebSocket] closed", code, reason.toString()); @@ -32,36 +33,37 @@ export async function Close(this: WebSocket, code: number, reason: Buffer) { const authSessionId = this.session?.session_id; const closedAt = Date.now(); - setTimeout(async () => { - console.log("Handling presence update after disconnect"); - try { - if (authSessionId && this.user_id) { - const s = await Session.findOne({ - where: { user_id: this.user_id, session_id: authSessionId }, - }); - if (s && (s.last_seen?.getTime() ?? 0) <= closedAt) { - console.log("... updating session"); - await Session.update({ user_id: this.user_id, session_id: authSessionId }, { status: "offline", activities: [], client_status: {} }); - this.session = await Session.findOneOrFail({ where: { session_id: this.session_id } }); - console.log("... distributing PRESENCE_UPDATE"); - await distributePresenceUpdate(this.user_id, { - event: "PRESENCE_UPDATE", - data: { - user: (await User.findOneOrFail({ where: { id: this.user_id } })).toPublicUser(), - status: this.session!.getPublicStatus(), - client_status: this.session!.client_status, - activities: this.session!.activities, - }, - origin: "GATEWAY_CLOSE", - transaction_id: `IDENT_${this.user_id}_${randomString()}`, - } satisfies PresenceUpdateEvent); - console.log("... done!"); - } else console.log("... Discarding presence update as the session reactivated"); + if (!(ProcessLifecycle.state === "stopping" || ProcessLifecycle.state === "stopped")) + setTimeout(async () => { + console.log("Handling presence update after disconnect"); + try { + if (authSessionId && this.user_id) { + const s = await Session.findOne({ + where: { user_id: this.user_id, session_id: authSessionId }, + }); + if (s && (s.last_seen?.getTime() ?? 0) <= closedAt) { + console.log("... updating session"); + await Session.update({ user_id: this.user_id, session_id: authSessionId }, { status: "offline", activities: [], client_status: {} }); + this.session = await Session.findOneOrFail({ where: { session_id: this.session_id } }); + console.log("... distributing PRESENCE_UPDATE"); + await distributePresenceUpdate(this.user_id, { + event: "PRESENCE_UPDATE", + data: { + user: (await User.findOneOrFail({ where: { id: this.user_id } })).toPublicUser(), + status: this.session!.getPublicStatus(), + client_status: this.session!.client_status, + activities: this.session!.activities, + }, + origin: "GATEWAY_CLOSE", + transaction_id: `IDENT_${this.user_id}_${randomString()}`, + } satisfies PresenceUpdateEvent); + console.log("... done!"); + } else console.log("... Discarding presence update as the session reactivated"); + } + } catch (e) { + console.error("[WebSocket] Close session cleanup failed", code, e); } - } catch (e) { - console.error("[WebSocket] Close session cleanup failed", code, e); - } - }, 10_000); + }, 10_000); const voiceState = await VoiceState.findOne({ where: { user_id: this.user_id }, diff --git a/src/gateway/events/Connection.ts b/src/gateway/events/Connection.ts index 6f67c0c597..8daa5998f4 100644 --- a/src/gateway/events/Connection.ts +++ b/src/gateway/events/Connection.ts @@ -29,6 +29,9 @@ import { Deflate, Inflate } from "fast-zlib"; import { URL } from "node:url"; import { Config } from "@spacebar/util"; import { Decoder, Encoder } from "@toondepauw/node-zstd"; +import { ProcessLifecycle } from "../../util/util/ProcessLifecycle"; +import { Monitoring } from "../../util/monitoring/Monitoring"; +import { Gauge } from "prom-client"; // TODO: check rate limit // TODO: specify rate limit in config @@ -36,23 +39,43 @@ import { Decoder, Encoder } from "@toondepauw/node-zstd"; export const openConnections: WebSocket[] = []; +const openConnectionCount = Monitoring.attachMetric( + "spacebar_gateway_open_connection_count", + new Gauge({ + name: "spacebar_gateway_open_connection_count", + help: "The total number of HTTP requests received", + }), +); + export async function Connection(this: WS.Server, socket: WebSocket, request: IncomingMessage) { openConnections.push(socket); + openConnectionCount.set(openConnections.length); socket.on("close", () => { const index = openConnections.indexOf(socket); if (index !== -1) openConnections.splice(index, 1); + openConnectionCount.set(openConnections.length); }); - for (const sig of ["SIGINT", "SIGTERM", "SIGQUIT"] as const) { - process.on(sig, async () => { - await Send(socket, { - op: OPCODES.Reconnect, - s: socket.sequence++, - d: Math.round(Math.random() * 5000), - }); - socket.close(1000); + const onShutdown = async () => { + await Send(socket, { + op: OPCODES.Reconnect, + s: socket.sequence++, + d: Math.round(Math.random() * 5000), }); - } + + const closeListeners = socket.listeners("close"); + for (const listener of closeListeners) { + socket.off("close", listener); + // noinspection JSVoidFunctionReturnValueUsed - awaiting results + const res = listener.call(socket, 1000, 0) as void | Promise; + if (res) await res; + } + + socket.close(1000); + }; + + if (ProcessLifecycle.state == "stopping" || ProcessLifecycle.state == "stopped") return await onShutdown(); + ProcessLifecycle.eventEmitter.on("stopping", onShutdown); const forwardedFor = Config.get().security.forwardedFor; const ipAddress = forwardedFor ? (request.headers[forwardedFor.toLowerCase()] as string) : request.socket.remoteAddress; diff --git a/src/util/util/Database.ts b/src/util/util/Database.ts index 004fd3f5e1..ecbb7110c1 100644 --- a/src/util/util/Database.ts +++ b/src/util/util/Database.ts @@ -23,6 +23,7 @@ import { DataSource } from "typeorm"; // noinspection ES6PreferShortImport import { ConfigEntity } from "../entities/Config"; import fs from "node:fs"; +import { ProcessLifecycle } from "./ProcessLifecycle"; // UUID extension option is only supported with postgres // We want to generate all id's with Snowflakes that's why we have our own BaseEntity class @@ -127,11 +128,13 @@ export async function initDatabase(): Promise { } } - console.log(`[Database] ${green("Connected")}`); + ProcessLifecycle.eventEmitter.on("stopped", async () => await closeDatabase()); + console.log(`[Database] ${green("Connected")}`); return dbConnection; } export async function closeDatabase() { - await dbConnection?.destroy(); + if (DataSourceOptions.isInitialized) await DataSourceOptions.destroy(); + if (dbConnection?.isInitialized) await dbConnection?.destroy(); } diff --git a/src/util/util/ProcessLifecycle.ts b/src/util/util/ProcessLifecycle.ts new file mode 100644 index 0000000000..6b4a6d7783 --- /dev/null +++ b/src/util/util/ProcessLifecycle.ts @@ -0,0 +1,62 @@ +/* + Spacebar: A FOSS re-implementation and extension of the Discord.com backend. + Copyright (C) 2026 Spacebar and Spacebar Contributors + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +import EventEmitter from "node:events"; +import whyIsNodeRunning from "why-is-node-running"; + +interface ProcessLifecycleEvents { + starting: unknown[]; + running: unknown[]; + stopping: unknown[]; + stopped: unknown[]; +} + +export class ProcessLifecycle { + static state: keyof ProcessLifecycleEvents = "starting"; + static eventEmitter: EventEmitter = new EventEmitter(); + + // to be ran after startup is finished + static async Ready() { + await this.emitAsync((this.state = "running")); + } + + // to be ran at the start of shutdown + static async Shutdown() { + await this.emitAsync((this.state = "stopping")); + } + + // to be ran at the end of shutdown (clean up sockets, ...) + static async Finalize() { + await this.emitAsync((this.state = "stopped")); + } + + // emit, except it awaits promises + private static async emitAsync(eventName: keyof ProcessLifecycleEvents) { + for (const evt of this.eventEmitter.listeners(eventName)) { + // noinspection JSVoidFunctionReturnValueUsed - we want to handle async functions blocking aswell + const res = evt() as void | Promise; + if (res) await res; + } + } +} + +process.on("SIGUSR1", () => { + console.log("Handling SIGUSR1:"); + whyIsNodeRunning(); + console.log("\nProcess state:", ProcessLifecycle.state); +}); diff --git a/src/util/util/ipc/listener/RabbitMqSingleListener.ts b/src/util/util/ipc/listener/RabbitMqSingleListener.ts index d120e70ac5..7e5867acc2 100644 --- a/src/util/util/ipc/listener/RabbitMqSingleListener.ts +++ b/src/util/util/ipc/listener/RabbitMqSingleListener.ts @@ -17,21 +17,36 @@ */ import EventEmitter from "node:events"; +import { randomUUID } from "node:crypto"; import { BaseEventListener } from "./BaseEventListener"; -import { EVENT, Event, EventOpts, sleep } from "@spacebar/util"; +import { arraySum, EVENT, Event, EventOpts, sleep } from "@spacebar/util"; import amqp, { Channel, ChannelModel } from "amqplib"; -import { randomUUID } from "node:crypto"; +import { ProcessLifecycle } from "../../ProcessLifecycle"; +import { Monitoring } from "../../../monitoring/Monitoring"; +import { Gauge } from "prom-client"; export class RabbitMqSingleListener extends BaseEventListener { + static openListenersMetric: Gauge; private readonly host: string; private connection?: ChannelModel; private channel?: Channel; eventEmitter: EventEmitter; + openListenersMetric: Gauge.Internal; constructor(host: string) { super(); this.eventEmitter = new EventEmitter(); this.host = host; + + RabbitMqSingleListener.openListenersMetric = Monitoring.attachMetric( + "spacebar_ipc_unix_listener_open_listener_count", + new Gauge({ + name: "spacebar_ipc_rabbitmqsingle_listener_open_listener_count", + help: "Amount of open listeners on unix socket", + labelNames: ["host"], + }), + ); + this.openListenersMetric = RabbitMqSingleListener.openListenersMetric.labels({ host }); } async init() { @@ -50,9 +65,7 @@ export class RabbitMqSingleListener extends BaseEventListener { } this.channel = await this.connection.createChannel(); - for (const sig of ["SIGINT", "SIGTERM", "SIGQUIT"] as const) { - process.on(sig, () => this.close()); - } + ProcessLifecycle.eventEmitter.on("stopped", async () => await this.close()); this.connection.on("error", (err) => { console.error("[RabbitMQSingleListener] Connection error:", err); @@ -94,6 +107,7 @@ export class RabbitMqSingleListener extends BaseEventListener { this.channel = undefined; await this.connection?.close(); this.connection = undefined; + RabbitMqSingleListener.openListenersMetric.remove({ host: this.host }); } async listen(event: string, callback: (event: EventOpts) => unknown): Promise<() => Promise> { @@ -105,10 +119,12 @@ export class RabbitMqSingleListener extends BaseEventListener { }; this.eventEmitter.addListener(event, listener); + this.openListenersMetric.set(arraySum(this.eventEmitter.eventNames().map((e) => this.eventEmitter.listeners(e).length))); const cancel = async () => { this.eventEmitter.removeListener(event, listener); this.eventEmitter.setMaxListeners(this.eventEmitter.getMaxListeners() - 1); + this.openListenersMetric.set(arraySum(this.eventEmitter.eventNames().map((e) => this.eventEmitter.listeners(e).length))); }; this.eventEmitter.setMaxListeners(this.eventEmitter.getMaxListeners() + 1); diff --git a/src/util/util/ipc/listener/UnixSocketListener.ts b/src/util/util/ipc/listener/UnixSocketListener.ts index 9465e9cadc..274b8ae128 100644 --- a/src/util/util/ipc/listener/UnixSocketListener.ts +++ b/src/util/util/ipc/listener/UnixSocketListener.ts @@ -20,18 +20,47 @@ import EventEmitter from "node:events"; import fs from "node:fs"; import net, { Server } from "node:net"; import { BaseEventListener } from "./BaseEventListener"; -import { EVENT, Event, EventOpts } from "@spacebar/util"; +import { arraySum, EVENT, Event, EventOpts } from "@spacebar/util"; +import { ProcessLifecycle } from "../../ProcessLifecycle"; +import { Gauge } from "prom-client"; +import { Monitoring } from "../../../monitoring/Monitoring"; export class UnixSocketListener extends BaseEventListener { + static openConnectionsMetric?: Gauge; + static openListenersMetric?: Gauge; + eventEmitter: EventEmitter; socketPath: string; server: Server; isInitialized = false; + openConnectionsMetric: Gauge.Internal; + openListenersMetric: Gauge.Internal; + isInitialized = false; constructor(socketPath: string) { super(); this.eventEmitter = new EventEmitter(); this.socketPath = socketPath; + + UnixSocketListener.openConnectionsMetric = Monitoring.attachMetric( + "spacebar_ipc_unix_listener_open_connection_count", + new Gauge({ + name: "spacebar_ipc_unix_listener_open_connection_count", + help: "Amount of open inbound connections on unix socket", + labelNames: ["path"], + }), + ); + this.openConnectionsMetric = UnixSocketListener.openConnectionsMetric.labels({ path: socketPath }); + + UnixSocketListener.openListenersMetric = Monitoring.attachMetric( + "spacebar_ipc_unix_listener_open_listener_count", + new Gauge({ + name: "spacebar_ipc_unix_listener_open_listener_count", + help: "Amount of open listeners on unix socket", + labelNames: ["path"], + }), + ); + this.openListenersMetric = UnixSocketListener.openListenersMetric.labels({ path: socketPath }); } async init() { @@ -49,6 +78,7 @@ export class UnixSocketListener extends BaseEventListener { this.server = net.createServer((socket) => { socket.on("connect", () => { console.log("[UnixSocketListener] Unix socket client connected, now at", this.server.connections, "connections..."); + this.openConnectionsMetric.set(this.server.connections); }); let buffer = Buffer.alloc(0); socket.on("data", (data: Buffer) => { @@ -71,16 +101,15 @@ export class UnixSocketListener extends BaseEventListener { }); socket.on("close", () => { console.log("[UnixSocketListener] Unix socket client disconnected"); + this.openConnectionsMetric.set(this.server.connections ?? 0); }); }); this.server.listen(this.socketPath, () => { - console.log(`[UnixSocketListener] Listening on ${this.socketPath}`); + console.log(`[UnixSocketListener] listening on ${this.socketPath}`); }); - for (const sig of ["SIGINT", "SIGTERM", "SIGQUIT"] as const) { - process.on(sig, () => this.close()); - } + ProcessLifecycle.eventEmitter.on("stopped", async () => await this.close()); this.isInitialized = true; } @@ -91,11 +120,13 @@ export class UnixSocketListener extends BaseEventListener { console.log("[UnixSocketListener] Closing unix socket server"); this.server.close(); + UnixSocketListener.openConnectionsMetric?.remove({ path: this.socketPath }); // clean up socket file try { fs.unlinkSync(this.socketPath); } catch (e) { + if (e instanceof Error && "errno" in e && e.errno == -2) return; console.error("[UnixSocketListener] Failed to unlink socket file:", e); } } @@ -109,10 +140,12 @@ export class UnixSocketListener extends BaseEventListener { }; this.eventEmitter.addListener(event, listener); + this.openListenersMetric.set(arraySum(this.eventEmitter.eventNames().map((e) => this.eventEmitter.listeners(e).length))); const cancel = async () => { this.eventEmitter.removeListener(event, listener); this.eventEmitter.setMaxListeners(this.eventEmitter.getMaxListeners() - 1); + this.openListenersMetric.set(arraySum(this.eventEmitter.eventNames().map((e) => this.eventEmitter.listeners(e).length))); }; this.eventEmitter.setMaxListeners(this.eventEmitter.getMaxListeners() + 1); diff --git a/src/util/util/ipc/writer/RabbitMqSingleWriter.ts b/src/util/util/ipc/writer/RabbitMqSingleWriter.ts index 72a0d08ce7..ad00aa2b51 100644 --- a/src/util/util/ipc/writer/RabbitMqSingleWriter.ts +++ b/src/util/util/ipc/writer/RabbitMqSingleWriter.ts @@ -19,6 +19,7 @@ import { BaseEventWriter } from "./BaseEventWriter"; import amqp, { Channel, ChannelModel } from "amqplib"; import { Event, sleep } from "@spacebar/util"; +import { ProcessLifecycle } from "../../ProcessLifecycle"; export class RabbitMqSingleWriter extends BaseEventWriter { private readonly host: string; @@ -46,10 +47,7 @@ export class RabbitMqSingleWriter extends BaseEventWriter { } this.channel = await this.connection.createChannel(); - for (const sig of ["SIGINT", "SIGTERM", "SIGQUIT"] as const) { - process.on(sig, () => this.close()); - } - + ProcessLifecycle.eventEmitter.on("stopped", async () => await this.close()); this.connection.on("error", (err) => { console.error("[RabbitMQSingleWriter] Connection error:", err); }); diff --git a/src/util/util/ipc/writer/UnixSocketWriter.ts b/src/util/util/ipc/writer/UnixSocketWriter.ts index e3b91de8fd..b50f53dcb8 100644 --- a/src/util/util/ipc/writer/UnixSocketWriter.ts +++ b/src/util/util/ipc/writer/UnixSocketWriter.ts @@ -22,8 +22,13 @@ import path from "node:path"; import { red } from "picocolors"; import { BaseEventWriter } from "./BaseEventWriter"; import { Event, Stopwatch } from "@spacebar/util"; +import { ProcessLifecycle } from "../../ProcessLifecycle"; +import { Monitoring } from "../../../monitoring/Monitoring"; +import { Gauge } from "prom-client"; export class UnixSocketWriter extends BaseEventWriter { + private static openConnectionsMetric: Gauge; + socketPath: string; clients: { [key: string]: Socket } = {}; watcher?: FSWatcher; @@ -31,10 +36,21 @@ export class UnixSocketWriter extends BaseEventWriter { broadcastLock: Promise = Promise.resolve(); replayLock: Promise = Promise.resolve(); isInitializing = true; + openConnectionsMetric: Gauge.Internal; constructor(socketPath: string) { super(); this.socketPath = socketPath; + + UnixSocketWriter.openConnectionsMetric = Monitoring.attachMetric( + "spacebar_ipc_unix_writer_open_connection_count", + new Gauge({ + name: "spacebar_ipc_unix_writer_open_connection_count", + help: "Amount of open outbound connections on unix socket", + labelNames: ["path"], + }), + ); + this.openConnectionsMetric = UnixSocketWriter.openConnectionsMetric.labels({ path: socketPath }); } async init() { @@ -79,6 +95,7 @@ export class UnixSocketWriter extends BaseEventWriter { try { this.clients[fullPath] = net.createConnection(fullPath, () => { console.log("[UnixSocketWriter] Unix socket client connected to", fullPath); + this.openConnectionsMetric.set(Object.entries(this.clients).length); }); this.clients[fullPath].on("error", (err) => { @@ -93,6 +110,7 @@ export class UnixSocketWriter extends BaseEventWriter { this.clients[fullPath].on("close", () => { console.log("[UnixSocketWriter] Unix socket client closed:", fullPath); delete this.clients[fullPath]; + this.openConnectionsMetric.set(Object.entries(this.clients).length); }); } catch (e) { console.error("[UnixSocketWriter] Failed to create connection to", fullPath, ":", e); @@ -142,10 +160,7 @@ export class UnixSocketWriter extends BaseEventWriter { console.error("[UnixSocketWriter] Unix socket writer failed to read directory:", err); } - for (const sig of ["SIGINT", "SIGTERM", "SIGQUIT"] as const) { - process.on(sig, () => this.close()); - } - + ProcessLifecycle.eventEmitter.on("stopped", async () => await this.close()); this.isInitializing = false; } @@ -226,6 +241,7 @@ export class UnixSocketWriter extends BaseEventWriter { } } this.clients = {}; + UnixSocketWriter.openConnectionsMetric.remove({ path: this.socketPath }); } } diff --git a/src/webrtc/Server.ts b/src/webrtc/Server.ts index 0f8ead2ce6..f3e8e2c268 100644 --- a/src/webrtc/Server.ts +++ b/src/webrtc/Server.ts @@ -15,14 +15,15 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ -import dotenv from "dotenv"; -dotenv.config({ quiet: true }); -import { closeDatabase, Config, initDatabase, initEvent, Session, TimeSpan } from "@spacebar/util"; + import http from "node:http"; import ws from "ws"; +import { green, yellow } from "picocolors"; +import { Config, initDatabase, initEvent } from "@spacebar/util"; import { Connection } from "./events/Connection"; import { loadWebRtcLibrary, mediaServer, WRTC_PORT_MAX, WRTC_PORT_MIN, WRTC_PUBLIC_IP } from "./util"; -import { green, yellow } from "picocolors"; +import { ProcessLifecycle } from "../util/util/ProcessLifecycle"; +import { Monitoring } from "../util/monitoring/Monitoring"; export class Server { public ws: ws.Server; @@ -36,8 +37,11 @@ export class Server { if (server) this.server = server; else { - this.server = http.createServer(function (req, res) { - res.writeHead(200).end("Online"); + this.server = http.createServer(async (req, res) => { + const requestUrl = new URL(`http://${req.headers.host}${req.url}`); + if (requestUrl.pathname === "/metrics") { + return await Monitoring.handleRawRequest(req, res); + } else res.writeHead(200).end("Online"); }); } @@ -59,6 +63,7 @@ export class Server { } async start(): Promise { + await Monitoring.init(); await initDatabase(); await Config.init(); await initEvent(); @@ -76,11 +81,14 @@ export class Server { this.server.listen(this.port); console.log(`[WebRTC] ${green(`online on 0.0.0.0:${this.port}`)}`); } + + await ProcessLifecycle.Ready(); } async stop() { - await closeDatabase(); + await ProcessLifecycle.Shutdown(); this.server.close(); - mediaServer?.stop(); + await mediaServer?.stop(); + await ProcessLifecycle.Finalize(); } } From 9f9c30186fc38d4a8044d9366f854086b366521c Mon Sep 17 00:00:00 2001 From: Rory& Date: Thu, 28 May 2026 13:53:30 +0200 Subject: [PATCH 02/11] Move why-is-node-running to dependencies --- package-lock.json | 5 ++--- package.json | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 05ebdfacd4..2b96765dcc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -50,6 +50,7 @@ "reflect-metadata": "^0.2.2", "tslib": "^2.8.1", "typeorm": "^0.3.30", + "why-is-node-running": "^3.2.2", "wretch": "^3.0.8", "ws": "^8.21.0" }, @@ -86,8 +87,7 @@ "pretty-quick": "^4.2.2", "ts-node": "^10.9.2", "typescript": "^6.0.3", - "typescript-json-schema": "^0.67.4", - "why-is-node-running": "^3.2.2" + "typescript-json-schema": "^0.67.4" }, "optionalDependencies": { "@sendgrid/mail": "^8.1.6", @@ -7392,7 +7392,6 @@ "version": "3.2.2", "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-3.2.2.tgz", "integrity": "sha512-NKUzAelcoCXhXL4dJzKIwXeR8iEVqsA0Lq6Vnd0UXvgaKbzVo4ZTHROF2Jidrv+SgxOQ03fMinnNhzZATxOD3A==", - "dev": true, "license": "MIT", "bin": { "why-is-node-running": "cli.js" diff --git a/package.json b/package.json index 35108b7160..dc3fe06a81 100644 --- a/package.json +++ b/package.json @@ -118,6 +118,7 @@ "reflect-metadata": "^0.2.2", "tslib": "^2.8.1", "typeorm": "^0.3.30", + "why-is-node-running": "^3.2.2", "wretch": "^3.0.8", "ws": "^8.21.0" }, From d8fb6d2ee836195165bf7f0f32c2c3677243e899 Mon Sep 17 00:00:00 2001 From: Rory& Date: Thu, 28 May 2026 16:18:30 +0200 Subject: [PATCH 03/11] CodeOwners: claim /nix dir --- .github/CODEOWNERS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b55d76cd50..b956be7ebd 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1,4 @@ # Nix stuff is owned by Rory& - we& want to be notified if these are changed. /flake.nix root@rory.gay -/nix-update.sh root@rory.gay \ No newline at end of file +/nix-update.sh root@rory.gay +/nix root@rory.gay \ No newline at end of file From 4a9c6cacc5cadccbf88d910d62e222bf3f4ff5ff Mon Sep 17 00:00:00 2001 From: Rory& Date: Wed, 20 May 2026 13:51:11 +0200 Subject: [PATCH 04/11] */Server.ts: Add monitoring init hook --- package-lock.json | 38 +++++++++++++++++++++++++++++++ package.json | 1 + src/api/Server.ts | 1 - src/cdn/Server.ts | 1 - src/util/monitoring/Monitoring.ts | 38 +++++++++++++++++++++++++++++++ 5 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 src/util/monitoring/Monitoring.ts diff --git a/package-lock.json b/package-lock.json index 2b96765dcc..659cd6a61e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -47,6 +47,7 @@ "pg-query-stream": "^4.15.0", "picocolors": "^1.1.1", "probe-image-size": "^7.3.0", + "prom-client": "^15.1.3", "reflect-metadata": "^0.2.2", "tslib": "^2.8.1", "typeorm": "^0.3.30", @@ -954,6 +955,15 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/@peculiar/asn1-schema": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/@peculiar/asn1-schema/-/asn1-schema-2.7.0.tgz", @@ -2325,6 +2335,12 @@ "integrity": "sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA==", "license": "CC0-1.0" }, + "node_modules/bintrees": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", + "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==", + "license": "MIT" + }, "node_modules/bmp-ts": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/bmp-ts/-/bmp-ts-1.0.9.tgz", @@ -5876,6 +5892,19 @@ "stream-parser": "~0.3.1" } }, + "node_modules/prom-client": { + "version": "15.1.3", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.3.tgz", + "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.4.0", + "tdigest": "^0.1.1" + }, + "engines": { + "node": "^16 || ^18 || >=20" + } + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -6551,6 +6580,15 @@ "node": ">=8" } }, + "node_modules/tdigest": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", + "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", + "license": "MIT", + "dependencies": { + "bintrees": "1.0.2" + } + }, "node_modules/thirty-two": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/thirty-two/-/thirty-two-1.0.2.tgz", diff --git a/package.json b/package.json index dc3fe06a81..f20e31d400 100644 --- a/package.json +++ b/package.json @@ -115,6 +115,7 @@ "pg-query-stream": "^4.15.0", "picocolors": "^1.1.1", "probe-image-size": "^7.3.0", + "prom-client": "^15.1.3", "reflect-metadata": "^0.2.2", "tslib": "^2.8.1", "typeorm": "^0.3.30", diff --git a/src/api/Server.ts b/src/api/Server.ts index f9f1c708f6..a1edb986c7 100644 --- a/src/api/Server.ts +++ b/src/api/Server.ts @@ -53,7 +53,6 @@ export class SpacebarServer extends Server { async start() { await Monitoring.init(); - Monitoring.attach(this.app); await initDatabase(); await Config.init(); await initEvent(); diff --git a/src/cdn/Server.ts b/src/cdn/Server.ts index 8964161dcc..9327de8367 100644 --- a/src/cdn/Server.ts +++ b/src/cdn/Server.ts @@ -37,7 +37,6 @@ export class CDNServer extends Server { async start() { await Monitoring.init(); - Monitoring.attach(this.app); await initDatabase(); await Config.init(); diff --git a/src/util/monitoring/Monitoring.ts b/src/util/monitoring/Monitoring.ts new file mode 100644 index 0000000000..3d6d4ed471 --- /dev/null +++ b/src/util/monitoring/Monitoring.ts @@ -0,0 +1,38 @@ +/* + Spacebar: A FOSS re-implementation and extension of the Discord.com backend. + Copyright (C) 2026 Spacebar and Spacebar Contributors + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +import * as client from "prom-client"; +import { Router } from "express"; + +export class Monitoring { + static isInitialised = false; + public static async init() { + if (Monitoring.isInitialised) return; + console.log("[Monitoring] Initialising prometheus metrics"); + client.collectDefaultMetrics(); + Monitoring.isInitialised = true; + } + + public static attach(router: Router) { + router.get("/metrics", async (req, res) => { + res.setHeader("Content-Type", client.register.contentType); + const metrics = await client.register.metrics(); + res.send(metrics); + }); + } +} From e2da753c2dcfcb1b6243aafa0086a5c086ba654c Mon Sep 17 00:00:00 2001 From: Rory& Date: Wed, 20 May 2026 14:06:19 +0200 Subject: [PATCH 05/11] */Server.ts: Handle metrics requests --- src/api/Server.ts | 1 + src/cdn/Server.ts | 1 + src/util/monitoring/Monitoring.ts | 6 ++++++ 3 files changed, 8 insertions(+) diff --git a/src/api/Server.ts b/src/api/Server.ts index a1edb986c7..f9f1c708f6 100644 --- a/src/api/Server.ts +++ b/src/api/Server.ts @@ -53,6 +53,7 @@ export class SpacebarServer extends Server { async start() { await Monitoring.init(); + Monitoring.attach(this.app); await initDatabase(); await Config.init(); await initEvent(); diff --git a/src/cdn/Server.ts b/src/cdn/Server.ts index 9327de8367..8964161dcc 100644 --- a/src/cdn/Server.ts +++ b/src/cdn/Server.ts @@ -37,6 +37,7 @@ export class CDNServer extends Server { async start() { await Monitoring.init(); + Monitoring.attach(this.app); await initDatabase(); await Config.init(); diff --git a/src/util/monitoring/Monitoring.ts b/src/util/monitoring/Monitoring.ts index 3d6d4ed471..9b668fcd17 100644 --- a/src/util/monitoring/Monitoring.ts +++ b/src/util/monitoring/Monitoring.ts @@ -18,6 +18,7 @@ import * as client from "prom-client"; import { Router } from "express"; +import http, { IncomingMessage, ServerResponse } from "node:http"; export class Monitoring { static isInitialised = false; @@ -35,4 +36,9 @@ export class Monitoring { res.send(metrics); }); } + + static async handleRawRequest(req: IncomingMessage, res: ServerResponse) { + const metrics = await client.register.metrics(); + res.setHeader("Content-Type", client.register.contentType).writeHead(200).end(metrics); + } } From 6712db6bce0895f13577b6ca957f4d3f3f1aa275 Mon Sep 17 00:00:00 2001 From: Rory& Date: Thu, 21 May 2026 15:37:02 +0200 Subject: [PATCH 06/11] Monitoring: tag HTTP response rate histogram by route rather than request url --- src/util/monitoring/Monitoring.ts | 39 +++++++++++++++++++++++--- src/util/util/lambert-server/Server.ts | 10 ++++++- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/src/util/monitoring/Monitoring.ts b/src/util/monitoring/Monitoring.ts index 9b668fcd17..eb714babe3 100644 --- a/src/util/monitoring/Monitoring.ts +++ b/src/util/monitoring/Monitoring.ts @@ -16,9 +16,10 @@ along with this program. If not, see . */ -import * as client from "prom-client"; -import { Router } from "express"; import http, { IncomingMessage, ServerResponse } from "node:http"; +import * as client from "prom-client"; +import { Application, Router } from "express"; +import { sleep } from "@spacebar/util"; export class Monitoring { static isInitialised = false; @@ -29,8 +30,38 @@ export class Monitoring { Monitoring.isInitialised = true; } - public static attach(router: Router) { - router.get("/metrics", async (req, res) => { + public static attach(app: Application) { + const a = app; + const http_request_total = new client.Counter({ + name: "node_http_request_total", + help: "The total number of HTTP requests received", + labelNames: ["path", "method", "status_code"], + }); + client.register.registerMetric(http_request_total); + + const http_response_rate_histogram = new client.Histogram({ + name: "node_http_duration", + labelNames: ["path", "method", "status_code"], + help: "The duration of HTTP requests in seconds", + buckets: [0.0, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0, 10], + }); + client.register.registerMetric(http_response_rate_histogram); + + app.use((req, res, next) => { + const endTimer = http_response_rate_histogram.startTimer(); + res.on("finish", () => { + const r = req; + const path = (res.locals.lambertRouteBase ?? req.baseUrl ?? "") + req.route?.path; + if (!req.route?.path) { + console.log(req); + } + endTimer({ method: req.method, path, status_code: res.statusCode }); + http_request_total.inc({ method: req.method, path, status_code: res.statusCode }); + }); + next(); + }); + + app.get("/metrics", async (req, res) => { res.setHeader("Content-Type", client.register.contentType); const metrics = await client.register.metrics(); res.send(metrics); diff --git a/src/util/util/lambert-server/Server.ts b/src/util/util/lambert-server/Server.ts index 617b279439..4a73f7c32e 100644 --- a/src/util/util/lambert-server/Server.ts +++ b/src/util/util/lambert-server/Server.ts @@ -54,7 +54,15 @@ export class Server { if (router.default) router = router.default; if (!router || router?.prototype?.constructor?.name !== "router") throw `File doesn't export any default router`; - this.app.use(path, router); + this.app.use( + path, + // TODO: I wish this middleware wasn't nessecary to preserve base path param names for monitoring... + (_, res, next) => { + res.locals.lambertRouteBase = path; + next(); + }, + router, + ); if (this.options.serverInitLogging && process.env.LOG_ROUTES !== "false") console.log(`[Server] Route ${path} registered`); From 0df292e90d869c0534903ab29ac3c3ee8f3a95a9 Mon Sep 17 00:00:00 2001 From: Rory& Date: Mon, 25 May 2026 08:55:43 +0200 Subject: [PATCH 07/11] nixos tests: check metrics endpoints on TS services --- nix/tests/test-bundle-starts.nix | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/nix/tests/test-bundle-starts.nix b/nix/tests/test-bundle-starts.nix index 2e088f6de9..852b17180f 100644 --- a/nix/tests/test-bundle-starts.nix +++ b/nix/tests/test-bundle-starts.nix @@ -73,6 +73,8 @@ in }; }; + # https://nixos.org/manual/nixos/stable/index.html#sec-nixos-tests + # https://nixos.org/manual/nixpkgs/unstable/#tester-runNixOSTest testScript = '' machine.wait_for_unit("spacebar-api") machine.wait_for_unit("spacebar-cdn") @@ -82,7 +84,13 @@ in machine.wait_for_open_port(3001) machine.wait_for_open_port(3002) machine.wait_for_open_port(3003) - # If well known works, its probably fine(tm)? + + # this should be working machine.succeed("curl -f http://api.sb.localhost/.well-known/spacebar/client") + + # check if metrics endpoint works on all services + machine.succeed("curl -f http://api.sb.localhost/metrics") + machine.succeed("curl -f http://gateway.sb.localhost/metrics") + machine.succeed("curl -f http://cdn.sb.localhost/metrics") ''; } From 22b5ed4434f87204206427d0e99a90a747f737d4 Mon Sep 17 00:00:00 2001 From: Rory& Date: Mon, 25 May 2026 09:12:36 +0200 Subject: [PATCH 08/11] monitoring: ignore OPTIONS requests that dont have a route handler --- src/util/monitoring/Monitoring.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/util/monitoring/Monitoring.ts b/src/util/monitoring/Monitoring.ts index eb714babe3..c72e33d92a 100644 --- a/src/util/monitoring/Monitoring.ts +++ b/src/util/monitoring/Monitoring.ts @@ -31,16 +31,15 @@ export class Monitoring { } public static attach(app: Application) { - const a = app; const http_request_total = new client.Counter({ - name: "node_http_request_total", + name: "spacebar_http_request_total", help: "The total number of HTTP requests received", labelNames: ["path", "method", "status_code"], }); client.register.registerMetric(http_request_total); const http_response_rate_histogram = new client.Histogram({ - name: "node_http_duration", + name: "spacebar_http_duration", labelNames: ["path", "method", "status_code"], help: "The duration of HTTP requests in seconds", buckets: [0.0, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0, 10], @@ -50,12 +49,15 @@ export class Monitoring { app.use((req, res, next) => { const endTimer = http_response_rate_histogram.startTimer(); res.on("finish", () => { - const r = req; const path = (res.locals.lambertRouteBase ?? req.baseUrl ?? "") + req.route?.path; - if (!req.route?.path) { - console.log(req); + if (!req.route?.path && req.method !== "OPTIONS") { + console.log("[Monitoring] Request route path was undefined? Request path:", req.path, "Request route:", req.route); } endTimer({ method: req.method, path, status_code: res.statusCode }); + + // OPTIONS requests don't set path due to not being routed... discard unhandled ones + if (!path && req.method === "OPTIONS") return; + http_request_total.inc({ method: req.method, path, status_code: res.statusCode }); }); next(); From acc4a10f09958ba6dc668fb7fc48099b40e1369b Mon Sep 17 00:00:00 2001 From: Rory& Date: Mon, 25 May 2026 09:14:29 +0200 Subject: [PATCH 09/11] monitoring: add helper to register a metric without duplicating --- src/util/monitoring/Monitoring.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/util/monitoring/Monitoring.ts b/src/util/monitoring/Monitoring.ts index c72e33d92a..2479267428 100644 --- a/src/util/monitoring/Monitoring.ts +++ b/src/util/monitoring/Monitoring.ts @@ -16,20 +16,28 @@ along with this program. If not, see . */ -import http, { IncomingMessage, ServerResponse } from "node:http"; +import { IncomingMessage, ServerResponse } from "node:http"; import * as client from "prom-client"; import { Application, Router } from "express"; -import { sleep } from "@spacebar/util"; +import { Metric } from "prom-client"; export class Monitoring { static isInitialised = false; public static async init() { if (Monitoring.isInitialised) return; console.log("[Monitoring] Initialising prometheus metrics"); - client.collectDefaultMetrics(); + client.collectDefaultMetrics({ prefix: "spacebar_" }); Monitoring.isInitialised = true; } + public static attachMetric(name: string, metric: T): T { + const existingMetric = client.register.getSingleMetric(name); + // TODO: is there any way to *ensure* the metric is T? We're assuming that there's no conflicting definitions across the app... + if (existingMetric) return existingMetric as T; + client.register.registerMetric(metric); + return metric; + } + public static attach(app: Application) { const http_request_total = new client.Counter({ name: "spacebar_http_request_total", From 63fb0a93a24c31dfb2d35d186b72d27174a92368 Mon Sep 17 00:00:00 2001 From: Rory& Date: Tue, 26 May 2026 09:12:22 +0200 Subject: [PATCH 10/11] IPC metrics --- src/util/util/extensions/Array.ts | 6 ++++++ src/util/util/ipc/Event.ts | 2 +- src/util/util/ipc/listener/UnixSocketListener.ts | 5 ++--- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/util/util/extensions/Array.ts b/src/util/util/extensions/Array.ts index e3e6478c2c..c95328b5af 100644 --- a/src/util/util/extensions/Array.ts +++ b/src/util/util/extensions/Array.ts @@ -74,3 +74,9 @@ export function arrayDistributeSequentially(array: T[], count: number): T[][] return groups; } + +//region Numerics +export function arraySum(array: number[]) { + return array.reduce((prev, curr) => prev + curr, 0); +} +//endregion diff --git a/src/util/util/ipc/Event.ts b/src/util/util/ipc/Event.ts index e836c269ea..af7c832df1 100644 --- a/src/util/util/ipc/Event.ts +++ b/src/util/util/ipc/Event.ts @@ -125,7 +125,7 @@ export async function listenEvent(event: string, callback: (event: EventOpts) => } if (!listener) { - listener = listener = new UnixSocketListener(path.join(process.env.EVENT_SOCKET_PATH, `${process.pid}.sock`)); + listener = new UnixSocketListener(path.join(process.env.EVENT_SOCKET_PATH, `${process.pid}.sock`)); await listener.init(); } return await listener.listen(event, callback); diff --git a/src/util/util/ipc/listener/UnixSocketListener.ts b/src/util/util/ipc/listener/UnixSocketListener.ts index 274b8ae128..03f58ab788 100644 --- a/src/util/util/ipc/listener/UnixSocketListener.ts +++ b/src/util/util/ipc/listener/UnixSocketListener.ts @@ -35,7 +35,6 @@ export class UnixSocketListener extends BaseEventListener { isInitialized = false; openConnectionsMetric: Gauge.Internal; openListenersMetric: Gauge.Internal; - isInitialized = false; constructor(socketPath: string) { super(); @@ -101,12 +100,12 @@ export class UnixSocketListener extends BaseEventListener { }); socket.on("close", () => { console.log("[UnixSocketListener] Unix socket client disconnected"); - this.openConnectionsMetric.set(this.server.connections ?? 0); + this.openConnectionsMetric.set(this.server.connections); }); }); this.server.listen(this.socketPath, () => { - console.log(`[UnixSocketListener] listening on ${this.socketPath}`); + console.log(`[UnixSocketListener] Listening on ${this.socketPath}`); }); ProcessLifecycle.eventEmitter.on("stopped", async () => await this.close()); From d3e81007e548059a4a0effbb19822eabe2f6c1f7 Mon Sep 17 00:00:00 2001 From: Rory& Date: Tue, 26 May 2026 10:24:06 +0200 Subject: [PATCH 11/11] IPC: shutdown hooks --- src/util/util/ipc/listener/UnixSocketListener.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/util/ipc/listener/UnixSocketListener.ts b/src/util/util/ipc/listener/UnixSocketListener.ts index 03f58ab788..50ffe2b099 100644 --- a/src/util/util/ipc/listener/UnixSocketListener.ts +++ b/src/util/util/ipc/listener/UnixSocketListener.ts @@ -100,7 +100,7 @@ export class UnixSocketListener extends BaseEventListener { }); socket.on("close", () => { console.log("[UnixSocketListener] Unix socket client disconnected"); - this.openConnectionsMetric.set(this.server.connections); + this.openConnectionsMetric.set(this.server.connections ?? 0); }); });