Prometheus monitoring support (stage 1)

.github/CODEOWNERS

@@ -1,3 +1,4 @@
 # Nix stuff is owned by Rory& - we& want to be notified if these are changed.
 /flake.nix root@rory.gay
-/nix-update.sh root@rory.gay
+/nix-update.sh root@rory.gay
+/nix root@rory.gay

nix/tests/test-bundle-starts.nix

@@ -73,6 +73,8 @@ in
     };
   };
 
+  # https://nixos.org/manual/nixos/stable/index.html#sec-nixos-tests
+  # https://nixos.org/manual/nixpkgs/unstable/#tester-runNixOSTest
   testScript = ''
     machine.wait_for_unit("spacebar-api")
     machine.wait_for_unit("spacebar-cdn")
@@ -82,7 +84,13 @@ in
     machine.wait_for_open_port(3001)
     machine.wait_for_open_port(3002)
     machine.wait_for_open_port(3003)
-    # If well known works, its probably fine(tm)?
+
+    # this should be working
     machine.succeed("curl -f http://api.sb.localhost/.well-known/spacebar/client")
+
+    # check if metrics endpoint works on all services
+    machine.succeed("curl -f http://api.sb.localhost/metrics")
+    machine.succeed("curl -f http://gateway.sb.localhost/metrics")
+    machine.succeed("curl -f http://cdn.sb.localhost/metrics")
   '';
 }

package.json

@@ -115,9 +115,11 @@
         "pg-query-stream": "^4.15.0",
         "picocolors": "^1.1.1",
         "probe-image-size": "^7.3.0",
+        "prom-client": "^15.1.3",
         "reflect-metadata": "^0.2.2",
         "tslib": "^2.8.1",
         "typeorm": "^0.3.30",
+        "why-is-node-running": "^3.2.2",
         "wretch": "^3.0.8",
         "ws": "^8.21.0"
     },

src/api/Server.ts

@@ -16,15 +16,17 @@
 	along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 
-import { Config, ConnectionConfig, ConnectionLoader, Email, JSONReplacer, WebAuthn, initDatabase, initEvent, registerRoutes, getDatabase, getRevInfoOrFail } from "@spacebar/util";
-import { Authentication, CORS, ImageProxy, BodyParser, ErrorHandler, initRateLimits, initTranslation } from "./middlewares";
+import path from "node:path";
 import { Request, Response, Router } from "express";
-import { Server, ServerOptions } from "lambert-server";
 import morgan from "morgan";
-import path from "node:path";
+import { Server, ServerOptions } from "lambert-server";
 import { red } from "picocolors";
+import { Config, ConnectionConfig, ConnectionLoader, Email, JSONReplacer, WebAuthn, initDatabase, initEvent, registerRoutes, getDatabase, getRevInfoOrFail } from "@spacebar/util";
+import { Authentication, CORS, ImageProxy, BodyParser, ErrorHandler, initRateLimits, initTranslation } from "./middlewares";
 import { initInstance } from "./util/handlers/Instance";
 import { route } from "./util";
+import { ProcessLifecycle } from "../util/util/ProcessLifecycle";
+import { Monitoring } from "../util/monitoring/Monitoring";
 
 const ASSETS_FOLDER = path.join(__dirname, "..", "..", "assets");
 const PUBLIC_ASSETS_FOLDER = path.join(ASSETS_FOLDER, "public");
@@ -50,6 +52,8 @@ export class SpacebarServer extends Server {
     }
 
     async start() {
+        await Monitoring.init();
+        Monitoring.attach(this.app);
         await initDatabase();
         await Config.init();
         await initEvent();
@@ -196,6 +200,7 @@ export class SpacebarServer extends Server {
 
         if (logRequests) console.log(red(`Warning: Request logging is enabled! This will spam your console!\nTo disable this, unset the 'LOG_REQUESTS' environment variable!`));
 
+        await ProcessLifecycle.Ready();
         return super.start();
     }
 }

src/api/routes/stop.ts

@@ -18,6 +18,7 @@
 
 import { route } from "@spacebar/api";
 import { Request, Response, Router } from "express";
+import { ProcessLifecycle } from "../../util/util/ProcessLifecycle";
 
 const router: Router = Router({ mergeParams: true });
 
@@ -35,7 +36,9 @@ router.post(
     (req: Request, res: Response) => {
         console.log(`/stop was called by ${req.user_id} at ${new Date()}`);
         res.sendStatus(200);
-        process.kill(process.pid, "SIGTERM");
+        ProcessLifecycle.Shutdown().catch((e) => {
+            console.error("Failed to shut down:", e);
+        });
     },
 );
 

src/bundle/Server.ts

@@ -16,21 +16,19 @@
 	along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 
-import morgan from "morgan";
-
-process.on("unhandledRejection", console.error);
-process.on("uncaughtException", console.error);
-
 import http from "node:http";
+import fs from "node:fs";
+import cluster from "node:cluster";
+import morgan from "morgan";
+import express from "express";
+import { green, bold } from "picocolors";
 import * as Api from "@spacebar/api";
 import * as Gateway from "@spacebar/gateway";
 import * as Webrtc from "@spacebar/webrtc";
 import { CDNServer } from "@spacebar/cdn";
-import express from "express";
-import { green, bold } from "picocolors";
 import { Config, initDatabase } from "@spacebar/util";
-import fs from "node:fs";
-import cluster from "node:cluster";
+import { ProcessLifecycle } from "../util/util/ProcessLifecycle";
+import { Monitoring } from "../util/monitoring/Monitoring";
 
 const app = express();
 const server = http.createServer();
@@ -48,8 +46,7 @@ const webrtc = new Webrtc.Server({
     production,
 });
 
-process.on("SIGTERM", async () => {
-    console.log("Shutting down due to SIGTERM");
+ProcessLifecycle.eventEmitter.on("stopping", async () => {
     await gateway.stop();
     await cdn.stop();
     await api.stop();
@@ -58,6 +55,7 @@ process.on("SIGTERM", async () => {
 });
 
 async function main() {
+    await Monitoring.init();
     await initDatabase();
     await Config.init();
 

src/cdn/Server.ts

@@ -16,13 +16,15 @@
 	along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 
+import path from "node:path";
+import morgan from "morgan";
 import { Server, ServerOptions } from "lambert-server";
 import { Attachment, Config, initDatabase, registerRoutes } from "@spacebar/util";
 import { CORS, BodyParser } from "@spacebar/api";
-import path from "node:path";
 import guildProfilesRoute from "./routes/guild-profiles";
-import morgan from "morgan";
 import { storage } from "./util";
+import { ProcessLifecycle } from "../util/util/ProcessLifecycle";
+import { Monitoring } from "../util/monitoring/Monitoring";
 
 export type CDNServerOptions = ServerOptions;
 
@@ -34,6 +36,8 @@ export class CDNServer extends Server {
     }
 
     async start() {
+        await Monitoring.init();
+        Monitoring.attach(this.app);
         await initDatabase();
         await Config.init();
 
@@ -71,6 +75,7 @@ export class CDNServer extends Server {
         this.app.use("/guilds/:guild_id/users/:user_id/banners", guildProfilesRoute);
         if (process.env.LOG_ROUTES !== "false") console.log("[Server] Route /guilds/:guild_id/users/:user_id/banners registered");
 
+        await ProcessLifecycle.Ready();
         return super.start();
     }
 
@@ -89,6 +94,8 @@ export class CDNServer extends Server {
     }
 
     async stop() {
+        await ProcessLifecycle.Shutdown();
+        await ProcessLifecycle.Finalize();
         return super.stop();
     }
 }

src/gateway/Server.ts

@@ -16,21 +16,22 @@
 	along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 
-import dotenv from "dotenv";
-dotenv.config({ quiet: true });
-import { checkToken, closeDatabase, Config, initDatabase, initEvent, Rights } from "@spacebar/util";
+import http from "node:http";
+import { setInterval } from "node:timers";
 import ws from "ws";
+import { checkToken, Config, initDatabase, initEvent, Rights } from "@spacebar/util";
+import { randomString } from "@spacebar/api"; // TODO: move to util
 import { Connection, openConnections } from "./events/Connection";
-import http from "node:http";
 import { cleanupOnStartup } from "./util";
-import { randomString } from "@spacebar/api";
-import { setInterval } from "node:timers";
+import { ProcessLifecycle } from "../util/util/ProcessLifecycle";
+import { Monitoring } from "../util/monitoring/Monitoring";
 
 export class Server {
     public ws: ws.Server;
     public port: number;
     public server: http.Server;
     public production: boolean;
+    private monitoringLoop: NodeJS.Timeout;
 
     constructor({ port, server, production }: { port: number; server?: http.Server; production?: boolean }) {
         this.port = port;
@@ -42,7 +43,7 @@ export class Server {
             const eluP = [1, 5, 15].map(() => performance.eventLoopUtilization());
             const cpu = [1, 5, 15].map(() => process.cpuUsage());
             let sec = 0;
-            setInterval(() => {
+            const monitoringLoop = setInterval(() => {
                 sec += 1;
                 // for some reason this behaves differently from cpuUsage, so we need an absolute reference as "previous"
                 const eluC = performance.eventLoopUtilization();
@@ -67,7 +68,9 @@ export class Server {
                     res.setHeader("Set-Cookie", `__sb_sessid=${randomString(32)}; Secure; HttpOnly; SameSite=None; Path=/`);
                 }
                 const requestUrl = new URL(`http://${req.headers.host}${req.url}`);
-                if (requestUrl.pathname === "/_spacebar/gateway/admin/introspect") {
+                if (requestUrl.pathname === "/metrics") {
+                    return await Monitoring.handleRawRequest(req, res);
+                } else if (requestUrl.pathname === "/_spacebar/gateway/admin/introspect") {
                     if (!req.headers.authorization) {
                         return res.writeHead(401).end("Unauthorized");
                     } else {
@@ -150,6 +153,8 @@ export class Server {
 
                 res.writeHead(200).end("Online");
             });
+
+            ProcessLifecycle.eventEmitter.on("stopping", () => clearTimeout(monitoringLoop));
         }
 
         this.server.on("upgrade", (request, socket, head) => {
@@ -167,6 +172,7 @@ export class Server {
     }
 
     async start(): Promise<void> {
+        await Monitoring.init();
         await initDatabase();
         await Config.init();
         await initEvent();
@@ -177,14 +183,16 @@ export class Server {
             this.server.listen(this.port);
             console.log(`[Gateway] online on 0.0.0.0:${this.port}`);
         }
+
+        await ProcessLifecycle.Ready();
     }
 
     async stop() {
+        await ProcessLifecycle.Shutdown();
+        clearInterval(this.monitoringLoop);
         this.ws.clients.forEach((x) => x.close());
-        this.ws.close(() => {
-            this.server.close(() => {
-                closeDatabase();
-            });
-        });
+        this.ws.close();
+        this.server.close();
+        await ProcessLifecycle.Finalize();
     }
 }