Applies clustering graceful shutdown patch

Author: jim-toth

operations/operator-registry-controller-live.hcl

@@ -13,11 +13,8 @@ job "operator-registry-controller-live" {
 
     update {
       max_parallel     = 1
-      canary           = 1
       min_healthy_time = "30s"
       healthy_deadline = "5m"
-      auto_revert      = true
-      auto_promote     = true
     }
 
     network {
@@ -28,6 +25,7 @@ job "operator-registry-controller-live" {
 
     task "operator-registry-controller-live-service" {
       driver = "docker"
+      kill_timeout = "30s"
       config {
         image = "ghcr.io/anyone-protocol/operator-registry-controller:[[ .commit_sha ]]"
         network_mode = "host"

operations/operator-registry-controller-stage.hcl

@@ -13,11 +13,8 @@ job "operator-registry-controller-stage" {
 
     update {
       max_parallel     = 1
-      canary           = 1
       min_healthy_time = "30s"
       healthy_deadline = "5m"
-      auto_revert      = true
-      auto_promote     = true
     }
 
     network {
@@ -28,6 +25,7 @@ job "operator-registry-controller-stage" {
 
     task "operator-registry-controller-stage-service" {
       driver = "docker"
+      kill_timeout = "30s"
       config {
         network_mode = "host"
         image = "ghcr.io/anyone-protocol/operator-registry-controller:[[ .commit_sha ]]"

src/cluster/cluster.service.ts

@@ -24,6 +24,7 @@ export class ClusterService
   private serviceName: string
   private sessionId: string | null = null
   private consul?: Consul
+  private renewInterval?: NodeJS.Timeout
 
   constructor(
     private readonly config: ConfigService<{
@@ -122,7 +123,13 @@ export class ClusterService
   }
 
   async beforeApplicationShutdown(): Promise<void> {
+    this.logger.log('Shutting down cluster...')
+    if (this.renewInterval) {
+      clearInterval(this.renewInterval);
+      this.renewInterval = undefined;
+    }
     if (this.consul && this.isLocalLeader() && this.sessionId) {
+      this.logger.log('Cleaning up leader locks...')
       await this.consul.session.destroy(this.sessionId)
     }
   }
@@ -135,11 +142,16 @@ export class ClusterService
     const { ID } = await this.consul.session.create({
       name: this.serviceId,
       ttl: '15s',
+      behavior: 'delete',
     })
 
-    setInterval(() => {
+    this.renewInterval = setInterval(async () =>{
       if (this.consul) {
-        this.consul.session.renew(ID)
+        try {
+          await this.consul.session.renew(ID)
+        } catch (error) {
+          this.logger.error('Failed to renew consul session', error)
+        }
       }
     }, 10000)
 
@@ -156,7 +168,7 @@ export class ClusterService
         const result = await this.consul.kv.set({
           key: leaderKey,
           value: this.serviceId,
-          acquire: this.sessionId,
+          acquire: this.sessionId
         })
 
         this.isLeader = result

src/main.ts

@@ -27,6 +27,8 @@ async function bootstrap() {
     })
   })
 
+  app.enableShutdownHooks()
+
   const port = process.env.PORT || 3000
   logz.info(`Listening on ${port}`)