fix: release prepared replacement cutover leases (#15)

src/carrier-binding.test.ts

@@ -195,12 +195,12 @@ test("active coordination-bound prepared cutover can terminally reconcile an exa
       expiresAt:new Date(f.clock()+60000).toISOString(),
       restart:{buildReady:{verifiedBy:"independent",verifiedAt:new Date(f.clock()).toISOString(),evidence:"exact package digest"},actuator:"launchd-self" as const,serviceLabel:"test.service",launchdTarget:"gui/501/test.service"},
       finish:{workspaceId:"ws_active",agentId:"agt_active"}};
-    f.store.approveLocal(pairing.pendingId,{...f.contract,scope:[f.root],operations:["cutover_start" as const],cutover});
+    const approved=f.store.approveLocal(pairing.pendingId,{...f.contract,scope:[f.root],operations:["cutover_start" as const],cutover});
     f.store.redeem(context,pairing.credential);
     const config=loadConfig({DEVSPACE_CONFIG_DIR:join(f.root,"config"),DEVSPACE_ALLOWED_ROOTS:f.workspace,DEVSPACE_WORKTREE_ROOT:join(f.root,"worktrees"),DEVSPACE_STATE_DIR:f.root,DEVSPACE_OAUTH_OWNER_TOKEN:"test-owner-token-long-enough",PORT:"1"});
     const manager=new DurableOperationManager(config,undefined,undefined,undefined,f.store.readers);
     try {
-      f.store.prepareEffect(context,planCutoverStart(f.root,cutover).subject);
+      const acquired=f.store.prepareEffect(context,planCutoverStart(f.root,cutover).subject);
       const start=manager.startCutover(cutover,context),id=start.receipt!.cutoverId as string;
       const replacement={serverInstanceId:"replacement",...cutover.expectedIdentity};
       const witness={workspaceQueryable:true,agentQueryable:true,agentReconciled:true,witnessWorkspaceId:cutover.finish.workspaceId,witnessAgentId:cutover.finish.agentId};
@@ -210,6 +210,31 @@ test("active coordination-bound prepared cutover can terminally reconcile an exa
       assert.equal(closed.restartRequest,undefined);
       assert.equal(f.store.ownership.get(closed.coordinationBinding!.leaseId)?.operationHandle,undefined);
       assert.equal(manager.store.getByOperationId(start.operationId)?.receipt?.lifecycleTerminal,true);
+      const terminalHash=cutoverTerminalRecordHash(closed);
+      const terminalLease=f.store.ownership.get(acquired.leaseId)!;
+      assert.equal(terminalLease.operationState,"finished");
+      assert.equal(terminalLease.terminalState,undefined);
+      const revoked=f.store.revokeLocal(approved.id,1);
+      assert.equal(revoked.version,2);
+      const terminalOperation=manager.store.getByOperationId(start.operationId)!;
+      const exactReceipt=terminalOperation.receipt!;
+      const lifecycleAction=exactReceipt.lifecycleAction as {currentIdentity:typeof replacement};
+      f.db.sqlite.prepare("update durable_operations set receipt_json=? where operation_id=?").run(JSON.stringify({
+        ...exactReceipt,lifecycleAction:{...lifecycleAction,currentIdentity:{...replacement,serverInstanceId:"wrong-replacement"}},
+      }),start.operationId);
+      assert.throws(()=>f.store.releaseClosedCutoverLeaseLocal({
+        cutoverId:id,leaseId:acquired.leaseId,expectedLeaseVersion:terminalLease.version,
+        carrierId:approved.id,expectedCarrierVersion:2,expectedTerminalRecordHash:terminalHash,confirmCutoverId:id,
+      }),/positive-witness prepared replacement/i);
+      f.db.sqlite.prepare("update durable_operations set receipt_json=? where operation_id=?").run(JSON.stringify(exactReceipt),start.operationId);
+      const released=f.store.releaseClosedCutoverLeaseLocal({
+        cutoverId:id,leaseId:acquired.leaseId,expectedLeaseVersion:terminalLease.version,
+        carrierId:approved.id,expectedCarrierVersion:2,expectedTerminalRecordHash:terminalHash,confirmCutoverId:id,
+      });
+      assert.equal(released.replayed,false);
+      assert.equal(released.lease.terminalState,"released");
+      assert.equal(released.lease.operationState,"finished");
+      assert.equal(released.lease.operationHandle,undefined);
     } finally {manager.close();}
   } finally {f.close();}
 });

src/carrier-binding.ts

@@ -936,13 +936,16 @@ export class CarrierBindingStore {
     if(!closed || closed.cutoverId!==input.cutoverId || closed.phase!=="closed" || !closed.coordinationBinding) {
       deny("Terminal lease release requires the exact closed coordination-bound cutover");
     }
+    const positiveWitness=closed.reconciliationReceipt?.workspaceQueryable===true &&
+      closed.reconciliationReceipt.agentQueryable===true &&
+      closed.reconciliationReceipt.agentReconciled===true;
+    const normalCompleted=Boolean(closed.drainEvidence && closed.restartRequest?.restartScheduledAt);
+    const preparedReplacementCompleted=!closed.drainEvidence && !closed.restartRequest;
     if(closed.coordinationBinding.leaseId!==input.leaseId ||
        closed.expiredPreparedNoEffect || closed.capabilityExpectationMismatch || closed.observedReplacement ||
-       closed.supersession || closed.bindingRepair ||
-       !closed.drainEvidence || !closed.restartRequest?.restartScheduledAt ||
-       !closed.reconciliationReceipt?.workspaceQueryable || !closed.reconciliationReceipt.agentQueryable ||
-       !closed.reconciliationReceipt.agentReconciled) {
-      deny("Terminal lease release requires one normally completed cutover generation");
+       closed.supersession || closed.bindingRepair || !positiveWitness ||
+       (!normalCompleted && !preparedReplacementCompleted)) {
+      deny("Terminal lease release requires one exact completed cutover generation");
     }
 
     const row=this.database.sqlite.prepare("select * from carrier_bindings where id=?").get(input.carrierId) as BindingRow|undefined;
@@ -986,6 +989,26 @@ export class CarrierBindingStore {
       if(!isDeepStrictEqual(coordinationBinding,correlation) || !isDeepStrictEqual(request,plan.request)) {
         throw new ControlPlaneOwnershipError("CAS_CONFLICT","Terminal lease release durable operation binding changed");
       }
+      if(preparedReplacementCompleted) {
+        const action=operation.receipt?.lifecycleAction as {
+          action?:unknown;
+          cutoverId?:unknown;
+          currentIdentity?:unknown;
+          preferredPair?:unknown;
+        }|undefined;
+        const exactReplacement={
+          serverInstanceId:closed.reconciliationReceipt!.closedByServerInstanceId,
+          ...approved.expectedIdentity,
+        };
+        if(!action || action.action!=="finish" || action.cutoverId!==closed.cutoverId ||
+           !isDeepStrictEqual(action.currentIdentity,exactReplacement) ||
+           !isDeepStrictEqual(action.preferredPair,approved.finish) ||
+           closed.reconciliationReceipt!.witnessWorkspaceId!==approved.finish.workspaceId ||
+           closed.reconciliationReceipt!.witnessAgentId!==approved.finish.agentId ||
+           exactReplacement.serverInstanceId===approved.currentIdentity.serverInstanceId) {
+          deny("Terminal lease release requires exact positive-witness prepared replacement evidence");
+        }
+      }
 
       const localContext=Object.freeze({});
       const grant:GrantEvidenceReference={