feat(FN-3922): enforce task skip on credential-failure heartbeat

Credential-missing recovery is now documented and enforced in the agent heartbeat layer, with comprehensive test coverage for both timer-triggered and assignment-triggered credential failure scenarios. Fusion-Task-Id: FN-3922
2026-05-10 05:43:00 -07:00
parent 0fa5af3cfe
commit 246b2b403c
4 changed files with 169 additions and 27 deletions
--- a/packages/engine/src/tests/heartbeat-executor.test.ts
+++ b/packages/engine/src/tests/heartbeat-executor.test.ts
@@ -2545,11 +2545,84 @@ describe("executeHeartbeat", () => {
      expect(result.resultJson).toMatchObject({
        reason: "heartbeat_model_unavailable",
        source: "timer",
+        detail: expect.stringContaining("No API key for provider: anthropic"),
      });
+      expect(result.stderrExcerpt).toContain("No API key for provider: anthropic");
      expect(store.updateAgentState).toHaveBeenCalledWith("agent-001", "active");
      expect(store.updateAgentState).not.toHaveBeenCalledWith("agent-001", "error");
    });

+    it.each(["on_demand", "assignment"] as const)("pauses on %s heartbeat when model provider credentials are unavailable", async (source) => {
+      const store = createStoreWithAgentForExec();
+      mockedCreateFnAgent.mockRejectedValue(new Error("No API key for provider: anthropic"));
+
+      const monitor = new HeartbeatMonitor({ store, taskStore: mockTaskStore, rootDir: "/tmp" });
+
+      const result = await monitor.executeHeartbeat({ agentId: "agent-001", source });
+
+      expect(result.status).toBe("completed");
+      expect(result.resultJson).toMatchObject({
+        reason: "heartbeat_model_unavailable",
+        source,
+        actionRequired: true,
+        detail: expect.stringContaining("Configure credentials for provider \"anthropic\""),
+      });
+      expect(result.stderrExcerpt).toContain("No API key for provider: anthropic");
+      expect(store.updateAgentState).toHaveBeenCalledWith("agent-001", "paused");
+      expect(store.updateAgent).toHaveBeenCalledWith("agent-001", {
+        pauseReason: "heartbeat-model-unavailable",
+        lastError: expect.stringContaining("No API key for provider: anthropic"),
+      });
+      expect(store.updateAgentState).not.toHaveBeenCalledWith("agent-001", "error");
+    });
+
+    it("keeps timer-triggered credential failures in recoverable state across consecutive wakeups", async () => {
+      const store = createStoreWithAgentForExec();
+      mockedCreateFnAgent.mockRejectedValue(new Error("No API key for provider: anthropic"));
+
+      const monitor = new HeartbeatMonitor({ store, taskStore: mockTaskStore, rootDir: "/tmp" });
+
+      const first = await monitor.executeHeartbeat({ agentId: "agent-001", source: "timer" });
+      const second = await monitor.executeHeartbeat({ agentId: "agent-001", source: "timer" });
+
+      for (const run of [first, second]) {
+        expect(run.status).toBe("completed");
+        expect(run.resultJson).toMatchObject({
+          reason: "heartbeat_model_unavailable",
+          source: "timer",
+          detail: expect.stringContaining("No API key for provider: anthropic"),
+        });
+        expect(run.stderrExcerpt).toContain("No API key for provider: anthropic");
+      }
+
+      expect(store.updateAgentState).toHaveBeenCalledWith("agent-001", "active");
+      expect(store.updateAgentState).not.toHaveBeenCalledWith("agent-001", "error");
+    });
+
+    it("keeps non-timer credential failures recoverable on consecutive wakeups", async () => {
+      const store = createStoreWithAgentForExec();
+      mockedCreateFnAgent.mockRejectedValue(new Error("No API key for provider: anthropic"));
+
+      const monitor = new HeartbeatMonitor({ store, taskStore: mockTaskStore, rootDir: "/tmp" });
+
+      const first = await monitor.executeHeartbeat({ agentId: "agent-001", source: "assignment" });
+      const second = await monitor.executeHeartbeat({ agentId: "agent-001", source: "assignment" });
+
+      expect(first.status).toBe("completed");
+      expect(first.resultJson).toMatchObject({
+        reason: "heartbeat_model_unavailable",
+        source: "assignment",
+        actionRequired: true,
+      });
+      expect(second.status).toBe("completed");
+      expect(second.resultJson).toMatchObject({
+        reason: "heartbeat_model_unavailable",
+        source: "assignment",
+        actionRequired: true,
+      });
+      expect(store.updateAgentState).not.toHaveBeenCalledWith("agent-001", "error");
+    });
+
    it("completes run as failed when promptWithFallback throws", async () => {
      const store = createStoreWithAgentForExec();
      const mockSession = createMockAgentSession();
--- a/packages/engine/src/agent-heartbeat.ts
+++ b/packages/engine/src/agent-heartbeat.ts
@@ -1877,14 +1877,61 @@ export class HeartbeatMonitor {
          });
        }

-        const resolveFailSoftProviderError = (errorMessage: string): boolean => {
-          if (source !== "timer") return false;
+        const isModelUnavailableError = (errorMessage: string): boolean => {
          const normalized = errorMessage.toLowerCase();
          return normalized.includes("no api key for provider")
            || normalized.includes("configured primary model")
            || normalized.includes("was not found in the pi model registry");
        };

+        const extractUnavailableProvider = (errorMessage: string): string | undefined => {
+          const providerMatch = /no api key for provider:\s*([^\s)]+)/i.exec(errorMessage);
+          if (providerMatch?.[1]) return providerMatch[1];
+          const modelMatch = /configured primary model\s+([^/\s]+)\//i.exec(errorMessage);
+          if (modelMatch?.[1]) return modelMatch[1];
+          return undefined;
+        };
+
+        const completeAsModelUnavailable = async (errorMessage: string): Promise<void> => {
+          const provider = extractUnavailableProvider(errorMessage);
+          const detail = provider
+            ? `${errorMessage}. Configure credentials for provider "${provider}" in settings, then resume the agent.`
+            : `${errorMessage}. Configure valid provider credentials in settings, then resume the agent.`;
+
+          if (source === "timer") {
+            await this.completeRun(agentId, run.id, {
+              status: "completed",
+              resultJson: {
+                reason: "heartbeat_model_unavailable",
+                source,
+                detail,
+              },
+              stderrExcerpt: detail,
+              stdoutExcerpt: stdoutExcerpt || undefined,
+            });
+            return;
+          }
+
+          await this.completeRun(agentId, run.id, {
+            status: "completed",
+            resultJson: {
+              reason: "heartbeat_model_unavailable",
+              source,
+              detail,
+              actionRequired: true,
+            },
+            stderrExcerpt: detail,
+            stdoutExcerpt: stdoutExcerpt || undefined,
+            skipStateTransition: true,
+          });
+
+          await this.store.updateAgentState(agentId, "paused");
+          await this.store.updateAgent(agentId, {
+            pauseReason: "heartbeat-model-unavailable",
+            lastError: detail,
+          });
+        };
+
        let heartbeatModelSettings: Settings | undefined;
        try {
          heartbeatModelSettings = await taskStore.getSettings();
@@ -2241,17 +2288,8 @@ export class HeartbeatMonitor {
          heartbeatLog.error(`Heartbeat execution failed for ${agentId}: ${errorDetail}`);
          await flushAgentLogger();

-          if (resolveFailSoftProviderError(errorDetail)) {
-            await this.completeRun(agentId, run.id, {
-              status: "completed",
-              resultJson: {
-                reason: "heartbeat_model_unavailable",
-                source,
-                detail: errorDetail,
-              },
-              stderrExcerpt: errorDetail,
-              stdoutExcerpt: stdoutExcerpt || undefined,
-            });
+          if (isModelUnavailableError(errorDetail)) {
+            await completeAsModelUnavailable(errorDetail);
          } else {
            await this.completeRun(agentId, run.id, {
              status: "failed",
@@ -2282,30 +2320,56 @@ export class HeartbeatMonitor {
        await flushAgentLogger();

        const normalizedError = errorDetail.toLowerCase();
-        const shouldFailSoft = source === "timer" && (
-          normalizedError.includes("no api key for provider")
+        const isModelUnavailable = normalizedError.includes("no api key for provider")
          || normalizedError.includes("configured primary model")
-          || normalizedError.includes("was not found in the pi model registry")
-        );
+          || normalizedError.includes("was not found in the pi model registry");

        // Attempt to complete the run if it's still active.
        // If completeRun also fails, fall back to a direct DB update to ensure
        // the run is not permanently stuck in "active" state.
        try {
-          await this.completeRun(agentId, run.id, shouldFailSoft
-            ? {
+          if (isModelUnavailable) {
+            const providerMatch = /no api key for provider:\s*([^\s)]+)/i.exec(errorDetail);
+            const modelMatch = /configured primary model\s+([^/\s]+)\//i.exec(errorDetail);
+            const provider = providerMatch?.[1] ?? modelMatch?.[1];
+            const detail = provider
+              ? `${errorDetail}. Configure credentials for provider "${provider}" in settings, then resume the agent.`
+              : `${errorDetail}. Configure valid provider credentials in settings, then resume the agent.`;
+
+            if (source === "timer") {
+              await this.completeRun(agentId, run.id, {
                status: "completed",
                resultJson: {
                  reason: "heartbeat_model_unavailable",
                  source,
-                  detail: errorDetail,
+                  detail,
                },
-                stderrExcerpt: errorDetail,
-              }
-            : {
-                status: "failed",
-                stderrExcerpt: errorDetail,
+                stderrExcerpt: detail,
              });
+            } else {
+              await this.completeRun(agentId, run.id, {
+                status: "completed",
+                resultJson: {
+                  reason: "heartbeat_model_unavailable",
+                  source,
+                  detail,
+                  actionRequired: true,
+                },
+                stderrExcerpt: detail,
+                skipStateTransition: true,
+              });
+              await this.store.updateAgentState(agentId, "paused");
+              await this.store.updateAgent(agentId, {
+                pauseReason: "heartbeat-model-unavailable",
+                lastError: detail,
+              });
+            }
+          } else {
+            await this.completeRun(agentId, run.id, {
+              status: "failed",
+              stderrExcerpt: errorDetail,
+            });
+          }
        } catch (completeRunErr) {
          const completeRunErrMsg = completeRunErr instanceof Error ? completeRunErr.message : String(completeRunErr);
          heartbeatLog.error(`completeRun failed for ${agentId}/${run.id}: ${completeRunErrMsg} — attempting safety-net completion`);