feat(FN-3102): add plugin contribution types, scheduled evaluator, and agen
This merge adds a new evaluation framework including signal collection (`eval-signal-collector.ts`), typed evaluation signals (`eval-types.ts`), and an evaluator engine module (`evaluator.ts`) with cron-runner integration for scheduled evaluation. It also establishes plugin contribution type constra Fusion-Task-Id: FN-3102
This commit is contained in:
Fusion
173
packages/core/src/__tests__/plugin-contribution-types.test.ts
Normal file
173
packages/core/src/__tests__/plugin-contribution-types.test.ts
Normal file
@@ -0,0 +1,173 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type {
|
||||
FusionPlugin,
|
||||
PluginPromptContributions,
|
||||
PluginPromptSurface,
|
||||
PluginSetupCheckResult,
|
||||
PluginSetupHooks,
|
||||
PluginSetupManifest,
|
||||
PluginSkillContribution,
|
||||
PluginWorkflowStepContribution,
|
||||
} from "../plugin-types.js";
|
||||
import { validatePluginManifest } from "../plugin-types.js";
|
||||
|
||||
describe("plugin contribution type constraints", () => {
|
||||
it("accepts setup check result status variants", () => {
|
||||
const installed: PluginSetupCheckResult = {
|
||||
status: "installed",
|
||||
version: "1.0.0",
|
||||
binaryPath: "/usr/local/bin/agent-browser",
|
||||
};
|
||||
const notInstalled: PluginSetupCheckResult = { status: "not-installed" };
|
||||
const error: PluginSetupCheckResult = { status: "error", error: "probe failed" };
|
||||
|
||||
expect(installed.binaryPath).toContain("agent-browser");
|
||||
expect(notInstalled.status).toBe("not-installed");
|
||||
expect(error.error).toBe("probe failed");
|
||||
});
|
||||
|
||||
it("supports contribution defaults as optional fields on skill/workflow types", () => {
|
||||
const skill: PluginSkillContribution = {
|
||||
skillId: "browser-navigation",
|
||||
name: "Browser Navigation",
|
||||
description: "Navigate pages",
|
||||
skillFiles: ["skills/browser-navigation/SKILL.md"],
|
||||
};
|
||||
const workflow: PluginWorkflowStepContribution = {
|
||||
stepId: "browser-verification",
|
||||
name: "Browser Verification",
|
||||
description: "Verify with browser",
|
||||
mode: "prompt",
|
||||
prompt: "Verify page behavior",
|
||||
};
|
||||
|
||||
expect(skill.enabled).toBeUndefined();
|
||||
expect(workflow.defaultOn).toBeUndefined();
|
||||
});
|
||||
|
||||
it("accepts explicit enabled flags and workflow phase/toolMode variants", () => {
|
||||
const enabledSkill: PluginSkillContribution = {
|
||||
skillId: "browser-interaction",
|
||||
name: "Browser Interaction",
|
||||
description: "Interact with pages",
|
||||
skillFiles: ["skills/browser-interaction/SKILL.md"],
|
||||
enabled: true,
|
||||
triggerPatterns: ["click", "type"],
|
||||
};
|
||||
const disabledStep: PluginWorkflowStepContribution = {
|
||||
stepId: "browser-post-merge-check",
|
||||
name: "Browser Post-Merge Check",
|
||||
description: "Post merge browser check",
|
||||
mode: "script",
|
||||
scriptName: "browser-check",
|
||||
enabled: false,
|
||||
phase: "post-merge",
|
||||
toolMode: "full-access",
|
||||
};
|
||||
|
||||
expect(enabledSkill.enabled).toBe(true);
|
||||
expect(disabledStep.enabled).toBe(false);
|
||||
expect(disabledStep.phase).toBe("post-merge");
|
||||
expect(disabledStep.toolMode).toBe("full-access");
|
||||
});
|
||||
|
||||
it("accepts setup manifest/hooks and extended FusionPlugin shape", async () => {
|
||||
const setupManifests: PluginSetupManifest[] = [
|
||||
{ binaryName: "agent-browser", description: "Browser runtime", version: "1.2.3", channel: "stable", defaultTimeoutMs: 120000 },
|
||||
{ binaryName: "agent-browser", description: "Browser runtime", channel: "beta", defaultTimeoutMs: 120000 },
|
||||
{ binaryName: "agent-browser", description: "Browser runtime", channel: "nightly", defaultTimeoutMs: 120000 },
|
||||
];
|
||||
const setupManifest = setupManifests[0]!;
|
||||
const setupHooks: PluginSetupHooks = {
|
||||
checkSetup: async () => ({ status: "installed", version: "1.2.3", binaryPath: "/tmp/agent-browser" }),
|
||||
install: async () => {},
|
||||
uninstall: async () => {},
|
||||
};
|
||||
const minimalHooks: PluginSetupHooks = {
|
||||
checkSetup: async () => ({ status: "not-installed" }),
|
||||
};
|
||||
|
||||
const plugin: FusionPlugin = {
|
||||
manifest: { id: "plugin-a", name: "Plugin A", version: "1.0.0" },
|
||||
state: "installed",
|
||||
hooks: {},
|
||||
skills: [{ skillId: "browser-extraction", name: "Browser Extraction", description: "Extract data", skillFiles: ["skills/extract/SKILL.md"] }],
|
||||
workflowSteps: [
|
||||
{
|
||||
stepId: "browser-qa",
|
||||
name: "Browser QA",
|
||||
description: "QA in browser",
|
||||
mode: "script",
|
||||
scriptName: "verify-browser",
|
||||
phase: "pre-merge",
|
||||
toolMode: "readonly",
|
||||
},
|
||||
],
|
||||
promptContributions: {
|
||||
enabledByDefault: false,
|
||||
contributions: [{ surface: "reviewer", content: "Review browser assumptions" }],
|
||||
},
|
||||
setup: { manifest: setupManifest, hooks: setupHooks },
|
||||
};
|
||||
|
||||
const check = await plugin.setup!.hooks.checkSetup({} as never);
|
||||
expect(setupManifests).toHaveLength(3);
|
||||
expect(plugin.setup?.manifest.channel).toBe("stable");
|
||||
expect(check.status).toBe("installed");
|
||||
expect((await minimalHooks.checkSetup({} as never)).status).toBe("not-installed");
|
||||
});
|
||||
|
||||
it("accepts prompt surface union and prompt contribution records", () => {
|
||||
const surfaces: PluginPromptSurface[] = ["executor-system", "executor-task", "triage", "reviewer", "heartbeat"];
|
||||
const byPlugin: Record<string, PluginPromptContributions> = {
|
||||
"fusion-plugin-agent-browser": {
|
||||
enabledByDefault: false,
|
||||
contributions: surfaces.map((surface) => ({ surface, content: `${surface} content` })),
|
||||
},
|
||||
};
|
||||
|
||||
expect(byPlugin["fusion-plugin-agent-browser"]?.contributions).toHaveLength(5);
|
||||
});
|
||||
|
||||
it("compile-time rejects invalid prompt surfaces", () => {
|
||||
const validSurface: PluginPromptSurface = "triage";
|
||||
expect(validSurface).toBe("triage");
|
||||
|
||||
// @ts-expect-error invalid PluginPromptSurface
|
||||
const invalidSurface: PluginPromptSurface = "invalid-surface";
|
||||
expect(invalidSurface).toBe("invalid-surface");
|
||||
});
|
||||
});
|
||||
|
||||
describe("validatePluginManifest contribution metadata scope", () => {
|
||||
// validatePluginManifest only validates manifest-level contribution metadata,
|
||||
// not full FusionPlugin nested contribution object shapes.
|
||||
it("accepts valid contribution metadata", () => {
|
||||
const valid = validatePluginManifest({
|
||||
id: "plugin-a",
|
||||
name: "Plugin A",
|
||||
version: "1.0.0",
|
||||
skills: [{ skillId: "browser-reader", name: "Browser Reader" }],
|
||||
workflowSteps: [{ stepId: "browser-check", name: "Browser Check", mode: "prompt" }],
|
||||
promptSurfaces: ["executor-system", "heartbeat"],
|
||||
setup: { binaryName: "agent-browser", description: "Browser runtime", channel: "stable" },
|
||||
});
|
||||
|
||||
expect(valid.valid).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects malformed contribution metadata", () => {
|
||||
const invalid = validatePluginManifest({
|
||||
id: "plugin-a",
|
||||
name: "Plugin A",
|
||||
version: "1.0.0",
|
||||
skills: [{ skillId: "Bad Skill", name: "Bad" }],
|
||||
workflowSteps: [{ stepId: "bad-step", name: "Bad Step", mode: "oops" as "prompt" }],
|
||||
promptSurfaces: ["not-a-surface" as PluginPromptSurface],
|
||||
setup: { binaryName: "", description: "" },
|
||||
});
|
||||
|
||||
expect(invalid.valid).toBe(false);
|
||||
expect(invalid.errors.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
142
packages/core/src/__tests__/plugin-loader-contributions.test.ts
Normal file
142
packages/core/src/__tests__/plugin-loader-contributions.test.ts
Normal file
@@ -0,0 +1,142 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { mkdtempSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { PluginLoader } from "../plugin-loader.js";
|
||||
import { PluginStore } from "../plugin-store.js";
|
||||
import type { FusionPlugin, PluginManifest } from "../plugin-types.js";
|
||||
|
||||
function makeManifest(overrides: Partial<PluginManifest> = {}): PluginManifest {
|
||||
return { id: "test-plugin", name: "Test Plugin", version: "1.0.0", ...overrides };
|
||||
}
|
||||
|
||||
function makePlugin(manifest: PluginManifest): FusionPlugin {
|
||||
return { manifest, state: "installed", hooks: {}, tools: [], routes: [] };
|
||||
}
|
||||
|
||||
async function writePluginModule(dir: string, filename: string, plugin: FusionPlugin): Promise<string> {
|
||||
const filepath = join(dir, filename);
|
||||
await mkdir(dir, { recursive: true });
|
||||
await writeFile(
|
||||
filepath,
|
||||
`const plugin = ${JSON.stringify(plugin, null, 2)}; export default plugin; export { plugin };`,
|
||||
);
|
||||
return filepath;
|
||||
}
|
||||
|
||||
const hasContributionApis =
|
||||
"getPluginSkills" in PluginLoader.prototype &&
|
||||
"getPluginWorkflowSteps" in PluginLoader.prototype &&
|
||||
"getPluginPromptContributions" in PluginLoader.prototype &&
|
||||
"getPluginSetupInfo" in PluginLoader.prototype;
|
||||
|
||||
describe.skipIf(!hasContributionApis)("PluginLoader contribution loading", () => {
|
||||
let rootDir: string;
|
||||
let pluginStore: PluginStore;
|
||||
let loader: PluginLoader;
|
||||
|
||||
beforeEach(() => {
|
||||
rootDir = mkdtempSync(join(tmpdir(), "kb-plugin-loader-contrib-"));
|
||||
pluginStore = new PluginStore(rootDir, { inMemoryDb: true });
|
||||
loader = new PluginLoader({ pluginStore, taskStore: { logActivity: vi.fn() } as any });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
const { rm } = await import("node:fs/promises");
|
||||
await rm(rootDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("aggregates skills/workflow/prompts with plugin ownership", async () => {
|
||||
await pluginStore.init();
|
||||
const pluginDir = join(rootDir, "plugins");
|
||||
|
||||
const alpha = makePlugin(
|
||||
makeManifest({ id: "plugin-alpha", skills: [{ skillId: "alpha", name: "Alpha" }], workflowSteps: [{ stepId: "wf-alpha", name: "WF Alpha", mode: "prompt" }], promptSurfaces: ["triage"] }),
|
||||
);
|
||||
alpha.skills = [{ skillId: "alpha", name: "Alpha", description: "alpha", enabled: false } as any];
|
||||
alpha.workflowSteps = [{ stepId: "wf-alpha", name: "WF Alpha", description: "wf", mode: "prompt", prompt: "Run", enabled: false } as any];
|
||||
alpha.promptContributions = { enabledByDefault: false, contributions: [{ surface: "triage", content: "Alpha triage" }] };
|
||||
|
||||
const beta = makePlugin(makeManifest({ id: "plugin-beta" }));
|
||||
beta.skills = [{ skillId: "beta", name: "Beta", description: "beta", enabled: true } as any];
|
||||
beta.workflowSteps = [{ stepId: "wf-beta", name: "WF Beta", description: "wf", mode: "script", scriptName: "test" } as any];
|
||||
beta.promptContributions = { enabledByDefault: true, contributions: [{ surface: "reviewer", content: "Beta reviewer" }] };
|
||||
|
||||
const alphaPath = await writePluginModule(pluginDir, "alpha.mjs", alpha);
|
||||
const betaPath = await writePluginModule(pluginDir, "beta.mjs", beta);
|
||||
|
||||
await pluginStore.registerPlugin({ manifest: alpha.manifest, path: alphaPath });
|
||||
await pluginStore.registerPlugin({ manifest: beta.manifest, path: betaPath });
|
||||
await loader.loadAllPlugins();
|
||||
|
||||
const skills = loader.getPluginSkills();
|
||||
const steps = loader.getPluginWorkflowSteps();
|
||||
const prompts = loader.getPluginPromptContributions();
|
||||
|
||||
expect(skills.map((s) => s.pluginId).sort()).toEqual(["plugin-alpha", "plugin-beta"]);
|
||||
expect(steps.map((s) => s.pluginId).sort()).toEqual(["plugin-alpha", "plugin-beta"]);
|
||||
expect(prompts.map((p) => p.pluginId).sort()).toEqual(["plugin-alpha", "plugin-beta"]);
|
||||
expect(skills.some((s) => s.skill.enabled === false)).toBe(true);
|
||||
expect(steps.some((s) => s.step.enabled === false)).toBe(true);
|
||||
});
|
||||
|
||||
it("removes contributions when stopping and refreshes when loaded again", async () => {
|
||||
await pluginStore.init();
|
||||
const pluginDir = join(rootDir, "plugins");
|
||||
const manifest = makeManifest({ id: "plugin-reload" });
|
||||
const plugin = makePlugin(manifest);
|
||||
plugin.skills = [{ skillId: "before", name: "Before", description: "before" } as any];
|
||||
|
||||
const path = await writePluginModule(pluginDir, "reload.mjs", plugin);
|
||||
await pluginStore.registerPlugin({ manifest, path });
|
||||
await loader.loadAllPlugins();
|
||||
|
||||
expect(loader.getPluginSkills().some((s) => s.skill.skillId === "before")).toBe(true);
|
||||
|
||||
await loader.stopPlugin("plugin-reload");
|
||||
expect(loader.getPluginSkills().some((s) => s.pluginId === "plugin-reload")).toBe(false);
|
||||
|
||||
const updated = makePlugin(manifest);
|
||||
updated.skills = [{ skillId: "after", name: "After", description: "after" } as any];
|
||||
await writePluginModule(pluginDir, "reload.mjs", updated);
|
||||
|
||||
await loader.loadPlugin("plugin-reload");
|
||||
expect(loader.getPluginSkills().some((s) => s.skill.skillId === "after")).toBe(true);
|
||||
});
|
||||
|
||||
it("provides setup info and delegates check/install hooks", async () => {
|
||||
await pluginStore.init();
|
||||
const pluginDir = join(rootDir, "plugins");
|
||||
const modulePath = join(pluginDir, "setup.mjs");
|
||||
await mkdir(pluginDir, { recursive: true });
|
||||
await writeFile(
|
||||
modulePath,
|
||||
`
|
||||
const plugin = {
|
||||
manifest: { id: "plugin-setup", name: "Plugin Setup", version: "1.0.0" },
|
||||
state: "installed",
|
||||
hooks: {},
|
||||
setup: {
|
||||
manifest: { binaryName: "agent-browser", description: "browser", defaultTimeoutMs: 5000 },
|
||||
hooks: {
|
||||
checkSetup: async () => ({ status: "installed", version: "1.0.0", binaryPath: "/tmp/agent-browser" }),
|
||||
install: async () => ({ ok: true }),
|
||||
},
|
||||
},
|
||||
};
|
||||
export default plugin;
|
||||
`,
|
||||
);
|
||||
|
||||
await pluginStore.registerPlugin({ manifest: { id: "plugin-setup", name: "Plugin Setup", version: "1.0.0" }, path: modulePath });
|
||||
await loader.loadAllPlugins();
|
||||
|
||||
const setupInfo = loader.getPluginSetupInfo();
|
||||
const check = await loader.checkPluginSetup("plugin-setup");
|
||||
await expect(loader.installPluginSetup("plugin-setup")).resolves.toBeUndefined();
|
||||
|
||||
expect(setupInfo).toHaveLength(1);
|
||||
expect(check.status).toBe("installed");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user