feat(FN-3896): support companies.sh repo subpaths in company imports
- Add secure archive subPath handling to company package parsing, including traversal/absolute-path validation - Parse GitHub website tree URLs from companies.sh metadata to derive preferred branch and nested subPath - Wire dashboard company import flow to download the correct branch archive and parse nested manifests with clear API errors - Add parser and route tests for subPath success/failure cases and include changeset for @runfusion/fusion Fusion-Task-Id: FN-3896
This commit is contained in:
Fusion
5
.changeset/FN-3896-companies-sh-subpath.md
Normal file
5
.changeset/FN-3896-companies-sh-subpath.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"@runfusion/fusion": patch
|
||||
---
|
||||
|
||||
Fix agent-company imports from companies.sh monorepos by honoring the catalog subdirectory path (for example `paperclipai/companies/gstack`) instead of parsing the alphabetically first sibling package.
|
||||
@@ -502,6 +502,64 @@ name: Nested Archive CEO
|
||||
expect(pkg.company?.name).toBe("Flat Zip Co");
|
||||
});
|
||||
|
||||
it("uses subPath to select the correct company from a monorepo archive", async () => {
|
||||
const root = createTempDir();
|
||||
const archivePath = join(root, "mono.zip");
|
||||
|
||||
createZipFromEntries(archivePath, [
|
||||
{ path: "companies-main/aeon/COMPANY.md", content: `---\nname: Aeon\n---` },
|
||||
{ path: "companies-main/gstack/COMPANY.md", content: `---\nname: GStack\n---` },
|
||||
{ path: "companies-main/gstack/agents/ceo/AGENTS.md", content: `---\nname: GStack CEO\n---` },
|
||||
]);
|
||||
|
||||
const pkg = await parseCompanyArchive(archivePath, { subPath: "gstack" });
|
||||
expect(pkg.company?.name).toBe("GStack");
|
||||
expect(pkg.company?.name).not.toBe("Aeon");
|
||||
expect(pkg.agents[0]?.name).toBe("GStack CEO");
|
||||
});
|
||||
|
||||
it("throws when subPath does not contain COMPANY.md", async () => {
|
||||
const root = createTempDir();
|
||||
const archivePath = join(root, "missing-subpath.zip");
|
||||
|
||||
createZipFromEntries(archivePath, [
|
||||
{ path: "companies-main/aeon/COMPANY.md", content: `---\nname: Aeon\n---` },
|
||||
]);
|
||||
|
||||
await expect(parseCompanyArchive(archivePath, { subPath: "gstack" })).rejects.toThrow(
|
||||
AgentCompaniesParseError,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects invalid subPath values", async () => {
|
||||
const root = createTempDir();
|
||||
const archivePath = join(root, "invalid-subpath.zip");
|
||||
|
||||
createZipFromEntries(archivePath, [
|
||||
{ path: "companies-main/gstack/COMPANY.md", content: `---\nname: GStack\n---` },
|
||||
]);
|
||||
|
||||
await expect(parseCompanyArchive(archivePath, { subPath: "../etc" })).rejects.toThrow(
|
||||
AgentCompaniesParseError,
|
||||
);
|
||||
await expect(parseCompanyArchive(archivePath, { subPath: "/abs" })).rejects.toThrow(
|
||||
AgentCompaniesParseError,
|
||||
);
|
||||
});
|
||||
|
||||
it("preserves legacy extraction behavior when subPath is omitted", async () => {
|
||||
const root = createTempDir();
|
||||
const archivePath = join(root, "legacy-behavior.zip");
|
||||
|
||||
createZipFromEntries(archivePath, [
|
||||
{ path: "companies-main/aeon/COMPANY.md", content: `---\nname: Aeon\n---` },
|
||||
{ path: "companies-main/gstack/COMPANY.md", content: `---\nname: GStack\n---` },
|
||||
]);
|
||||
|
||||
const pkg = await parseCompanyArchive(archivePath);
|
||||
expect(pkg.company?.name).toBe("Aeon");
|
||||
});
|
||||
|
||||
it("throws for unsupported archive extension", async () => {
|
||||
const root = createTempDir();
|
||||
const archivePath = join(root, "company.rar");
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { isAbsolute, join, normalize, resolve } from "node:path";
|
||||
|
||||
import extractZip from "extract-zip";
|
||||
import { parse as parseYaml } from "yaml";
|
||||
@@ -488,7 +488,42 @@ async function extractTarArchive(archivePath: string, outputDir: string): Promis
|
||||
await execFileAsync("tar", ["xzf", archivePath, "-C", outputDir]);
|
||||
}
|
||||
|
||||
export async function parseCompanyArchive(archivePath: string): Promise<AgentCompaniesPackage> {
|
||||
function sanitizeCompanySubPath(subPath: string): string {
|
||||
const trimmed = subPath.trim();
|
||||
if (trimmed.length === 0) {
|
||||
throw new AgentCompaniesParseError("subPath must not be empty");
|
||||
}
|
||||
if (trimmed.includes("\\")) {
|
||||
throw new AgentCompaniesParseError(`Invalid subPath "${subPath}": backslashes are not allowed`);
|
||||
}
|
||||
if (isAbsolute(trimmed)) {
|
||||
throw new AgentCompaniesParseError(`Invalid subPath "${subPath}": absolute paths are not allowed`);
|
||||
}
|
||||
|
||||
const normalized = normalize(trimmed).replace(/^\/+/, "");
|
||||
if (normalized === "" || normalized === "." || normalized.split("/").some((segment) => segment === "..")) {
|
||||
throw new AgentCompaniesParseError(`Invalid subPath "${subPath}": path traversal is not allowed`);
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function findArchiveWrapperRoot(tempDir: string): string {
|
||||
const directories = readdirSync(tempDir, { withFileTypes: true }).filter((entry) =>
|
||||
entry.isDirectory(),
|
||||
);
|
||||
|
||||
if (directories.length === 1) {
|
||||
return join(tempDir, directories[0].name);
|
||||
}
|
||||
|
||||
return tempDir;
|
||||
}
|
||||
|
||||
export async function parseCompanyArchive(
|
||||
archivePath: string,
|
||||
options?: { subPath?: string },
|
||||
): Promise<AgentCompaniesPackage> {
|
||||
const resolvedArchivePath = resolve(archivePath);
|
||||
const tempDir = mkdtempSync(join(tmpdir(), "agent-companies-"));
|
||||
|
||||
@@ -503,6 +538,19 @@ export async function parseCompanyArchive(archivePath: string): Promise<AgentCom
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof options?.subPath === "string") {
|
||||
const wrapperRoot = findArchiveWrapperRoot(tempDir);
|
||||
const sanitizedSubPath = sanitizeCompanySubPath(options.subPath);
|
||||
const candidateRoot = join(wrapperRoot, sanitizedSubPath);
|
||||
const companyManifestPath = join(candidateRoot, "COMPANY.md");
|
||||
if (!existsSync(companyManifestPath)) {
|
||||
throw new AgentCompaniesParseError(
|
||||
`Company manifest not found at archive subPath "${sanitizedSubPath}" (expected ${companyManifestPath})`,
|
||||
);
|
||||
}
|
||||
return parseCompanyDirectory(candidateRoot);
|
||||
}
|
||||
|
||||
return parseCompanyDirectory(resolveExtractionRoot(tempDir));
|
||||
} catch (error) {
|
||||
if (error instanceof AgentCompaniesParseError) {
|
||||
|
||||
@@ -576,7 +576,7 @@ describe("POST /api/agents/import", () => {
|
||||
const url = String(input);
|
||||
if (url === "https://companies.sh/api/companies") {
|
||||
return Promise.resolve(new Response(
|
||||
JSON.stringify({ items: [{ slug: "acme-ai", name: "Acme AI", repo: "acme/reviewers" }] }),
|
||||
JSON.stringify({ items: [{ slug: "acme-ai", name: "Acme AI", repo: "acme/reviewers", website: "https://github.com/acme/reviewers/tree/main" }] }),
|
||||
{
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
@@ -618,6 +618,109 @@ describe("POST /api/agents/import", () => {
|
||||
expect(body.skills).toEqual([{ name: "review", description: "Review implementation details" }]);
|
||||
});
|
||||
|
||||
it("passes website-derived subPath for monorepo company imports", async () => {
|
||||
const archiveBuffer = createGitHubArchiveBuffer("gstack");
|
||||
|
||||
globalThis.fetch = vi.fn().mockImplementation((input: unknown) => {
|
||||
const url = String(input);
|
||||
if (url === "https://companies.sh/api/companies") {
|
||||
return Promise.resolve(new Response(
|
||||
JSON.stringify({
|
||||
items: [
|
||||
{ slug: "gstack", name: "GStack", repo: "paperclipai/companies", website: "https://github.com/paperclipai/companies/tree/main/gstack" },
|
||||
{ slug: "aeon", name: "Aeon", repo: "paperclipai/companies", website: "https://github.com/paperclipai/companies/tree/main/aeon" },
|
||||
],
|
||||
}),
|
||||
{ status: 200, headers: { "content-type": "application/json" } },
|
||||
));
|
||||
}
|
||||
|
||||
if (url === "https://github.com/paperclipai/companies/archive/refs/heads/main.tar.gz") {
|
||||
return Promise.resolve(new Response(archiveBuffer, {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/gzip" },
|
||||
}));
|
||||
}
|
||||
|
||||
return Promise.reject(new Error(`Unexpected fetch URL: ${url}`));
|
||||
});
|
||||
|
||||
mockParseCompanyArchive.mockImplementation(async (_archivePath: string, options?: { subPath?: string }) => {
|
||||
expect(options).toEqual({ subPath: "gstack" });
|
||||
if (options?.subPath === "gstack") {
|
||||
return {
|
||||
company: { name: "GStack", slug: "gstack" },
|
||||
agents: [{ name: "GStack Agent" }],
|
||||
teams: [],
|
||||
projects: [],
|
||||
tasks: [],
|
||||
skills: [],
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
company: { name: "Aeon", slug: "aeon" },
|
||||
agents: [{ name: "Aeon Agent" }],
|
||||
teams: [],
|
||||
projects: [],
|
||||
tasks: [],
|
||||
skills: [],
|
||||
};
|
||||
});
|
||||
|
||||
const response = await postImport(app, {
|
||||
importSource: "companies.sh",
|
||||
companySlug: "gstack",
|
||||
dryRun: true,
|
||||
});
|
||||
|
||||
expect(response.status, JSON.stringify(response.body)).toBe(200);
|
||||
const body = response.body as any;
|
||||
expect(body.companyName).toBe("GStack");
|
||||
expect(body.companySlug).toBe("gstack");
|
||||
});
|
||||
|
||||
it("omits subPath when website is absent or invalid", async () => {
|
||||
const archiveBuffer = createGitHubArchiveBuffer("acme-ai");
|
||||
|
||||
globalThis.fetch = vi.fn().mockImplementation((input: unknown) => {
|
||||
const url = String(input);
|
||||
if (url === "https://companies.sh/api/companies") {
|
||||
return Promise.resolve(new Response(
|
||||
JSON.stringify({ items: [{ slug: "acme-ai", name: "Acme AI", repo: "acme/reviewers", website: "https://example.com/not-github" }] }),
|
||||
{ status: 200, headers: { "content-type": "application/json" } },
|
||||
));
|
||||
}
|
||||
|
||||
if (url === "https://github.com/acme/reviewers/archive/refs/heads/main.tar.gz") {
|
||||
return Promise.resolve(new Response(archiveBuffer, {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/gzip" },
|
||||
}));
|
||||
}
|
||||
|
||||
return Promise.reject(new Error(`Unexpected fetch URL: ${url}`));
|
||||
});
|
||||
|
||||
mockParseCompanyArchive.mockResolvedValue({
|
||||
company: { name: "Acme AI", slug: "acme-ai" },
|
||||
agents: [{ name: "Reviewer", skills: ["review"] }],
|
||||
teams: [],
|
||||
projects: [],
|
||||
tasks: [],
|
||||
skills: [],
|
||||
});
|
||||
|
||||
const response = await postImport(app, {
|
||||
importSource: "companies.sh",
|
||||
companySlug: "acme-ai",
|
||||
dryRun: true,
|
||||
});
|
||||
|
||||
expect(response.status, JSON.stringify(response.body)).toBe(200);
|
||||
expect(mockParseCompanyArchive).toHaveBeenCalledWith(expect.any(String), undefined);
|
||||
});
|
||||
|
||||
it("returns companies.sh live import with skill import result", async () => {
|
||||
const archiveBuffer = createGitHubArchiveBuffer("acme-ai");
|
||||
|
||||
@@ -625,7 +728,7 @@ describe("POST /api/agents/import", () => {
|
||||
const url = String(input);
|
||||
if (url === "https://companies.sh/api/companies") {
|
||||
return Promise.resolve(new Response(
|
||||
JSON.stringify({ items: [{ slug: "acme-ai", name: "Acme AI", repo: "acme/reviewers" }] }),
|
||||
JSON.stringify({ items: [{ slug: "acme-ai", name: "Acme AI", repo: "acme/reviewers", website: "https://github.com/acme/reviewers/tree/main" }] }),
|
||||
{
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
@@ -689,7 +792,7 @@ describe("POST /api/agents/import", () => {
|
||||
const url = String(input);
|
||||
if (url === "https://companies.sh/api/companies") {
|
||||
return Promise.resolve(new Response(
|
||||
JSON.stringify({ items: [{ slug: "acme-ai", name: "Acme AI", repo: "acme/reviewers" }] }),
|
||||
JSON.stringify({ items: [{ slug: "acme-ai", name: "Acme AI", repo: "acme/reviewers", website: "https://github.com/acme/reviewers/tree/main" }] }),
|
||||
{
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
|
||||
@@ -112,6 +112,48 @@ export function registerAgentImportExportRoutes(ctx: ApiRoutesContext): void {
|
||||
return /^[a-z0-9][a-z0-9-]{0,48}[a-z0-9]$/.test(slug) || /^[a-z0-9]$/.test(slug);
|
||||
}
|
||||
|
||||
function parseCompanyWebsiteSubPath(
|
||||
website: string | undefined,
|
||||
expectedRepo: string,
|
||||
): { branch?: string; subPath?: string } | null {
|
||||
if (typeof website !== "string" || website.trim().length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const repoMatch = expectedRepo.match(/^([a-zA-Z0-9_-]+)\/([a-zA-Z0-9_.-]+)$/);
|
||||
if (!repoMatch) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(website);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (url.hostname.toLowerCase() !== "github.com") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const segments = url.pathname.split("/").filter(Boolean);
|
||||
if (segments.length < 5 || segments[2] !== "tree") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const [, expectedOwner, expectedRepoName] = repoMatch;
|
||||
const [owner, repoName, , branch, ...subPathParts] = segments;
|
||||
if (owner !== expectedOwner || repoName !== expectedRepoName) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const subPath = subPathParts.join("/").trim();
|
||||
return {
|
||||
branch: typeof branch === "string" && branch.length > 0 ? branch : undefined,
|
||||
subPath: subPath.length > 0 ? subPath : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /api/agents/companies
|
||||
* Browse companies from companies.sh catalog.
|
||||
@@ -496,7 +538,7 @@ async function persistImportedSkills(
|
||||
// Note: The per-company endpoint (/api/companies/:slug) returns HTML (SPA),
|
||||
// so we fetch the full list and filter by slug.
|
||||
const companyApiUrl = "https://companies.sh/api/companies";
|
||||
let companyInfo: { name: string; repo?: string; tagline?: string } | null = null;
|
||||
let companyInfo: { name: string; repo?: string; tagline?: string; website?: string } | null = null;
|
||||
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
@@ -538,8 +580,9 @@ async function persistImportedSkills(
|
||||
const name = typeof match.name === "string" ? match.name : importCompanySlug;
|
||||
const repo = typeof match.repo === "string" ? match.repo : undefined;
|
||||
const tagline = typeof match.tagline === "string" ? match.tagline : undefined;
|
||||
const website = typeof match.website === "string" ? match.website : undefined;
|
||||
|
||||
companyInfo = { name, repo, tagline };
|
||||
companyInfo = { name, repo, tagline, website };
|
||||
} catch (fetchErr) {
|
||||
const message = fetchErr instanceof Error ? fetchErr.message : String(fetchErr);
|
||||
if (fetchErr instanceof ApiError) throw fetchErr;
|
||||
@@ -563,8 +606,10 @@ async function persistImportedSkills(
|
||||
}
|
||||
|
||||
const [, repoOwner, repoName] = repoMatch;
|
||||
// Use GitHub's archive API to get the default branch archive
|
||||
const archiveUrl = `https://github.com/${repoOwner}/${repoName}/archive/refs/heads/main.tar.gz`;
|
||||
const websiteInfo = parseCompanyWebsiteSubPath(companyInfo.website, `${repoOwner}/${repoName}`);
|
||||
const preferredBranch = websiteInfo?.branch?.trim() || "main";
|
||||
const subPath = websiteInfo?.subPath;
|
||||
const archiveUrl = `https://github.com/${repoOwner}/${repoName}/archive/refs/heads/${preferredBranch}.tar.gz`;
|
||||
|
||||
// Download and extract to temp directory
|
||||
let tempDir: string | null = null;
|
||||
@@ -616,7 +661,14 @@ async function persistImportedSkills(
|
||||
);
|
||||
|
||||
// Parse the downloaded archive directly to avoid requiring shell tar tools.
|
||||
pkg = await parseCompanyArchive(archivePath);
|
||||
try {
|
||||
pkg = await parseCompanyArchive(archivePath, subPath ? { subPath } : undefined);
|
||||
} catch (parseErr) {
|
||||
if (parseErr instanceof Error && parseErr.name === "AgentCompaniesParseError" && subPath) {
|
||||
throw badRequest(`Failed to import companies.sh slug "${importCompanySlug}" from subpath "${subPath}": ${parseErr.message}`);
|
||||
}
|
||||
throw parseErr;
|
||||
}
|
||||
|
||||
// Override company info if available from API
|
||||
if (companyInfo) {
|
||||
|
||||
Reference in New Issue
Block a user