root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(dashboard): detect GitHub App bots (CodeRabbit/Greptile) in PR/issue comments

Browse Source 
gh pr/issue view --json comments returns only author.login (no type), and app-bot logins (coderabbitai, greptileai) lack the [bot] suffix — so every app bot was classified human. Fetch comments via gh api graphql with author{ __typename login avatarUrl } so authorIsBot = __typename === 'Bot' (any app bot by type, no hardcoded names); REST token-fallback already used user.type/[bot]. Output shape + avatars preserved.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
gsxdsm
2026-06-22 12:04:24 -07:00
parent 2505d09771
commit b8051e3551
2 changed files with 192 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
packages/dashboard/src/__tests__/github.test.ts
View File

@@ -506,6 +506,110 @@ describe("GitHubClient", () => {
});
});
// FNXC:GitHubImport 2026-06-22-12:00:
// Regression coverage for the human-vs-bot misclassification of GitHub App reviewers.
// `gh pr/issue view --json comments` surfaces only `{ login }` (no type, and an app bot's
// bare display login such as `coderabbitai`/`greptileai` WITHOUT the `[bot]` suffix), so the
// comment fetch must read the authoritative Actor `__typename` via `gh api graphql`.
// Surfaces: gh PR conversation, gh issue conversation, and the `[bot]`-login suffix fallback.
describe("getPullRequestDetail / getIssueDetail bot detection (FN bot-misclassification)", () => {
// Drive the two runGhJsonAsync calls in the gh PR path by inspecting the gh argv:
// - ["api","graphql", ...] -> comments via Actor.__typename
// - ["pr","view", ...] -> statusCheckRollup
function mockGhPrDetail(commentNodes: unknown[]) {
mockRunGhJsonAsync.mockImplementation(async (args: string[]) => {
if (args[0] === "api" && args[1] === "graphql") {
return { data: { repository: { pullRequest: { comments: { nodes: commentNodes } } } } } as any;
}
return { statusCheckRollup: [] } as any;
});
}
function mockGhIssueDetail(commentNodes: unknown[]) {
mockRunGhJsonAsync.mockImplementation(async (args: string[]) => {
if (args[0] === "api" && args[1] === "graphql") {
return { data: { repository: { issue: { comments: { nodes: commentNodes } } } } } as any;
}
return {} as any;
});
}
it("flags a GitHub App reviewer (CodeRabbit) as bot via Actor __typename even with a bare login", async () => {
// CodeRabbit's gh-surfaced login has NO `[bot]` suffix — only __typename distinguishes it.
mockGhPrDetail([
{ author: { __typename: "User", login: "alice", avatarUrl: "https://avatars/alice" }, body: "human review", createdAt: "2024-01-01T00:00:00Z" },
{ author: { __typename: "Bot", login: "coderabbitai", avatarUrl: "https://avatars/cr" }, body: "automated review", createdAt: "2024-01-02T00:00:00Z" },
{ author: { __typename: "Bot", login: "greptileai", avatarUrl: "https://avatars/gr" }, body: "greptile review", createdAt: "2024-01-03T00:00:00Z" },
]);
const result = await client.getPullRequestDetail("owner", "repo", 42);
// The comment fetch must use `gh api graphql`, not `gh pr view --json comments`.
expect(mockRunGhJsonAsync).toHaveBeenCalledWith(
expect.arrayContaining(["api", "graphql"]),
);
const byAuthor = Object.fromEntries(result.comments.map((c) => [c.author, c]));
expect(byAuthor["alice"].authorIsBot).toBe(false);
expect(byAuthor["coderabbitai"].authorIsBot).toBe(true);
expect(byAuthor["greptileai"].authorIsBot).toBe(true);
// Bots keep the API avatar; humans get a github.com fallback.
expect(byAuthor["coderabbitai"].authorAvatarUrl).toBe("https://avatars/cr");
expect(byAuthor["alice"].authorAvatarUrl).toBe("https://avatars/alice");
});
it("flags a `[bot]`-suffixed login as bot via the suffix fallback", async () => {
mockGhPrDetail([
{ author: { __typename: "Bot", login: "github-actions[bot]" }, body: "ci", createdAt: "2024-01-01T00:00:00Z" },
]);
const result = await client.getPullRequestDetail("owner", "repo", 7);
expect(result.comments[0].authorIsBot).toBe(true);
});
it("keeps a normal user classified as human", async () => {
mockGhPrDetail([
{ author: { __typename: "User", login: "bob" }, body: "hi", createdAt: "2024-01-01T00:00:00Z" },
]);
const result = await client.getPullRequestDetail("owner", "repo", 8);
expect(result.comments[0].authorIsBot).toBe(false);
});
it("flags CodeRabbit on the issue conversation path too (surface: gh issue)", async () => {
mockGhIssueDetail([
{ author: { __typename: "Bot", login: "coderabbitai" }, body: "issue triage", createdAt: "2024-01-02T00:00:00Z" },
{ author: { __typename: "User", login: "carol" }, body: "real user", createdAt: "2024-01-03T00:00:00Z" },
]);
const result = await client.getIssueDetail("owner", "repo", 99);
const byAuthor = Object.fromEntries(result.comments.map((c) => [c.author, c]));
expect(byAuthor["coderabbitai"].authorIsBot).toBe(true);
expect(byAuthor["carol"].authorIsBot).toBe(false);
});
it("flags bots on the REST token fallback via user.type and [bot] login", async () => {
// gh path fails -> token REST fallback. REST issues/{n}/comments has user.type + `[bot]` login.
mockRunGhJsonAsync.mockRejectedValue(new Error("gh failed"));
const clientWithToken = new GitHubClient("ghp_token");
const mockFetch = vi.fn().mockImplementation((url: string) => {
if (url.includes("/issues/") && url.includes("/comments")) {
return Promise.resolve({
ok: true,
json: () => Promise.resolve([
{ user: { login: "dave", type: "User", avatar_url: "https://avatars/dave" }, body: "human", created_at: "2024-01-01T00:00:00Z" },
{ user: { login: "coderabbitai[bot]", type: "Bot", avatar_url: "https://avatars/cr" }, body: "bot", created_at: "2024-01-02T00:00:00Z" },
]),
});
}
// pulls/{n} -> no head sha -> checks degrade to []
return Promise.resolve({ ok: true, json: () => Promise.resolve({}) });
});
global.fetch = mockFetch as any;
const result = await clientWithToken.getPullRequestDetail("owner", "repo", 5);
const byAuthor = Object.fromEntries(result.comments.map((c) => [c.author, c]));
expect(byAuthor["dave"].authorIsBot).toBe(false);
expect(byAuthor["coderabbitai[bot]"].authorIsBot).toBe(true);
vi.restoreAllMocks();
});
});
describe("listPrComments", () => {
const mockComments = [
{

122
packages/dashboard/src/github.ts
View File

@@ -19,8 +19,11 @@ const execAsync = promisify(exec);
/*
FNXC:GitHubImport 2026-06-23-03:30:
Resolve a comment author's bot flag + avatar URL for the Import Tasks preview.
isBot: true when the author type is a GitHub Bot (gh GraphQL `__typename === "Bot"` / `is_bot`, REST `user.type === "Bot"`) OR the login ends in `[bot]`.
isBot: true when the author type is a GitHub Bot (gh GraphQL Actor `__typename === "Bot"` / `is_bot`, REST `user.type === "Bot"`) OR the login ends in `[bot]` (case-insensitive).
avatarUrl: prefer the API-provided avatar; otherwise fall back to `https://github.com/{login}.png?size=40` — but NOT for bots, whose `[bot]`-suffixed login does not resolve to a real avatar (the frontend renders a generic bot icon instead of a broken image).
FNXC:GitHubImport 2026-06-22-12:00:
The TYPE field is the real bot signal and must be read directly. `gh pr/issue view --json comments` does NOT expose `__typename`/`type`/`is_bot` and surfaces an app bot's bare display login (e.g. `coderabbitai`, `greptileai`) WITHOUT the `[bot]` suffix, so the suffix heuristic alone misclassified GitHub App reviewers (CodeRabbit, Greptile) as HUMAN. The comment fetch now reads Actor `__typename` via `gh api graphql` (and REST `user.type`/`[bot]` login on the token path) — never hardcode specific app names; the type field catches ANY app bot.
*/
function resolveCommentAuthor(input: {
login: string;
@@ -46,6 +49,17 @@ function resolveCommentAuthor(input: {
return { authorIsBot, authorAvatarUrl };
}
/*
FNXC:GitHubImport 2026-06-22-12:00:
Shape of a single comment node from the `gh api graphql` conversation query. The Actor
`__typename` is the authoritative bot signal (`gh pr/issue view --json comments` omits it).
*/
interface GhGraphqlCommentNode {
author?: { __typename?: string | null; login?: string | null; avatarUrl?: string | null } | null;
body?: string | null;
createdAt?: string | null;
}
function quoteGitArg(value: string): string {
return `'${value.replaceAll("'", "'\\''")}'`;
}
@@ -3638,6 +3652,62 @@ export class GitHubClient {
throw new Error("GitHub CLI (gh) is not available or not authenticated, and no GITHUB_TOKEN provided. Run 'gh auth login' to authenticate.");
}
/*
FNXC:GitHubImport 2026-06-22-12:00:
Fetch a PR/issue's conversation comments via `gh api graphql` so the author's authoritative
Actor `__typename` (User | Bot | Organization | Mannequin) is available per comment. The
`gh pr/issue view --json comments` path only surfaces `{ login }` with no type and a bot's bare
display login (no `[bot]` suffix), which silently misclassified GitHub App reviewers as human.
Returns the same `{ author, body, createdAt, authorAvatarUrl?, authorIsBot }` shape; `authorIsBot`
is true when `__typename === "Bot"` (or the `[bot]`-login suffix fallback inside resolveCommentAuthor).
*/
private async fetchCommentsWithGhGraphql(
owner: string,
repo: string,
number: number,
kind: "pullRequest" | "issue",
): Promise<Array<{ author: string; body: string; createdAt: string; authorAvatarUrl?: string; authorIsBot: boolean }>> {
const query = `query($owner:String!,$repo:String!,$number:Int!){
repository(owner:$owner,name:$repo){
${kind}(number:$number){
comments(first:100){
nodes{ author{ __typename login avatarUrl } body createdAt }
}
}
}
}`;
const result = await runGhJsonAsync<{
data?: {
repository?: {
pullRequest?: { comments?: { nodes?: GhGraphqlCommentNode[] } } | null;
issue?: { comments?: { nodes?: GhGraphqlCommentNode[] } } | null;
} | null;
};
}>([
"api", "graphql",
"-f", `query=${query}`,
"-F", `owner=${owner}`,
"-F", `repo=${repo}`,
"-F", `number=${number}`,
]);
const container = kind === "pullRequest"
? result.data?.repository?.pullRequest
: result.data?.repository?.issue;
const nodes = container?.comments?.nodes ?? [];
return nodes.map((c) => {
const author = c.author?.login ?? "unknown";
const { authorIsBot, authorAvatarUrl } = resolveCommentAuthor({
login: author,
// Actor.__typename is the real signal: "Bot" for any GitHub App (CodeRabbit, Greptile, ...).
typename: c.author?.__typename,
avatarUrl: c.author?.avatarUrl,
});
return { author, body: c.body ?? "", createdAt: c.createdAt ?? "", authorAvatarUrl, authorIsBot };
});
}
private async getPullRequestDetailWithGh(
owner: string,
repo: string,
@@ -3646,9 +3716,18 @@ export class GitHubClient {
comments: Array<{ author: string; body: string; createdAt: string; authorAvatarUrl?: string; authorIsBot: boolean }>;
checks: Array<{ name: string; status: string; conclusion?: string; detailsUrl?: string }>;
}> {
// FNXC:GitHubImport 2026-06-22-12:00:
// `gh pr view --json comments` author is just `{ login }` — no `__typename`/`type`/`is_bot`,
// and the surfaced login is the app's bare display login (e.g. `coderabbitai`, `greptileai`)
// WITHOUT the `[bot]` suffix. That made every GitHub App reviewer (CodeRabbit, Greptile, etc.)
// misclassify as HUMAN, since neither the type field nor the `[bot]` suffix heuristic could fire.
// Fix: read the authoritative Actor `__typename` (User | Bot | Organization | Mannequin) via
// `gh api graphql`, so `authorIsBot = __typename === "Bot"` catches ANY app bot by type, not by name.
// statusCheckRollup is still only on `gh pr view`, so it stays a separate (best-effort) call.
const comments = await this.fetchCommentsWithGhGraphql(owner, repo, number, "pullRequest");
let checks: Array<{ name: string; status: string; conclusion?: string; detailsUrl?: string }> = [];
const pr = await runGhJsonAsync<{
// gh pr view comment authors expose login + avatarUrl; `__typename`/`is_bot` surface bot actors when present.
comments?: Array<{ author?: { login?: string; avatarUrl?: string; __typename?: string; is_bot?: boolean } | null; body?: string; createdAt?: string }>;
// `gh pr view --json statusCheckRollup` returns a flat array of mixed CheckRun/StatusContext shapes.
statusCheckRollup?: Array<{
name?: string;
@@ -3663,21 +3742,10 @@ export class GitHubClient {
}>([
"pr", "view", String(number),
"--repo", `${owner}/${repo}`,
"--json", "comments,statusCheckRollup",
"--json", "statusCheckRollup",
]);
const comments = (pr.comments ?? []).map((c) => {
const author = c.author?.login ?? "unknown";
const { authorIsBot, authorAvatarUrl } = resolveCommentAuthor({
login: author,
typename: c.author?.__typename,
isBot: c.author?.is_bot,
avatarUrl: c.author?.avatarUrl,
});
return { author, body: c.body ?? "", createdAt: c.createdAt ?? "", authorAvatarUrl, authorIsBot };
});
const checks = (pr.statusCheckRollup ?? []).map((c) => ({
checks = (pr.statusCheckRollup ?? []).map((c) => ({
name: c.name ?? c.context ?? "check",
// CheckRun uses `status`; StatusContext uses `state`. Surface whichever is present.
status: (c.status ?? c.state ?? "").toLowerCase(),
@@ -3789,24 +3857,10 @@ export class GitHubClient {
): Promise<{
comments: Array<{ author: string; body: string; createdAt: string; authorAvatarUrl?: string; authorIsBot: boolean }>;
}> {
const issue = await runGhJsonAsync<{
comments?: Array<{ author?: { login?: string; avatarUrl?: string; __typename?: string; is_bot?: boolean } | null; body?: string; createdAt?: string }>;
}>([
"issue", "view", String(number),
"--repo", `${owner}/${repo}`,
"--json", "comments",
]);
const comments = (issue.comments ?? []).map((c) => {
const author = c.author?.login ?? "unknown";
const { authorIsBot, authorAvatarUrl } = resolveCommentAuthor({
login: author,
typename: c.author?.__typename,
isBot: c.author?.is_bot,
avatarUrl: c.author?.avatarUrl,
});
return { author, body: c.body ?? "", createdAt: c.createdAt ?? "", authorAvatarUrl, authorIsBot };
});
// FNXC:GitHubImport 2026-06-22-12:00:
// Use the graphql Actor.__typename path (not `gh issue view --json comments`, which omits the
// type) so GitHub App reviewers like CodeRabbit/Greptile are correctly flagged as bots.
const comments = await this.fetchCommentsWithGhGraphql(owner, repo, number, "issue");
return { comments };
}