Listen to GitHub webhooks to update issueSources cache (#9414)

* Listen to GitHub webhooks to update issue-sources cache

* Add `GitHubWebhookTask`

* review

plugins/github/server/GitHubIssueProvider.ts

@@ -1,7 +1,14 @@
 import { Endpoints } from "@octokit/types";
+import {
+  InstallationNewPermissionsAcceptedEvent,
+  InstallationRepositoriesEvent,
+  RepositoryRenamedEvent,
+} from "@octokit/webhooks-types";
 import { IssueSource } from "@shared/schema";
 import { IntegrationService, IntegrationType } from "@shared/types";
-import { Integration } from "@server/models";
+import Logger from "@server/logging/Logger";
+import { Integration, IntegrationAuthentication } from "@server/models";
+import { sequelize } from "@server/storage/database";
 import { BaseIssueProvider } from "@server/utils/BaseIssueProvider";
 import { GitHub } from "./github";
 
@@ -37,4 +44,215 @@ export class GitHubIssueProvider extends BaseIssueProvider {
 
     return sources;
   }
+
+  async handleWebhook({
+    payload,
+    headers,
+  }: {
+    payload: Record<string, unknown>;
+    headers: Record<string, unknown>;
+  }) {
+    const hookId = headers["x-github-hook-id"] as string;
+    const eventName = headers["x-github-event"] as string;
+    const action = payload.action as string;
+
+    if (!eventName || !action) {
+      Logger.warn(
+        `Received GitHub webhook without event name or action; hookId: ${hookId}, eventName: ${eventName}, action: ${action}`
+      );
+      return;
+    }
+
+    switch (eventName) {
+      case "installation": {
+        await this.handleInstallationEvent(payload, action);
+        break;
+      }
+
+      case "installation_repositories": {
+        await this.handleInstallationRepositoriesEvent(
+          payload as unknown as InstallationRepositoriesEvent
+        );
+        break;
+      }
+
+      case "repository": {
+        await this.handleRepositoryEvent(payload, action, hookId);
+        break;
+      }
+
+      default:
+    }
+  }
+
+  private async handleInstallationEvent(
+    payload: Record<string, unknown>,
+    action: string
+  ): Promise<void> {
+    if (action !== "new_permissions_accepted") {
+      return;
+    }
+
+    const event = payload as unknown as InstallationNewPermissionsAcceptedEvent;
+    const installationId = event.installation.id;
+    const integration = await Integration.findOne({
+      where: {
+        service: IntegrationService.GitHub,
+        "settings.github.installation.id": installationId,
+      },
+    });
+
+    if (!integration) {
+      Logger.warn(
+        `GitHub installation new_permissions_accepted event without integration; installationId: ${installationId}`
+      );
+      return;
+    }
+
+    const sources = await this.fetchSources(integration);
+
+    const client = await GitHub.authenticateAsInstallation(installationId);
+    const installation = await client.requestAppInstallation(installationId);
+
+    const scopes = Object.entries(installation.data.permissions).map(
+      ([name, permission]) => `${name}:${permission}`
+    );
+
+    await sequelize.transaction(async (transaction) => {
+      await integration.reload({
+        include: {
+          model: IntegrationAuthentication,
+          as: "authentication",
+          required: true,
+        },
+        transaction,
+        lock: transaction.LOCK.UPDATE,
+      });
+      const authentication = integration.authentication;
+
+      if (!authentication) {
+        Logger.warn(
+          `GitHub integration without authentication; integrationId: ${integration.id}`
+        );
+        return;
+      }
+
+      authentication.scopes = scopes;
+      await authentication.save({ transaction });
+
+      integration.issueSources = sources;
+      integration.changed("issueSources", true);
+      await integration.save({ transaction });
+    });
+  }
+
+  private async handleInstallationRepositoriesEvent(
+    event: InstallationRepositoriesEvent
+  ): Promise<void> {
+    const installationId = event.installation.id;
+    const account = event.installation.account;
+
+    await sequelize.transaction(async (transaction) => {
+      const integration = await Integration.findOne({
+        where: {
+          service: IntegrationService.GitHub,
+          "settings.github.installation.id": installationId,
+        },
+        transaction,
+        lock: transaction.LOCK.UPDATE,
+      });
+
+      if (!integration) {
+        Logger.warn(
+          `GitHub installation_repositories event without integration; installationId: ${installationId}`
+        );
+        return;
+      }
+
+      let sources = integration.issueSources ?? [];
+
+      if (event.action === "added") {
+        const addedSources = event.repositories_added.map<IssueSource>(
+          (repo) => ({
+            id: String(repo.id),
+            name: repo.name,
+            owner: {
+              id: String(account.id),
+              name: account.login,
+            },
+            service: IntegrationService.GitHub,
+          })
+        );
+        sources.push(...addedSources);
+      } else {
+        const removedSourceIds = event.repositories_removed.map((repo) =>
+          String(repo.id)
+        );
+        sources = sources.filter(
+          (source) => !removedSourceIds.includes(source.id)
+        );
+      }
+
+      integration.issueSources = sources;
+      integration.changed("issueSources", true);
+      await integration.save({ transaction });
+    });
+  }
+
+  private async handleRepositoryEvent(
+    payload: Record<string, unknown>,
+    action: string,
+    hookId: string
+  ): Promise<void> {
+    if (action !== "renamed") {
+      return;
+    }
+
+    const event = payload as unknown as RepositoryRenamedEvent;
+    const installationId = event.installation?.id;
+
+    if (!installationId) {
+      Logger.warn(
+        `GitHub repository renamed event without installation ID; hookId: ${hookId}`
+      );
+      return;
+    }
+
+    const repoId = event.repository.id;
+    const repoName = event.repository.name;
+
+    await sequelize.transaction(async (transaction) => {
+      const integration = await Integration.findOne({
+        where: {
+          service: IntegrationService.GitHub,
+          "settings.github.installation.id": installationId,
+        },
+        transaction,
+        lock: transaction.LOCK.UPDATE,
+      });
+
+      if (!integration) {
+        Logger.warn(
+          `GitHub repository renamed event without integration; installationId: ${installationId}`
+        );
+        return;
+      }
+
+      const source = integration.issueSources?.find(
+        (s) => s.id === String(repoId)
+      );
+
+      if (!source) {
+        Logger.info(
+          "task",
+          `No matching issue source found for repository ID: ${repoId}, integration ID: ${integration.id}`
+        );
+        return;
+      }
+
+      source.name = repoName;
+      integration.changed("issueSources", true);
+      await integration.save({ transaction });
+    });
+  }
 }

plugins/github/server/api/github.ts

@@ -6,10 +6,13 @@ import apexAuthRedirect from "@server/middlewares/apexAuthRedirect";
 import auth from "@server/middlewares/authentication";
 import { transaction } from "@server/middlewares/transaction";
 import validate from "@server/middlewares/validate";
+import validateWebhook from "@server/middlewares/validateWebhook";
 import { IntegrationAuthentication, Integration } from "@server/models";
 import { APIContext } from "@server/types";
 import { GitHubUtils } from "../../shared/GitHubUtils";
+import env from "../env";
 import { GitHub } from "../github";
+import GitHubWebhookTask from "../tasks/GitHubWebhookTask";
 import * as T from "./schema";
 
 const router = new Router();
@@ -60,11 +63,16 @@ router.get(
       return ctx.redirect(GitHubUtils.errorUrl("unauthenticated"));
     }
 
+    const scopes = Object.entries(installation.permissions).map(
+      ([name, permission]) => `${name}:${permission}`
+    );
+
     const authentication = await IntegrationAuthentication.create(
       {
         service: IntegrationService.GitHub,
         userId: user.id,
         teamId: user.teamId,
+        scopes,
       },
       { transaction }
     );
@@ -92,4 +100,29 @@ router.get(
   }
 );
 
+router.post(
+  "github.webhooks",
+  validateWebhook({
+    secretKey: env.GITHUB_WEBHOOK_SECRET!,
+    getSignatureFromHeader: (ctx) => {
+      const { headers } = ctx.request;
+      const signatureHeader = headers["x-hub-signature-256"];
+      const signature = Array.isArray(signatureHeader)
+        ? signatureHeader[0]
+        : signatureHeader;
+      return signature?.split("=")[1];
+    },
+  }),
+  async (ctx: APIContext) => {
+    const { headers, body } = ctx.request;
+
+    await new GitHubWebhookTask().schedule({
+      payload: body,
+      headers,
+    });
+
+    ctx.status = 202;
+  }
+);
+
 export default router;

plugins/github/server/env.ts

@@ -26,6 +26,12 @@ class GitHubPluginEnvironment extends Environment {
     environment.GITHUB_CLIENT_SECRET
   );
 
+  @IsOptional()
+  @CannotUseWithout("GITHUB_CLIENT_ID")
+  public GITHUB_WEBHOOK_SECRET = this.toOptionalString(
+    environment.GITHUB_WEBHOOK_SECRET
+  );
+
   @IsOptional()
   @CannotUseWithout("GITHUB_APP_PRIVATE_KEY")
   public GITHUB_APP_ID = this.toOptionalString(environment.GITHUB_APP_ID);

plugins/github/server/github.ts

@@ -22,6 +22,8 @@ type PR =
   Endpoints["GET /repos/{owner}/{repo}/pulls/{pull_number}"]["response"]["data"];
 type Issue =
   Endpoints["GET /repos/{owner}/{repo}/issues/{issue_number}"]["response"]["data"];
+type Installation =
+  Endpoints["GET /app/installations/{installation_id}"]["response"]["data"];
 
 const requestPlugin = (octokit: Octokit) => ({
   requestRepos: () =>
@@ -86,6 +88,23 @@ const requestPlugin = (octokit: Octokit) => ({
     }
   },
 
+  /**
+   * Fetches details of a specific GitHub app installation
+   *
+   * @param installationId Id of the installation to fetch
+   * @returns Response containing installation details
+   */
+  requestAppInstallation: async (
+    installationId: number
+  ): Promise<OctokitResponse<Installation>> =>
+    octokit.request("GET /app/installations/{installation_id}", {
+      installation_id: installationId,
+      headers: {
+        Accept: "application/vnd.github+json",
+        "X-GitHub-Api-Version": "2022-11-28",
+      },
+    }),
+
   /**
    * Uninstalls the GitHub app from a given target
    *

plugins/github/server/index.ts

@@ -5,6 +5,7 @@ import { GitHubIssueProvider } from "./GitHubIssueProvider";
 import router from "./api/github";
 import env from "./env";
 import { GitHub } from "./github";
+import GitHubWebhookTask from "./tasks/GitHubWebhookTask";
 import { uninstall } from "./uninstall";
 
 const enabled =
@@ -21,6 +22,10 @@ if (enabled) {
       type: Hook.API,
       value: router,
     },
+    {
+      type: Hook.Task,
+      value: GitHubWebhookTask,
+    },
     {
       type: Hook.IssueProvider,
       value: new GitHubIssueProvider(),

plugins/github/server/tasks/GitHubWebhookTask.ts

@@ -0,0 +1,26 @@
+import { IntegrationService } from "@shared/types";
+import BaseTask from "@server/queues/tasks/BaseTask";
+import { Hook, PluginManager } from "@server/utils/PluginManager";
+
+type Props = {
+  headers: Record<string, unknown>;
+  payload: Record<string, unknown>;
+};
+
+export default class GitHubWebhookTask extends BaseTask<Props> {
+  public async perform({ headers, payload }: Props): Promise<void> {
+    const plugins = PluginManager.getHooks(Hook.IssueProvider);
+    const plugin = plugins.find(
+      (p) => p.value.service === IntegrationService.GitHub
+    );
+
+    if (!plugin) {
+      return;
+    }
+
+    await plugin.value.handleWebhook({
+      headers,
+      payload,
+    });
+  }
+}