From 51cb5bffcec061220e6b261994a43a4f3783f46f Mon Sep 17 00:00:00 2001 From: Hemachandar <132386067+hmacr@users.noreply.github.com> Date: Tue, 22 Apr 2025 19:29:39 +0530 Subject: [PATCH] Cache `issueSources` for embed integrations (#8952) * Cache `issueSources` for embed integrations * lock model before update --- plugins/github/server/GitHubIssueProvider.ts | 40 +++++++++++++++++++ plugins/github/server/api/github.ts | 38 +++++++++--------- plugins/github/server/github.ts | 11 +++++ plugins/github/server/index.ts | 5 +++ ...184249-add-issueSources-to-integrations.js | 15 +++++++ server/models/Integration.ts | 4 ++ .../processors/IntegrationCreatedProcessor.ts | 6 +++ server/queues/tasks/CacheIssueSourcesTask.ts | 32 +++++++++++++++ server/utils/BaseIssueProvider.ts | 15 +++++++ server/utils/PluginManager.ts | 3 ++ shared/schema.ts | 13 ++++++ shared/types.ts | 9 +++++ 12 files changed, 171 insertions(+), 20 deletions(-) create mode 100644 plugins/github/server/GitHubIssueProvider.ts create mode 100644 server/migrations/20250409184249-add-issueSources-to-integrations.js create mode 100644 server/queues/tasks/CacheIssueSourcesTask.ts create mode 100644 server/utils/BaseIssueProvider.ts diff --git a/plugins/github/server/GitHubIssueProvider.ts b/plugins/github/server/GitHubIssueProvider.ts new file mode 100644 index 0000000000..a8da02f5db --- /dev/null +++ b/plugins/github/server/GitHubIssueProvider.ts @@ -0,0 +1,40 @@ +import { Endpoints } from "@octokit/types"; +import { IssueSource } from "@shared/schema"; +import { IntegrationService, IntegrationType } from "@shared/types"; +import { Integration } from "@server/models"; +import { BaseIssueProvider } from "@server/utils/BaseIssueProvider"; +import { GitHub } from "./github"; + +// This is needed to handle Octokit paginate response type mismatch. +type ReposForInstallation = + Endpoints["GET /installation/repositories"]["response"]["data"]["repositories"]; + +export class GitHubIssueProvider extends BaseIssueProvider { + constructor() { + super(IntegrationService.GitHub); + } + + async fetchSources( + integration: Integration + ): Promise { + const client = await GitHub.authenticateAsInstallation( + integration.settings.github!.installation.id + ); + + const sources: IssueSource[] = []; + + for await (const response of client.requestRepos()) { + const repos = response.data as unknown as ReposForInstallation; + sources.push( + ...repos.map((repo) => ({ + id: String(repo.id), + name: repo.name, + owner: { id: String(repo.owner.id), name: repo.owner.login }, + service: IntegrationService.GitHub, + })) + ); + } + + return sources; + } +} diff --git a/plugins/github/server/api/github.ts b/plugins/github/server/api/github.ts index 793be354b8..993b630b94 100644 --- a/plugins/github/server/api/github.ts +++ b/plugins/github/server/api/github.ts @@ -2,6 +2,7 @@ import Router from "koa-router"; import find from "lodash/find"; import { IntegrationService, IntegrationType } from "@shared/types"; import { parseDomain } from "@shared/utils/domains"; +import { createContext } from "@server/context"; import Logger from "@server/logging/Logger"; import auth from "@server/middlewares/authentication"; import { transaction } from "@server/middlewares/transaction"; @@ -88,30 +89,27 @@ router.get( }, { transaction } ); - await Integration.create( - { - service: IntegrationService.GitHub, - type: IntegrationType.Embed, - userId: user.id, - teamId: user.teamId, - authenticationId: authentication.id, - settings: { - github: { - installation: { - id: installationId!, - account: { - id: installation.account?.id, - name: - // @ts-expect-error Property 'login' does not exist on type - installation.account?.login, - avatarUrl: installation.account?.avatar_url, - }, + await Integration.createWithCtx(createContext({ user, transaction }), { + service: IntegrationService.GitHub, + type: IntegrationType.Embed, + userId: user.id, + teamId: user.teamId, + authenticationId: authentication.id, + settings: { + github: { + installation: { + id: installationId!, + account: { + id: installation.account?.id, + name: + // @ts-expect-error Property 'login' does not exist on type + installation.account?.login, + avatarUrl: installation.account?.avatar_url, }, }, }, }, - { transaction } - ); + }); ctx.redirect(GitHubUtils.url); } ); diff --git a/plugins/github/server/github.ts b/plugins/github/server/github.ts index b952432a36..a5bb995818 100644 --- a/plugins/github/server/github.ts +++ b/plugins/github/server/github.ts @@ -24,6 +24,17 @@ type Issue = Endpoints["GET /repos/{owner}/{repo}/issues/{issue_number}"]["response"]["data"]; const requestPlugin = (octokit: Octokit) => ({ + requestRepos: () => + octokit.paginate.iterator( + octokit.rest.apps.listReposAccessibleToInstallation, + { + headers: { + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + }, + } + ), + requestPR: async (params: NonNullable>) => octokit.request(`GET /repos/{owner}/{repo}/pulls/{pull_number}`, { owner: params.owner, diff --git a/plugins/github/server/index.ts b/plugins/github/server/index.ts index f42927aaa7..b56dd83a9d 100644 --- a/plugins/github/server/index.ts +++ b/plugins/github/server/index.ts @@ -1,6 +1,7 @@ import { Minute } from "@shared/utils/time"; import { PluginManager, Hook } from "@server/utils/PluginManager"; import config from "../plugin.json"; +import { GitHubIssueProvider } from "./GitHubIssueProvider"; import router from "./api/github"; import env from "./env"; import { GitHub } from "./github"; @@ -20,6 +21,10 @@ if (enabled) { type: Hook.API, value: router, }, + { + type: Hook.IssueProvider, + value: new GitHubIssueProvider(), + }, { type: Hook.UnfurlProvider, value: { unfurl: GitHub.unfurl, cacheExpiry: Minute.seconds }, diff --git a/server/migrations/20250409184249-add-issueSources-to-integrations.js b/server/migrations/20250409184249-add-issueSources-to-integrations.js new file mode 100644 index 0000000000..69e2b5bd23 --- /dev/null +++ b/server/migrations/20250409184249-add-issueSources-to-integrations.js @@ -0,0 +1,15 @@ +"use strict"; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + queryInterface.addColumn("integrations", "issueSources", { + type: Sequelize.JSONB, + allowNull: true, + }); + }, + + async down(queryInterface, Sequelize) { + queryInterface.removeColumn("integrations", "issueSources"); + }, +}; diff --git a/server/models/Integration.ts b/server/models/Integration.ts index fc67a61969..9f52293d3e 100644 --- a/server/models/Integration.ts +++ b/server/models/Integration.ts @@ -13,6 +13,7 @@ import { IsIn, AfterDestroy, } from "sequelize-typescript"; +import { IssueSource } from "@shared/schema"; import { IntegrationType, IntegrationService } from "@shared/types"; import type { IntegrationSettings } from "@shared/types"; import Collection from "@server/models/Collection"; @@ -53,6 +54,9 @@ class Integration extends ParanoidModel< @Column(DataType.ARRAY(DataType.STRING)) events: string[]; + @Column(DataType.JSONB) + issueSources: IssueSource[] | null; + // associations @BelongsTo(() => User, "userId") diff --git a/server/queues/processors/IntegrationCreatedProcessor.ts b/server/queues/processors/IntegrationCreatedProcessor.ts index cdf5d7e0cb..8dc45bd8a4 100644 --- a/server/queues/processors/IntegrationCreatedProcessor.ts +++ b/server/queues/processors/IntegrationCreatedProcessor.ts @@ -3,6 +3,7 @@ import { Integration } from "@server/models"; import BaseProcessor from "@server/queues/processors/BaseProcessor"; import { IntegrationEvent, Event } from "@server/types"; import { CacheHelper } from "@server/utils/CacheHelper"; +import CacheIssueSourcesTask from "../tasks/CacheIssueSourcesTask"; export default class IntegrationCreatedProcessor extends BaseProcessor { static applicableEvents: Event["name"][] = ["integrations.create"]; @@ -18,6 +19,11 @@ export default class IntegrationCreatedProcessor extends BaseProcessor { return; } + // Store the available issue sources in the integration record. + await CacheIssueSourcesTask.schedule({ + integrationId: integration.id, + }); + // Clear the cache of unfurled data for the team as it may be stale now. await CacheHelper.clearData(CacheHelper.getUnfurlKey(integration.teamId)); } diff --git a/server/queues/tasks/CacheIssueSourcesTask.ts b/server/queues/tasks/CacheIssueSourcesTask.ts new file mode 100644 index 0000000000..053e98d44c --- /dev/null +++ b/server/queues/tasks/CacheIssueSourcesTask.ts @@ -0,0 +1,32 @@ +import { Integration } from "@server/models"; +import { sequelize } from "@server/storage/database"; +import { Hook, PluginManager } from "@server/utils/PluginManager"; +import BaseTask from "./BaseTask"; + +const plugins = PluginManager.getHooks(Hook.IssueProvider); + +type Props = { + integrationId: string; +}; + +export default class CacheIssueSourcesTask extends BaseTask { + async perform({ integrationId }: Props) { + const integration = await Integration.findByPk(integrationId); + if (!integration) { + return; + } + + const plugin = plugins.find((p) => p.value.service === integration.service); + if (!plugin) { + return; + } + + const sources = await plugin.value.fetchSources(integration); + + await sequelize.transaction(async (transaction) => { + await integration.reload({ transaction, lock: transaction.LOCK.UPDATE }); + integration.issueSources = sources; + await integration.save({ transaction }); + }); + } +} diff --git a/server/utils/BaseIssueProvider.ts b/server/utils/BaseIssueProvider.ts new file mode 100644 index 0000000000..6a6b961663 --- /dev/null +++ b/server/utils/BaseIssueProvider.ts @@ -0,0 +1,15 @@ +import { IssueSource } from "@shared/schema"; +import { IntegrationType, IssueTrackerIntegrationService } from "@shared/types"; +import { Integration } from "@server/models"; + +export abstract class BaseIssueProvider { + service: IssueTrackerIntegrationService; + + constructor(service: IssueTrackerIntegrationService) { + this.service = service; + } + + abstract fetchSources( + integration: Integration + ): Promise; +} diff --git a/server/utils/PluginManager.ts b/server/utils/PluginManager.ts index 2db8d7010a..3d3af6619c 100644 --- a/server/utils/PluginManager.ts +++ b/server/utils/PluginManager.ts @@ -9,6 +9,7 @@ import Logger from "@server/logging/Logger"; import type BaseProcessor from "@server/queues/processors/BaseProcessor"; import type BaseTask from "@server/queues/tasks/BaseTask"; import { UnfurlSignature, UninstallSignature } from "@server/types"; +import { BaseIssueProvider } from "./BaseIssueProvider"; export enum PluginPriority { VeryHigh = 0, @@ -25,6 +26,7 @@ export enum Hook { API = "api", AuthProvider = "authProvider", EmailTemplate = "emailTemplate", + IssueProvider = "issueProvider", Processor = "processor", Task = "task", UnfurlProvider = "unfurl", @@ -39,6 +41,7 @@ type PluginValueMap = { [Hook.API]: Router; [Hook.AuthProvider]: { router: Router; id: string }; [Hook.EmailTemplate]: typeof BaseEmail; + [Hook.IssueProvider]: BaseIssueProvider; [Hook.Processor]: typeof BaseProcessor; [Hook.Task]: typeof BaseTask; [Hook.Uninstall]: UninstallSignature; diff --git a/shared/schema.ts b/shared/schema.ts index 6a48435050..295043a0ee 100644 --- a/shared/schema.ts +++ b/shared/schema.ts @@ -3,6 +3,7 @@ import { CollectionPermission, type ImportableIntegrationService, IntegrationService, + IssueTrackerIntegrationService, ProsemirrorDoc, } from "./types"; import { PageType } from "plugins/notion/shared/types"; @@ -57,3 +58,15 @@ export type ImportTaskOutput = { createdAt?: Date; updatedAt?: Date; }[]; + +export const IssueSource = z.object({ + id: z.string().nonempty(), + name: z.string().nonempty(), + owner: z.object({ + id: z.string().nonempty(), + name: z.string().nonempty(), + }), + service: z.nativeEnum(IssueTrackerIntegrationService), +}); + +export type IssueSource = z.infer; diff --git a/shared/types.ts b/shared/types.ts index 135d22fd91..63cba80c0b 100644 --- a/shared/types.ts +++ b/shared/types.ts @@ -129,6 +129,15 @@ export const ImportableIntegrationService = { Notion: IntegrationService.Notion, } as const; +export type IssueTrackerIntegrationService = Extract< + IntegrationService, + IntegrationService.GitHub +>; + +export const IssueTrackerIntegrationService = { + GitHub: IntegrationService.GitHub, +} as const; + export type UserCreatableIntegrationService = Extract< IntegrationService, | IntegrationService.Diagrams