From e10ac83192a21500292fb9c6423a2c680a4898bc Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 30 Dec 2025 14:31:09 +0100 Subject: [PATCH 01/17] feat: implement DAL for new repositories tables --- .../src/repositories/index.ts | 230 ++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 services/libs/data-access-layer/src/repositories/index.ts diff --git a/services/libs/data-access-layer/src/repositories/index.ts b/services/libs/data-access-layer/src/repositories/index.ts new file mode 100644 index 0000000000..b5535114a0 --- /dev/null +++ b/services/libs/data-access-layer/src/repositories/index.ts @@ -0,0 +1,230 @@ +import { QueryExecutor } from '../queryExecutor' + +/** + * Repository entity from the public.repositories table + */ +export interface IRepository { + id: string + url: string + segmentId: string + gitIntegrationId: string + sourceIntegrationId: string + insightsProjectId: string + archived: boolean + forkedFrom: string | null + excluded: boolean + createdAt: string + updatedAt: string + deletedAt: string | null + lastArchivedCheckAt: string | null +} + +export interface ICreateRepository { + id: string + url: string + segmentId: string + gitIntegrationId: string + sourceIntegrationId: string + insightsProjectId: string + archived?: boolean + forkedFrom?: string | null + excluded?: boolean +} + +export async function createRepository( + qx: QueryExecutor, + data: ICreateRepository, +): Promise { + // TODO: Implement + throw new Error('Not implemented') +} + +/** + * Bulk inserts repositories into public.repositories and git.repositoryProcessing + * @param qx - Query executor (should be transactional) + * @param repositories - Array of repositories to insert + */ +export async function insertRepositories( + qx: QueryExecutor, + repositories: ICreateRepository[], +): Promise { + if (repositories.length === 0) { + return + } + + const values = repositories.map((repo) => ({ + id: repo.id, + url: repo.url, + segmentId: repo.segmentId, + gitIntegrationId: repo.gitIntegrationId, + sourceIntegrationId: repo.sourceIntegrationId, + insightsProjectId: repo.insightsProjectId, + archived: repo.archived ?? false, + forkedFrom: repo.forkedFrom ?? null, + excluded: repo.excluded ?? false, + })) + + await qx.result( + ` + INSERT INTO public.repositories ( + id, + url, + "segmentId", + "gitIntegrationId", + "sourceIntegrationId", + "insightsProjectId", + archived, + "forkedFrom", + excluded, + "createdAt", + "updatedAt" + ) + SELECT + v.id::uuid, + v.url, + v."segmentId"::uuid, + v."gitIntegrationId"::uuid, + v."sourceIntegrationId"::uuid, + v."insightsProjectId"::uuid, + v.archived::boolean, + v."forkedFrom", + v.excluded::boolean, + NOW(), + NOW() + FROM jsonb_to_recordset($(values)::jsonb) AS v( + id text, + url text, + "segmentId" text, + "gitIntegrationId" text, + "sourceIntegrationId" text, + "insightsProjectId" text, + archived boolean, + "forkedFrom" text, + excluded boolean + ) + `, + { values: JSON.stringify(values) }, + ) + + // Insert into git.repositoryProcessing to sync into git integration worker + const repositoryIds = repositories.map((repo) => ({ repositoryId: repo.id })) + await qx.result( + ` + INSERT INTO git."repositoryProcessing" ( + "repositoryId", + "createdAt", + "updatedAt" + ) + SELECT + v."repositoryId"::uuid, + NOW(), + NOW() + FROM jsonb_to_recordset($(repositoryIds)::jsonb) AS v( + "repositoryId" text + ) + `, + { repositoryIds: JSON.stringify(repositoryIds) }, + ) +} + +/** + * Get repositories by source integration ID + * @param qx - Query executor + * @param sourceIntegrationId - The source integration ID + * @returns Array of repositories for the given integration (excluding soft-deleted) + */ +export async function getRepositoriesBySourceIntegrationId( + qx: QueryExecutor, + sourceIntegrationId: string, +): Promise { + return qx.select( + ` + SELECT + id, + url, + "segmentId", + "gitIntegrationId", + "sourceIntegrationId", + "insightsProjectId", + archived, + "forkedFrom", + excluded, + "createdAt", + "updatedAt", + "deletedAt", + "lastArchivedCheckAt" + FROM public.repositories + WHERE "sourceIntegrationId" = $(sourceIntegrationId) + AND "deletedAt" IS NULL + `, + { sourceIntegrationId }, + ) +} + +/** + * Get git repository IDs by URLs from git.repositories table + * @param qx - Query executor + * @param urls - Array of repository URLs + * @returns Map of URL to repository ID + */ +export async function getGitRepositoryIdsByUrl( + qx: QueryExecutor, + urls: string[], +): Promise> { + if (urls.length === 0) { + return new Map() + } + + const results = await qx.select( + ` + SELECT id, url + FROM git.repositories + WHERE url IN ($(urls:csv)) + `, + { urls }, + ) + + return new Map(results.map((row: { id: string; url: string }) => [row.url, row.id])) +} + +/** + * Get repositories by their URLs + * @param qx - Query executor + * @param repoUrls - Array of repository URLs to search for + * @param includeSoftDeleted - Whether to include soft-deleted repositories (default: false) + * @returns Array of repositories matching the given URLs + */ +export async function getRepositoriesByUrl( + qx: QueryExecutor, + repoUrls: string[], + includeSoftDeleted = false, +): Promise { + if (repoUrls.length === 0) { + return [] + } + + const deletedFilter = includeSoftDeleted ? '' : 'AND "deletedAt" IS NULL' + + return qx.select( + ` + SELECT + id, + url, + "segmentId", + "gitIntegrationId", + "sourceIntegrationId", + "insightsProjectId", + archived, + "forkedFrom", + excluded, + "createdAt", + "updatedAt", + "deletedAt", + "lastArchivedCheckAt" + FROM public.repositories + WHERE url IN ($(repoUrls:csv)) + ${deletedFilter} + `, + { repoUrls }, + ) +} From f6fc8631152d91dda851cb28f5380667ada16e7f Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 30 Dec 2025 14:32:05 +0100 Subject: [PATCH 02/17] feat: implement repos insertions and enable it for github-nango --- backend/src/services/integrationService.ts | 191 ++++++++++++++++++++- 1 file changed, 190 insertions(+), 1 deletion(-) diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index da784b7809..ab888da9ed 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -6,7 +6,7 @@ import lodash from 'lodash' import moment from 'moment' import { QueryTypes, Transaction } from 'sequelize' -import { EDITION, Error400, Error404, Error542, encryptData } from '@crowd/common' +import { EDITION, Error400, Error404, Error542, encryptData, generateUUIDv4 } from '@crowd/common' import { CommonIntegrationService, getGithubInstallationToken } from '@crowd/common_services' import { syncRepositoriesToGitV2 } from '@crowd/data-access-layer' import { @@ -16,6 +16,14 @@ import { upsertSegmentRepositories, } from '@crowd/data-access-layer/src/collections' import { findRepositoriesForSegment } from '@crowd/data-access-layer/src/integrations' +import { + getRepositoriesByUrl, + getRepositoriesBySourceIntegrationId, + getGitRepositoryIdsByUrl, + insertRepositories, + IRepository, + ICreateRepository, +} from '@crowd/data-access-layer/src/repositories' import { getGithubMappedRepos, getGitlabMappedRepos } from '@crowd/data-access-layer/src/segments' import { NangoIntegration, @@ -916,6 +924,7 @@ export default class IntegrationService { // create github mapping - this also creates git integration await txService.mapGithubRepos(integration.id, mapping, false) + } else { // update existing integration integration = await txService.findById(integrationId) @@ -945,6 +954,9 @@ export default class IntegrationService { ) } + // sync to public.repositories + await txService.mapUnifiedRepositories(PlatformType.GITHUB_NANGO, integration.id, mapping) + if (!existingTransaction) { await SequelizeRepository.commitTransaction(transaction) } @@ -3033,4 +3045,181 @@ export default class IntegrationService { ) return integration } + + private validateRepoIntegrationMapping( + existingRepos: IRepository[], + sourceIntegrationId: string, + ): void { + const integrationMismatches = existingRepos.filter( + (repo) => repo.deletedAt === null && repo.sourceIntegrationId !== sourceIntegrationId + ) + + if (integrationMismatches.length > 0) { + const mismatchDetails = integrationMismatches.map( + (repo) => `${repo.url} belongs to integration ${repo.sourceIntegrationId}` + ).join(', ') + throw new Error400( + this.options.language, + `Cannot remap repositories from different integration: ${mismatchDetails}` + ) + } + } + + /** + * Builds repository payloads for insertion into public.repositories + */ + private async buildRepositoryPayloads( + qx: any, + urls: string[], + mapping: { [url: string]: string }, + sourcePlatform: PlatformType, + sourceIntegrationId: string, + txOptions: IRepositoryOptions, + ): Promise { + if (urls.length === 0) { + return [] + } + + const segmentIds = [...new Set(urls.map((url) => mapping[url]))] + + const isGitHubPlatform = [PlatformType.GITHUB, PlatformType.GITHUB_NANGO].includes(sourcePlatform) + + const [gitRepoIdMap, sourceIntegration] = await Promise.all([ + // TODO: after migration, generate UUIDs instead of fetching from git.repositories + getGitRepositoryIdsByUrl(qx, urls), + isGitHubPlatform ? IntegrationRepository.findById(sourceIntegrationId, txOptions) : null, + ]) + + const collectionService = new CollectionService(txOptions) + const insightsProjectMap = new Map() + const gitIntegrationMap = new Map() + + for (const segmentId of segmentIds) { + const [insightsProject] = await collectionService.findInsightsProjectsBySegmentId(segmentId) + if (!insightsProject) { + throw new Error400(this.options.language, `Insights project not found for segment ${segmentId}`) + } + insightsProjectMap.set(segmentId, insightsProject.id) + + if (sourcePlatform === PlatformType.GIT) { + gitIntegrationMap.set(segmentId, sourceIntegrationId) + continue + } + + try { + const segmentOptions: IRepositoryOptions = { + ...txOptions, + currentSegments: [{ ...this.options.currentSegments[0], id: segmentId }], + } + const gitIntegration = await IntegrationRepository.findByPlatform( + PlatformType.GIT, + segmentOptions, + ) + gitIntegrationMap.set(segmentId, gitIntegration.id) + } catch { + throw new Error400(this.options.language, `Git integration not found for segment ${segmentId}`) + } + } + + // Build forkedFrom map from integration settings (for GITHUB repositories) + const forkedFromMap = new Map() + if (sourceIntegration?.settings?.orgs) { + const allRepos = sourceIntegration.settings.orgs.flatMap((org: any) => org.repos || []) + for (const repo of allRepos) { + if (repo.url && repo.forkedFrom) { + forkedFromMap.set(repo.url, repo.forkedFrom) + } + } + } + + // Build payloads + const payloads: ICreateRepository[] = [] + for (const url of urls) { + const segmentId = mapping[url] + let id = gitRepoIdMap.get(url) + const insightsProjectId = insightsProjectMap.get(segmentId) + const gitIntegrationId = gitIntegrationMap.get(segmentId) + + if (!id) { + // TODO: after migration, this will be the default behavior + this.options.log.warn(`No git.repositories ID found for URL ${url}, generating new UUID...`) + id = generateUUIDv4() + } + + payloads.push({ + id, + url, + segmentId, + gitIntegrationId, + sourceIntegrationId, + insightsProjectId, + forkedFrom: forkedFromMap.get(url) ?? null, + }) + } + + return payloads + } + + async mapUnifiedRepositories(sourcePlatform: PlatformType, sourceIntegrationId: string, mapping: { [url: string]: string }){ + const transaction = await SequelizeRepository.createTransaction(this.options) + const txOptions = { + ...this.options, + transaction, + } + + try { + const qx = SequelizeRepository.getQueryExecutor(txOptions) + const mappedUrls = Object.keys(mapping) + const mappedUrlSet = new Set(mappedUrls) + + const [existingMappedRepos, activeIntegrationRepos] = await Promise.all([ + getRepositoriesByUrl(qx, mappedUrls, true), + getRepositoriesBySourceIntegrationId(qx, sourceIntegrationId), + ]) + + // Block repos that belong to a different integration + this.validateRepoIntegrationMapping(existingMappedRepos, sourceIntegrationId) + + const existingUrlSet = new Set(existingMappedRepos.map((repo) => repo.url)) + const toInsertUrls = mappedUrls.filter((url) => !existingUrlSet.has(url)) + // Repos to restore: soft-deleted OR segment changed (both need re-onboarding) + const toRestoreRepos = existingMappedRepos.filter( + (repo) => repo.deletedAt !== null || repo.segmentId !== mapping[repo.url] + ) + const toSoftDeleteRepos = activeIntegrationRepos.filter((repo) => !mappedUrlSet.has(repo.url)) + + this.options.log.info( + `Repository mapping: ${toInsertUrls.length} to insert, ${toRestoreRepos.length} to restore, ${toSoftDeleteRepos.length} to soft-delete` + ) + + if (toInsertUrls.length > 0) { + this.options.log.info(`Inserting ${toInsertUrls.length} new repos into public.repositories...`) + const payloads = await this.buildRepositoryPayloads( + qx, + toInsertUrls, + mapping, + sourcePlatform, + sourceIntegrationId, + txOptions, + ) + if (payloads.length > 0) { + await insertRepositories(qx, payloads) + this.options.log.info(`Inserted ${payloads.length} repos into public.repositories`) + } + } + + // TODO: restore repos & re-onboard git integration + // TODO: implement soft-deletion + + await SequelizeRepository.commitTransaction(transaction) + } catch (err) { + this.options.log.error(err, 'Error while mapping unified repositories!') + try { + await SequelizeRepository.rollbackTransaction(transaction) + } catch (rErr) { + this.options.log.error(rErr, 'Error while rolling back transaction!') + } + throw err + } + } } From 3db29f38fe157cab55dd9f7c8e40bc4abe72a6da Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 30 Dec 2025 16:42:54 +0100 Subject: [PATCH 03/17] feat: enable unified mapping for rest of code platforms --- backend/src/services/integrationService.ts | 49 +++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index ab888da9ed..02728c1c63 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -270,6 +270,7 @@ export default class IntegrationService { remotes: repositories.map((url) => ({ url, forkedFrom: null })), }, txOptions, + platform, ) } @@ -459,6 +460,7 @@ export default class IntegrationService { remotes: remainingRemotes.map((url: string) => ({ url, forkedFrom: null })), }, segmentOptions, + integration.platform, ) } } @@ -1065,6 +1067,7 @@ export default class IntegrationService { }), }, segmentOptions, + PlatformType.GITHUB, ) } else { this.options.log.info(`Updating Git integration for segment ${segmentId}!`) @@ -1076,6 +1079,7 @@ export default class IntegrationService { }), }, segmentOptions, + PlatformType.GITHUB, ) } } @@ -1357,6 +1361,7 @@ export default class IntegrationService { * @param integrationData.remotes - Repository objects with url and optional forkedFrom (parent repo URL). * If forkedFrom is null, existing DB value is preserved. * @param options - Optional repository options + * @param sourcePlatform - If provided, mapUnifiedRepositories is skipped (caller handles it) * @returns Integration object or null if no remotes */ async gitConnectOrUpdate( @@ -1364,6 +1369,7 @@ export default class IntegrationService { remotes: Array<{ url: string; forkedFrom?: string | null }> }, options?: IRepositoryOptions, + sourcePlatform?: PlatformType, ) { const stripGit = (url: string) => { if (url.endsWith('.git')) { @@ -1436,6 +1442,7 @@ export default class IntegrationService { } // upsert repositories to git.repositories in order to be processed by git-integration V2 + const currentSegmentId = (options || this.options).currentSegments[0].id const qx = SequelizeRepository.getQueryExecutor({ ...(options || this.options), transaction, @@ -1444,9 +1451,22 @@ export default class IntegrationService { qx, remotes, integration.id, - (options || this.options).currentSegments[0].id, + currentSegmentId, ) + // sync to public.repositories (only for direct GIT connections, other platforms handle it themselves) + if (!sourcePlatform) { + const mapping = remotes.reduce((acc, remote) => { + acc[remote.url] = currentSegmentId + return acc + }, {} as Record) + + // Use service with transaction context so mapUnifiedRepositories joins this transaction + const txOptions = { ...(options || this.options), transaction } + const txService = new IntegrationService(txOptions) + await txService.mapUnifiedRepositories(PlatformType.GIT, integration.id, mapping) + } + // Only commit if we created the transaction ourselves if (!existingTransaction) { await SequelizeRepository.commitTransaction(transaction) @@ -1802,6 +1822,20 @@ export default class IntegrationService { transaction, ) + const stripGit = (url: string) => { + if (url.endsWith('.git')) { + return url.slice(0, -4) + } + return url + } + + // Build full repository URLs from orgURL and repo names + const currentSegmentId = this.options.currentSegments[0].id + const remotes = integrationData.remote.repoNames.map((repoName) => { + const fullUrl = stripGit(`${integrationData.remote.orgURL}/${repoName}`) + return { url: fullUrl, forkedFrom: null } + }) + if (integrationData.remote.enableGit) { const segmentOptions: IRepositoryOptions = { ...this.options, @@ -1818,9 +1852,20 @@ export default class IntegrationService { remotes, }, segmentOptions, + PlatformType.GERRIT, ) } + // sync to public.repositories + const mapping = remotes.reduce((acc, remote) => { + acc[remote.url] = currentSegmentId + return acc + }, {} as Record) + + const txOptions = { ...this.options, transaction } + const txService = new IntegrationService(txOptions) + await txService.mapUnifiedRepositories(PlatformType.GERRIT, integration.id, mapping) + await startNangoSync(NangoIntegration.GERRIT, connectionId) await SequelizeRepository.commitTransaction(transaction) @@ -2896,6 +2941,7 @@ export default class IntegrationService { }), }, { ...segmentOptions, transaction }, + PlatformType.GITLAB, ) } else { await this.gitConnectOrUpdate( @@ -2906,6 +2952,7 @@ export default class IntegrationService { }), }, { ...segmentOptions, transaction }, + PlatformType.GITLAB, ) } } From deab37198c16f388b0047eedbeb09f7dc58ec07f Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 30 Dec 2025 17:45:12 +0100 Subject: [PATCH 04/17] feat: implement restore & delete --- backend/src/services/integrationService.ts | 27 ++++- .../src/repositories/index.ts | 108 ++++++++++++++++-- 2 files changed, 125 insertions(+), 10 deletions(-) diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index 02728c1c63..99a7bce1ca 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -23,6 +23,8 @@ import { insertRepositories, IRepository, ICreateRepository, + softDeleteRepositories, + restoreRepositories, } from '@crowd/data-access-layer/src/repositories' import { getGithubMappedRepos, getGitlabMappedRepos } from '@crowd/data-access-layer/src/segments' import { @@ -3255,8 +3257,29 @@ export default class IntegrationService { } } - // TODO: restore repos & re-onboard git integration - // TODO: implement soft-deletion + if (toRestoreRepos.length > 0) { + this.options.log.info(`Restoring ${toRestoreRepos.length} repos in public.repositories...`) + const toRestoreUrls = toRestoreRepos.map((repo) => repo.url) + const restorePayloads = await this.buildRepositoryPayloads( + qx, + toRestoreUrls, + mapping, + sourcePlatform, + sourceIntegrationId, + txOptions, + ) + if (restorePayloads.length > 0) { + await restoreRepositories(qx, restorePayloads) + this.options.log.info(`Restored ${restorePayloads.length} repos in public.repositories`) + } + } + + if (toSoftDeleteRepos.length > 0) { + this.options.log.info(`Soft-deleting ${toSoftDeleteRepos.length} repos from public.repositories...`) + //TODO: post migration, add sourceIntegrationId to the delete condition to avoid cross-integration conflicts + await softDeleteRepositories(qx, toSoftDeleteRepos.map((repo) => repo.url)) + this.options.log.info(`Soft-deleted ${toSoftDeleteRepos.length} repos from public.repositories`) + } await SequelizeRepository.commitTransaction(transaction) } catch (err) { diff --git a/services/libs/data-access-layer/src/repositories/index.ts b/services/libs/data-access-layer/src/repositories/index.ts index b5535114a0..02bca09f2d 100644 --- a/services/libs/data-access-layer/src/repositories/index.ts +++ b/services/libs/data-access-layer/src/repositories/index.ts @@ -31,14 +31,6 @@ export interface ICreateRepository { excluded?: boolean } -export async function createRepository( - qx: QueryExecutor, - data: ICreateRepository, -): Promise { - // TODO: Implement - throw new Error('Not implemented') -} - /** * Bulk inserts repositories into public.repositories and git.repositoryProcessing * @param qx - Query executor (should be transactional) @@ -228,3 +220,103 @@ export async function getRepositoriesByUrl( { repoUrls }, ) } + +/** + * Soft deletes repositories by setting deletedAt = NOW() + * @param qx - Query executor + * @param urls - Array of repository URLs to soft delete + * @returns Number of rows affected + */ +export async function softDeleteRepositories(qx: QueryExecutor, urls: string[]): Promise { + if (urls.length === 0) { + return 0 + } + + return qx.result( + ` + UPDATE public.repositories + SET "deletedAt" = NOW(), "updatedAt" = NOW() + WHERE url IN ($(urls:csv)) + AND "deletedAt" IS NULL + `, + { urls }, + ) +} + +/** + * Restores soft-deleted and/or updated repositories and resets their processing state + * Updates fields in public.repositories and resets git.repositoryProcessing for re-onboarding + * @param qx - Query executor + * @param repositories - Array of repositories with url (required) and optional fields to update + */ +export async function restoreRepositories( + qx: QueryExecutor, + repositories: Partial[], +): Promise { + if (repositories.length === 0) { + return + } + + const urls = repositories.map((repo) => repo.url).filter(Boolean) as string[] + if (urls.length === 0) { + return + } + + const values = repositories.map((repo) => ({ + url: repo.url, + segmentId: repo.segmentId ?? null, + gitIntegrationId: repo.gitIntegrationId ?? null, + sourceIntegrationId: repo.sourceIntegrationId ?? null, + insightsProjectId: repo.insightsProjectId ?? null, + archived: repo.archived ?? null, + forkedFrom: repo.forkedFrom ?? null, + excluded: repo.excluded ?? null, + })) + + await qx.result( + ` + UPDATE public.repositories r + SET + "segmentId" = COALESCE(v."segmentId"::uuid, r."segmentId"), + "gitIntegrationId" = COALESCE(v."gitIntegrationId"::uuid, r."gitIntegrationId"), + "sourceIntegrationId" = COALESCE(v."sourceIntegrationId"::uuid, r."sourceIntegrationId"), + "insightsProjectId" = COALESCE(v."insightsProjectId"::uuid, r."insightsProjectId"), + archived = COALESCE(v.archived::boolean, r.archived), + "forkedFrom" = COALESCE(v."forkedFrom", r."forkedFrom"), + excluded = COALESCE(v.excluded::boolean, r.excluded), + "deletedAt" = NULL, + "updatedAt" = NOW() + FROM jsonb_to_recordset($(values)::jsonb) AS v( + url text, + "segmentId" text, + "gitIntegrationId" text, + "sourceIntegrationId" text, + "insightsProjectId" text, + archived boolean, + "forkedFrom" text, + excluded boolean + ) + WHERE r.url = v.url + `, + { values: JSON.stringify(values) }, + ) + + // Reset git.repositoryProcessing for git re-onboarding + await qx.result( + ` + UPDATE git."repositoryProcessing" rp + SET + "lastProcessedAt" = NULL, + "lastProcessedCommit" = NULL, + "lastMaintainerRunAt" = NULL, + branch = NULL, + "lockedAt" = NULL, + state = 'pending', + "updatedAt" = NOW() + FROM public.repositories r + WHERE rp."repositoryId" = r.id + AND r.url IN ($(urls:csv)) + `, + { urls }, + ) +} From 058d85ba3d1c7531d5bedc9f2e7867bf2d021cd9 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 30 Dec 2025 17:50:20 +0100 Subject: [PATCH 05/17] fix: lint --- backend/src/services/integrationService.ts | 29 +++++++++++----------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index 99a7bce1ca..4a2bb3ade4 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -3152,21 +3152,20 @@ export default class IntegrationService { if (sourcePlatform === PlatformType.GIT) { gitIntegrationMap.set(segmentId, sourceIntegrationId) - continue - } - - try { - const segmentOptions: IRepositoryOptions = { - ...txOptions, - currentSegments: [{ ...this.options.currentSegments[0], id: segmentId }], + } else { + try { + const segmentOptions: IRepositoryOptions = { + ...txOptions, + currentSegments: [{ ...this.options.currentSegments[0], id: segmentId }], + } + const gitIntegration = await IntegrationRepository.findByPlatform( + PlatformType.GIT, + segmentOptions, + ) + gitIntegrationMap.set(segmentId, gitIntegration.id) + } catch { + throw new Error400(this.options.language, `Git integration not found for segment ${segmentId}`) } - const gitIntegration = await IntegrationRepository.findByPlatform( - PlatformType.GIT, - segmentOptions, - ) - gitIntegrationMap.set(segmentId, gitIntegration.id) - } catch { - throw new Error400(this.options.language, `Git integration not found for segment ${segmentId}`) } } @@ -3276,7 +3275,7 @@ export default class IntegrationService { if (toSoftDeleteRepos.length > 0) { this.options.log.info(`Soft-deleting ${toSoftDeleteRepos.length} repos from public.repositories...`) - //TODO: post migration, add sourceIntegrationId to the delete condition to avoid cross-integration conflicts + // TODO: post migration, add sourceIntegrationId to the delete condition to avoid cross-integration conflicts await softDeleteRepositories(qx, toSoftDeleteRepos.map((repo) => repo.url)) this.options.log.info(`Soft-deleted ${toSoftDeleteRepos.length} repos from public.repositories`) } From d9fa51b2bc51ae14a1f6acccf4c5455fbd556e37 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 30 Dec 2025 17:54:03 +0100 Subject: [PATCH 06/17] fix: formatting --- backend/src/services/integrationService.ts | 91 +++++++++++++--------- 1 file changed, 56 insertions(+), 35 deletions(-) diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index 4a2bb3ade4..00e1958181 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -17,14 +17,14 @@ import { } from '@crowd/data-access-layer/src/collections' import { findRepositoriesForSegment } from '@crowd/data-access-layer/src/integrations' import { - getRepositoriesByUrl, - getRepositoriesBySourceIntegrationId, + ICreateRepository, + IRepository, getGitRepositoryIdsByUrl, + getRepositoriesBySourceIntegrationId, + getRepositoriesByUrl, insertRepositories, - IRepository, - ICreateRepository, - softDeleteRepositories, restoreRepositories, + softDeleteRepositories, } from '@crowd/data-access-layer/src/repositories' import { getGithubMappedRepos, getGitlabMappedRepos } from '@crowd/data-access-layer/src/segments' import { @@ -928,7 +928,6 @@ export default class IntegrationService { // create github mapping - this also creates git integration await txService.mapGithubRepos(integration.id, mapping, false) - } else { // update existing integration integration = await txService.findById(integrationId) @@ -1449,19 +1448,17 @@ export default class IntegrationService { ...(options || this.options), transaction, }) - await syncRepositoriesToGitV2( - qx, - remotes, - integration.id, - currentSegmentId, - ) + await syncRepositoriesToGitV2(qx, remotes, integration.id, currentSegmentId) // sync to public.repositories (only for direct GIT connections, other platforms handle it themselves) if (!sourcePlatform) { - const mapping = remotes.reduce((acc, remote) => { - acc[remote.url] = currentSegmentId - return acc - }, {} as Record) + const mapping = remotes.reduce( + (acc, remote) => { + acc[remote.url] = currentSegmentId + return acc + }, + {} as Record, + ) // Use service with transaction context so mapUnifiedRepositories joins this transaction const txOptions = { ...(options || this.options), transaction } @@ -1859,10 +1856,13 @@ export default class IntegrationService { } // sync to public.repositories - const mapping = remotes.reduce((acc, remote) => { - acc[remote.url] = currentSegmentId - return acc - }, {} as Record) + const mapping = remotes.reduce( + (acc, remote) => { + acc[remote.url] = currentSegmentId + return acc + }, + {} as Record, + ) const txOptions = { ...this.options, transaction } const txService = new IntegrationService(txOptions) @@ -3100,16 +3100,16 @@ export default class IntegrationService { sourceIntegrationId: string, ): void { const integrationMismatches = existingRepos.filter( - (repo) => repo.deletedAt === null && repo.sourceIntegrationId !== sourceIntegrationId + (repo) => repo.deletedAt === null && repo.sourceIntegrationId !== sourceIntegrationId, ) if (integrationMismatches.length > 0) { - const mismatchDetails = integrationMismatches.map( - (repo) => `${repo.url} belongs to integration ${repo.sourceIntegrationId}` - ).join(', ') + const mismatchDetails = integrationMismatches + .map((repo) => `${repo.url} belongs to integration ${repo.sourceIntegrationId}`) + .join(', ') throw new Error400( this.options.language, - `Cannot remap repositories from different integration: ${mismatchDetails}` + `Cannot remap repositories from different integration: ${mismatchDetails}`, ) } } @@ -3131,7 +3131,9 @@ export default class IntegrationService { const segmentIds = [...new Set(urls.map((url) => mapping[url]))] - const isGitHubPlatform = [PlatformType.GITHUB, PlatformType.GITHUB_NANGO].includes(sourcePlatform) + const isGitHubPlatform = [PlatformType.GITHUB, PlatformType.GITHUB_NANGO].includes( + sourcePlatform, + ) const [gitRepoIdMap, sourceIntegration] = await Promise.all([ // TODO: after migration, generate UUIDs instead of fetching from git.repositories @@ -3146,7 +3148,10 @@ export default class IntegrationService { for (const segmentId of segmentIds) { const [insightsProject] = await collectionService.findInsightsProjectsBySegmentId(segmentId) if (!insightsProject) { - throw new Error400(this.options.language, `Insights project not found for segment ${segmentId}`) + throw new Error400( + this.options.language, + `Insights project not found for segment ${segmentId}`, + ) } insightsProjectMap.set(segmentId, insightsProject.id) @@ -3164,7 +3169,10 @@ export default class IntegrationService { ) gitIntegrationMap.set(segmentId, gitIntegration.id) } catch { - throw new Error400(this.options.language, `Git integration not found for segment ${segmentId}`) + throw new Error400( + this.options.language, + `Git integration not found for segment ${segmentId}`, + ) } } } @@ -3208,7 +3216,11 @@ export default class IntegrationService { return payloads } - async mapUnifiedRepositories(sourcePlatform: PlatformType, sourceIntegrationId: string, mapping: { [url: string]: string }){ + async mapUnifiedRepositories( + sourcePlatform: PlatformType, + sourceIntegrationId: string, + mapping: { [url: string]: string }, + ) { const transaction = await SequelizeRepository.createTransaction(this.options) const txOptions = { ...this.options, @@ -3232,16 +3244,18 @@ export default class IntegrationService { const toInsertUrls = mappedUrls.filter((url) => !existingUrlSet.has(url)) // Repos to restore: soft-deleted OR segment changed (both need re-onboarding) const toRestoreRepos = existingMappedRepos.filter( - (repo) => repo.deletedAt !== null || repo.segmentId !== mapping[repo.url] + (repo) => repo.deletedAt !== null || repo.segmentId !== mapping[repo.url], ) const toSoftDeleteRepos = activeIntegrationRepos.filter((repo) => !mappedUrlSet.has(repo.url)) this.options.log.info( - `Repository mapping: ${toInsertUrls.length} to insert, ${toRestoreRepos.length} to restore, ${toSoftDeleteRepos.length} to soft-delete` + `Repository mapping: ${toInsertUrls.length} to insert, ${toRestoreRepos.length} to restore, ${toSoftDeleteRepos.length} to soft-delete`, ) if (toInsertUrls.length > 0) { - this.options.log.info(`Inserting ${toInsertUrls.length} new repos into public.repositories...`) + this.options.log.info( + `Inserting ${toInsertUrls.length} new repos into public.repositories...`, + ) const payloads = await this.buildRepositoryPayloads( qx, toInsertUrls, @@ -3274,10 +3288,17 @@ export default class IntegrationService { } if (toSoftDeleteRepos.length > 0) { - this.options.log.info(`Soft-deleting ${toSoftDeleteRepos.length} repos from public.repositories...`) + this.options.log.info( + `Soft-deleting ${toSoftDeleteRepos.length} repos from public.repositories...`, + ) // TODO: post migration, add sourceIntegrationId to the delete condition to avoid cross-integration conflicts - await softDeleteRepositories(qx, toSoftDeleteRepos.map((repo) => repo.url)) - this.options.log.info(`Soft-deleted ${toSoftDeleteRepos.length} repos from public.repositories`) + await softDeleteRepositories( + qx, + toSoftDeleteRepos.map((repo) => repo.url), + ) + this.options.log.info( + `Soft-deleted ${toSoftDeleteRepos.length} repos from public.repositories`, + ) } await SequelizeRepository.commitTransaction(transaction) From 15ea1a938373ce763a9bbd766afa4d1c28e6c054 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 31 Dec 2025 11:39:50 +0100 Subject: [PATCH 07/17] feat: enable unified mapping for gitlab --- backend/src/services/integrationService.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index 00e1958181..3c6bbe4ea0 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -2958,6 +2958,10 @@ export default class IntegrationService { ) } } + + // sync to public.repositories + const txService = new IntegrationService(txOptions) + await txService.mapUnifiedRepositories(PlatformType.GITLAB, integrationId, mapping) } const integration = await IntegrationRepository.update( From e0c4da12337321cf5fc1eb140310218d95174854 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 31 Dec 2025 16:03:49 +0100 Subject: [PATCH 08/17] feat: validate repo ownership in deletions --- backend/src/services/integrationService.ts | 57 ++++++++++++++++++- .../src/repositories/index.ts | 11 +++- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index 3c6bbe4ea0..60f5edfe07 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -16,6 +16,7 @@ import { upsertSegmentRepositories, } from '@crowd/data-access-layer/src/collections' import { findRepositoriesForSegment } from '@crowd/data-access-layer/src/integrations' +import { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' import { ICreateRepository, IRepository, @@ -518,12 +519,23 @@ export default class IntegrationService { // Soft delete git.repositories for git integration if (integration.platform === PlatformType.GIT) { + await this.validateGitIntegrationDeletion(integration.id, { + ...this.options, + transaction, + }) + await GitReposRepository.delete(integration.id, { ...this.options, transaction, }) } + // Soft delete from public.repositories for code integrations + if (IntegrationService.isCodePlatform(integration.platform)) { + const txService = new IntegrationService({ ...this.options, transaction }) + await txService.mapUnifiedRepositories(integration.platform, integration.id, {}) + } + await IntegrationRepository.destroy(id, { ...this.options, transaction, @@ -3118,6 +3130,47 @@ export default class IntegrationService { } } + private validateReposOwnership(repos: IRepository[], sourceIntegrationId: string): void { + const ownershipMismatches = repos.filter( + (repo) => repo.sourceIntegrationId !== sourceIntegrationId, + ) + + if (ownershipMismatches.length > 0) { + const mismatchUrls = ownershipMismatches.map((repo) => repo.url).join(', ') + throw new Error400( + this.options.language, + `These repos are managed by another integration: ${mismatchUrls}`, + ) + } + } + + private async validateGitIntegrationDeletion( + gitIntegrationId: string, + options: IRepositoryOptions, + ): Promise { + const qx = SequelizeRepository.getQueryExecutor(options) + + // Find repos linked to this GIT integration but owned by a different integration + const ownedByOthers = await qx.select( + ` + SELECT url + FROM public.repositories + WHERE "gitIntegrationId" = $(gitIntegrationId) + AND "sourceIntegrationId" != $(gitIntegrationId) + AND "deletedAt" IS NULL + `, + { gitIntegrationId }, + ) + + if (ownedByOthers.length > 0) { + const mismatchUrls = ownedByOthers.map((repo: { url: string }) => repo.url).join(', ') + throw new Error400( + this.options.language, + `Cannot delete GIT integration: these repos are managed by another integration: ${mismatchUrls}`, + ) + } + } + /** * Builds repository payloads for insertion into public.repositories */ @@ -3292,13 +3345,15 @@ export default class IntegrationService { } if (toSoftDeleteRepos.length > 0) { + this.validateReposOwnership(toSoftDeleteRepos, sourceIntegrationId) + this.options.log.info( `Soft-deleting ${toSoftDeleteRepos.length} repos from public.repositories...`, ) - // TODO: post migration, add sourceIntegrationId to the delete condition to avoid cross-integration conflicts await softDeleteRepositories( qx, toSoftDeleteRepos.map((repo) => repo.url), + sourceIntegrationId, ) this.options.log.info( `Soft-deleted ${toSoftDeleteRepos.length} repos from public.repositories`, diff --git a/services/libs/data-access-layer/src/repositories/index.ts b/services/libs/data-access-layer/src/repositories/index.ts index 02bca09f2d..3607884eea 100644 --- a/services/libs/data-access-layer/src/repositories/index.ts +++ b/services/libs/data-access-layer/src/repositories/index.ts @@ -223,11 +223,17 @@ export async function getRepositoriesByUrl( /** * Soft deletes repositories by setting deletedAt = NOW() + * Only deletes repos matching both the URLs and sourceIntegrationId * @param qx - Query executor * @param urls - Array of repository URLs to soft delete + * @param sourceIntegrationId - Only delete repos belonging to this integration * @returns Number of rows affected */ -export async function softDeleteRepositories(qx: QueryExecutor, urls: string[]): Promise { +export async function softDeleteRepositories( + qx: QueryExecutor, + urls: string[], + sourceIntegrationId: string, +): Promise { if (urls.length === 0) { return 0 } @@ -237,9 +243,10 @@ export async function softDeleteRepositories(qx: QueryExecutor, urls: string[]): UPDATE public.repositories SET "deletedAt" = NOW(), "updatedAt" = NOW() WHERE url IN ($(urls:csv)) + AND "sourceIntegrationId" = $(sourceIntegrationId) AND "deletedAt" IS NULL `, - { urls }, + { urls, sourceIntegrationId }, ) } From f2deee8f94e8bcfab67059115d1294de821cd541 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 31 Dec 2025 16:10:14 +0100 Subject: [PATCH 09/17] fix: gerrit bug of overriding existing remotes in git integration --- backend/src/services/integrationService.ts | 36 +++++++++++++++++----- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index 60f5edfe07..9ceb9aa5e6 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -1858,13 +1858,35 @@ export default class IntegrationService { ], } - await this.gitConnectOrUpdate( - { - remotes, - }, - segmentOptions, - PlatformType.GERRIT, - ) + // Check if git integration already exists and merge remotes + let isGitIntegrationConfigured = false + try { + await IntegrationRepository.findByPlatform(PlatformType.GIT, segmentOptions) + isGitIntegrationConfigured = true + } catch (err) { + isGitIntegrationConfigured = false + } + + if (isGitIntegrationConfigured) { + const gitInfo = await this.gitGetRemotes(segmentOptions) + const gitRemotes = gitInfo[currentSegmentId]?.remotes || [] + const allUrls = Array.from(new Set([...gitRemotes, ...remotes.map((r) => r.url)])) + await this.gitConnectOrUpdate( + { + remotes: allUrls.map((url) => ({ url, forkedFrom: null })), + }, + segmentOptions, + PlatformType.GERRIT, + ) + } else { + await this.gitConnectOrUpdate( + { + remotes, + }, + segmentOptions, + PlatformType.GERRIT, + ) + } } // sync to public.repositories From 57f09d382e98ce18b7d038e7ee88fe527d5bf9c1 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 31 Dec 2025 17:23:44 +0100 Subject: [PATCH 10/17] fix: always enable git integration for gerrit to ensure code platform consistency --- .../gerrit/components/gerrit-settings-drawer.vue | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/frontend/src/config/integrations/gerrit/components/gerrit-settings-drawer.vue b/frontend/src/config/integrations/gerrit/components/gerrit-settings-drawer.vue index b0de5d0007..d748336675 100644 --- a/frontend/src/config/integrations/gerrit/components/gerrit-settings-drawer.vue +++ b/frontend/src/config/integrations/gerrit/components/gerrit-settings-drawer.vue @@ -71,9 +71,9 @@ Enable All Projects - + @@ -139,7 +139,7 @@ const form = reactive({ // user: '', // pass: '', enableAllRepos: false, - enableGit: false, + enableGit: true, repoNames: [], }); @@ -164,7 +164,6 @@ onMounted(() => { // form.pass = props.integration?.settings.remote.pass; form.repoNames = props.integration?.settings.remote.repoNames; form.enableAllRepos = props.integration?.settings.remote.enableAllRepos; - form.enableGit = props.integration?.settings.remote.enableGit; } formSnapshot(); }); From 2f05800036165dade40c5c63d69571294d7c4f90 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Mon, 5 Jan 2026 13:26:03 +0100 Subject: [PATCH 11/17] fix: transactions handling & ensure repo id consistency --- backend/src/services/integrationService.ts | 28 +++++++++++++++------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index 9ceb9aa5e6..7e38ff2042 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -6,7 +6,7 @@ import lodash from 'lodash' import moment from 'moment' import { QueryTypes, Transaction } from 'sequelize' -import { EDITION, Error400, Error404, Error542, encryptData, generateUUIDv4 } from '@crowd/common' +import { EDITION, Error400, Error404, Error500, Error542, encryptData } from '@crowd/common' import { CommonIntegrationService, getGithubInstallationToken } from '@crowd/common_services' import { syncRepositoriesToGitV2 } from '@crowd/data-access-layer' import { @@ -3276,9 +3276,9 @@ export default class IntegrationService { const gitIntegrationId = gitIntegrationMap.get(segmentId) if (!id) { - // TODO: after migration, this will be the default behavior + // TODO: post migration generate id and remove lookup this.options.log.warn(`No git.repositories ID found for URL ${url}, generating new UUID...`) - id = generateUUIDv4() + throw new Error500(this.options.language, 'Repo not found in git.repositories') } payloads.push({ @@ -3300,7 +3300,11 @@ export default class IntegrationService { sourceIntegrationId: string, mapping: { [url: string]: string }, ) { - const transaction = await SequelizeRepository.createTransaction(this.options) + // Check for existing transaction to support nested calls within outer transactions + const existingTransaction = SequelizeRepository.getTransaction(this.options) + const transaction = + existingTransaction || (await SequelizeRepository.createTransaction(this.options)) + const txOptions = { ...this.options, transaction, @@ -3382,13 +3386,19 @@ export default class IntegrationService { ) } - await SequelizeRepository.commitTransaction(transaction) + // Only commit if we created the transaction ourselves + if (!existingTransaction) { + await SequelizeRepository.commitTransaction(transaction) + } } catch (err) { this.options.log.error(err, 'Error while mapping unified repositories!') - try { - await SequelizeRepository.rollbackTransaction(transaction) - } catch (rErr) { - this.options.log.error(rErr, 'Error while rolling back transaction!') + // Only rollback if we created the transaction ourselves + if (!existingTransaction) { + try { + await SequelizeRepository.rollbackTransaction(transaction) + } catch (rErr) { + this.options.log.error(rErr, 'Error while rolling back transaction!') + } } throw err } From 07080eb36dfe3af502e67a3233e41cab5e28c4ce Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Mon, 5 Jan 2026 17:50:29 +0100 Subject: [PATCH 12/17] feat: unify get repositories by integration --- .../src/api/integration/helpers/githubMapReposGet.ts | 9 --------- .../src/api/integration/helpers/gitlabMapReposGet.ts | 9 --------- backend/src/api/integration/index.ts | 10 ++++++++-- 3 files changed, 8 insertions(+), 20 deletions(-) delete mode 100644 backend/src/api/integration/helpers/githubMapReposGet.ts delete mode 100644 backend/src/api/integration/helpers/gitlabMapReposGet.ts diff --git a/backend/src/api/integration/helpers/githubMapReposGet.ts b/backend/src/api/integration/helpers/githubMapReposGet.ts deleted file mode 100644 index a630c64252..0000000000 --- a/backend/src/api/integration/helpers/githubMapReposGet.ts +++ /dev/null @@ -1,9 +0,0 @@ -import Permissions from '../../../security/permissions' -import IntegrationService from '../../../services/integrationService' -import PermissionChecker from '../../../services/user/permissionChecker' - -export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.tenantEdit) - const payload = await new IntegrationService(req).getGithubRepos(req.params.id) - await req.responseHandler.success(req, res, payload) -} diff --git a/backend/src/api/integration/helpers/gitlabMapReposGet.ts b/backend/src/api/integration/helpers/gitlabMapReposGet.ts deleted file mode 100644 index f66c808aec..0000000000 --- a/backend/src/api/integration/helpers/gitlabMapReposGet.ts +++ /dev/null @@ -1,9 +0,0 @@ -import Permissions from '../../../security/permissions' -import IntegrationService from '../../../services/integrationService' -import PermissionChecker from '../../../services/user/permissionChecker' - -export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.tenantEdit) - const payload = await new IntegrationService(req).getGitlabRepos(req.params.id) - await req.responseHandler.success(req, res, payload) -} diff --git a/backend/src/api/integration/index.ts b/backend/src/api/integration/index.ts index a40350fe4a..cfce181007 100644 --- a/backend/src/api/integration/index.ts +++ b/backend/src/api/integration/index.ts @@ -40,9 +40,15 @@ export default (app) => { app.get(`/integration`, safeWrap(require('./integrationList').default)) app.get(`/integration/:id`, safeWrap(require('./integrationFind').default)) + // Unified endpoint for all code platform integrations (github, gitlab, git, gerrit) + app.get( + `/integration/:id/repositories`, + safeWrap(require('./helpers/getIntegrationRepositories').default), + ) + app.put(`/authenticate/:code`, safeWrap(require('./helpers/githubAuthenticate').default)) app.put(`/integration/:id/github/repos`, safeWrap(require('./helpers/githubMapRepos').default)) - app.get(`/integration/:id/github/repos`, safeWrap(require('./helpers/githubMapReposGet').default)) + app.get( `/integration/github/search/orgs`, safeWrap(require('./helpers/githubSearchOrgs').default), @@ -110,7 +116,7 @@ export default (app) => { app.get('/gitlab/callback', safeWrap(require('./helpers/gitlabAuthenticateCallback').default)) app.put(`/integration/:id/gitlab/repos`, safeWrap(require('./helpers/gitlabMapRepos').default)) - app.get(`/integration/:id/gitlab/repos`, safeWrap(require('./helpers/gitlabMapReposGet').default)) + if (TWITTER_CONFIG.clientId) { /** From 59f1aaaa659b59f8f4e4b05a522c8615908a8210 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Mon, 5 Jan 2026 17:52:03 +0100 Subject: [PATCH 13/17] fix: handle edge case & cross integration update/delete --- backend/src/services/integrationService.ts | 252 +++++++++++++----- .../src/integrations/index.ts | 59 ++-- .../src/repositories/index.ts | 45 ++++ 3 files changed, 255 insertions(+), 101 deletions(-) diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index 7e38ff2042..27da1b3d6a 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -20,7 +20,9 @@ import { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' import { ICreateRepository, IRepository, + IRepositoryMapping, getGitRepositoryIdsByUrl, + getIntegrationReposMapping, getRepositoriesBySourceIntegrationId, getRepositoriesByUrl, insertRepositories, @@ -374,7 +376,6 @@ export default class IntegrationService { integration.platform === PlatformType.GITHUB_NANGO || integration.platform === PlatformType.GERRIT ) { - let shouldUpdateGit: boolean let repos: Record = {} // Get repos based on platform @@ -397,20 +398,23 @@ export default class IntegrationService { repos[integration.segmentId] = gerritUrls } } else { - // For GitHub/GitLab, use mapping tables - const mapping = - integration.platform === PlatformType.GITHUB || - integration.platform === PlatformType.GITHUB_NANGO - ? await this.getGithubRepos(id) - : await this.getGitlabRepos(id) - - repos = mapping.reduce((acc, { url, segment }) => { - if (!acc[segment.id]) { - acc[segment.id] = [] - } - acc[segment.id].push(url) - return acc - }, {}) + // Use public.repositories to get repos owned by this integration + const qx = SequelizeRepository.getQueryExecutor({ + ...this.options, + transaction, + }) + const integrationRepos = await getRepositoriesBySourceIntegrationId(qx, id) + + repos = integrationRepos.reduce( + (acc, repo) => { + if (!acc[repo.segmentId]) { + acc[repo.segmentId] = [] + } + acc[repo.segmentId].push(repo.url) + return acc + }, + {} as Record, + ) } for (const [segmentId, urls] of Object.entries(repos)) { @@ -426,46 +430,66 @@ export default class IntegrationService { ], } - try { - await IntegrationRepository.findByPlatform(PlatformType.GIT, segmentOptions) - shouldUpdateGit = true - } catch (err) { - shouldUpdateGit = false - } + const gitIntegration = await IntegrationRepository.findByPlatform( + PlatformType.GIT, + segmentOptions, + ) - if (shouldUpdateGit) { - const gitInfo = await this.gitGetRemotes(segmentOptions) - const gitRemotes = gitInfo[segmentId].remotes - const remainingRemotes = gitRemotes.filter((remote) => !urls.includes(remote)) + // Get all repos for this git integration from public.repositories + const qxForGit = SequelizeRepository.getQueryExecutor({ + ...this.options, + transaction, + }) + const allGitRepos = await getIntegrationReposMapping(qxForGit, gitIntegration.id) - if (remainingRemotes.length === 0) { - // If no remotes left, delete the Git integration entirely - const gitIntegration = await IntegrationRepository.findByPlatform( - PlatformType.GIT, - segmentOptions, - ) + // Filter to get repos NOT owned by the source integration being deleted + const remainingRepos = allGitRepos.filter( + (repo) => repo.sourceIntegrationId !== id, + ) - // Soft delete git.repositories for git-integration V2 - await GitReposRepository.delete(gitIntegration.id, { - ...this.options, - transaction, - }) - - // Then delete the git integration - await IntegrationRepository.destroy(gitIntegration.id, { - ...this.options, - transaction, - }) - } else { - // Update with remaining remotes - await this.gitConnectOrUpdate( - { - remotes: remainingRemotes.map((url: string) => ({ url, forkedFrom: null })), - }, - segmentOptions, - integration.platform, + if (remainingRepos.length === 0) { + // If no repos left, delete the Git integration entirely + // Soft delete git.repositories for git-integration V2 + await GitReposRepository.delete(gitIntegration.id, { + ...this.options, + transaction, + }) + + // Then delete the git integration + await IntegrationRepository.destroy(gitIntegration.id, { + ...this.options, + transaction, + }) + } else { + // Soft delete from git.repositories only the repos owned by the deleted integration + const urlsToRemove = allGitRepos + .filter((repo) => repo.sourceIntegrationId === id) + .map((r) => r.url) + + if (urlsToRemove.length > 0) { + await qxForGit.result( + ` + UPDATE git.repositories + SET "deletedAt" = NOW() + WHERE url IN ($(urlsToRemove:csv)) + AND "deletedAt" IS NULL + `, + { urlsToRemove }, + ) + this.options.log.info( + `Soft deleted ${urlsToRemove.length} repos from git.repositories for integration ${id}`, ) } + + // Update git integration settings with remaining remotes + const remainingRemotes = remainingRepos.map((r) => r.url) + await this.gitConnectOrUpdate( + { + remotes: remainingRemotes.map((url: string) => ({ url, forkedFrom: null })), + }, + segmentOptions, + integration.platform, + ) } } @@ -473,29 +497,24 @@ export default class IntegrationService { integration.platform === PlatformType.GITHUB || integration.platform === PlatformType.GITHUB_NANGO ) { - // soft delete github repos + // soft delete github repos from legacy table await GithubReposRepository.delete(integration.id, { ...this.options, transaction, }) - // Also soft delete from git.repositories for git-integration V2 - try { - // Find the Git integration ID for this segment - const gitIntegration = await IntegrationRepository.findByPlatform(PlatformType.GIT, { - ...this.options, - currentSegments: [{ id: integration.segmentId } as any], - transaction, - }) - if (gitIntegration) { - await GitReposRepository.delete(gitIntegration.id, { - ...this.options, - transaction, - }) - } - } catch (err) { + // Soft delete from public.repositories only repos owned by this GitHub integration + // This preserves native Git repos that aren't mirrored from GitHub + const qx = SequelizeRepository.getQueryExecutor({ + ...this.options, + transaction, + }) + const reposToDelete = await getRepositoriesBySourceIntegrationId(qx, integration.id) + if (reposToDelete.length > 0) { + const urlsToDelete = reposToDelete.map((r) => r.url) + await softDeleteRepositories(qx, urlsToDelete, integration.id) this.options.log.info( - 'No Git integration found for segment, skipping git.repositories cleanup', + `Soft deleted ${urlsToDelete.length} repos from public.repositories for GitHub integration ${integration.id}`, ) } } @@ -533,7 +552,8 @@ export default class IntegrationService { // Soft delete from public.repositories for code integrations if (IntegrationService.isCodePlatform(integration.platform)) { const txService = new IntegrationService({ ...this.options, transaction }) - await txService.mapUnifiedRepositories(integration.platform, integration.id, {}) + // When destroying, don't skip mirrored repos - delete all + await txService.mapUnifiedRepositories(integration.platform, integration.id, {}, false) } await IntegrationRepository.destroy(id, { @@ -1141,6 +1161,17 @@ export default class IntegrationService { } } + /** + * Get repository mappings for an integration + * Uses the unified public.repositories table instead of legacy githubRepos table + * @param integrationId - The source integration ID to filter by + * @returns Array of repositories with segment info and integration IDs + */ + async getIntegrationRepositories(integrationId: string): Promise { + const qx = SequelizeRepository.getQueryExecutor(this.options) + return getIntegrationReposMapping(qx, integrationId) + } + /** * Adds discord integration to a tenant * @param guildId Guild id of the discord server @@ -1460,6 +1491,40 @@ export default class IntegrationService { ...(options || this.options), transaction, }) + + // Soft-delete repos from git.repositories that are no longer in the remotes list + // Only delete repos owned by this Git integration (not mirrored from other integrations) + const newRemoteUrls = new Set(remotes.map((r) => r.url)) + const existingOwnedRepos: Array<{ url: string }> = await qx.select( + ` + SELECT gr.url + FROM git.repositories gr + JOIN public.repositories pr ON pr.url = gr.url AND pr."deletedAt" IS NULL + WHERE gr."integrationId" = $(integrationId) + AND gr."deletedAt" IS NULL + AND pr."sourceIntegrationId" = $(integrationId) + `, + { integrationId: integration.id }, + ) + const urlsToDelete = existingOwnedRepos + .map((r) => r.url) + .filter((url) => !newRemoteUrls.has(url)) + + if (urlsToDelete.length > 0) { + await qx.result( + ` + UPDATE git.repositories + SET "deletedAt" = NOW() + WHERE url IN ($(urlsToDelete:csv)) + AND "deletedAt" IS NULL + `, + { urlsToDelete }, + ) + this.options.log.info( + `Soft-deleted ${urlsToDelete.length} owned repos from git.repositories`, + ) + } + await syncRepositoriesToGitV2(qx, remotes, integration.id, currentSegmentId) // sync to public.repositories (only for direct GIT connections, other platforms handle it themselves) @@ -3166,6 +3231,25 @@ export default class IntegrationService { } } + /** + * Identifies mirrored repo URLs for a Git integration. + * Mirrored repos are those linked to this Git integration but owned by another source integration. + */ + private getMirroredRepoUrls( + repos: IRepository[], + gitIntegrationId: string, + ): Set { + return new Set( + repos + .filter( + (repo) => + repo.gitIntegrationId === gitIntegrationId && + repo.sourceIntegrationId !== gitIntegrationId, + ) + .map((repo) => repo.url), + ) + } + private async validateGitIntegrationDeletion( gitIntegrationId: string, options: IRepositoryOptions, @@ -3299,6 +3383,7 @@ export default class IntegrationService { sourcePlatform: PlatformType, sourceIntegrationId: string, mapping: { [url: string]: string }, + skipMirroredRepos = true, ) { // Check for existing transaction to support nested calls within outer transactions const existingTransaction = SequelizeRepository.getTransaction(this.options) @@ -3320,17 +3405,38 @@ export default class IntegrationService { getRepositoriesBySourceIntegrationId(qx, sourceIntegrationId), ]) - // Block repos that belong to a different integration - this.validateRepoIntegrationMapping(existingMappedRepos, sourceIntegrationId) + // For Git integration updates, identify mirrored repos (owned by other integrations) + // These should be skipped from all operations unless destroying the integration + const isGitIntegration = sourcePlatform === PlatformType.GIT + const mirroredRepoUrls = + isGitIntegration && skipMirroredRepos + ? this.getMirroredRepoUrls(existingMappedRepos, sourceIntegrationId) + : new Set() + + // Filter out mirrored repos for validation and processing + const reposToValidate = existingMappedRepos.filter( + (repo) => !mirroredRepoUrls.has(repo.url), + ) + + // Block repos that belong to a different integration (skip mirrored for Git) + this.validateRepoIntegrationMapping(reposToValidate, sourceIntegrationId) - const existingUrlSet = new Set(existingMappedRepos.map((repo) => repo.url)) - const toInsertUrls = mappedUrls.filter((url) => !existingUrlSet.has(url)) + // Filter out mirrored URLs from processing + const ownedMappedUrls = mappedUrls.filter((url) => !mirroredRepoUrls.has(url)) + const existingUrlSet = new Set(reposToValidate.map((repo) => repo.url)) + const toInsertUrls = ownedMappedUrls.filter((url) => !existingUrlSet.has(url)) // Repos to restore: soft-deleted OR segment changed (both need re-onboarding) - const toRestoreRepos = existingMappedRepos.filter( + const toRestoreRepos = reposToValidate.filter( (repo) => repo.deletedAt !== null || repo.segmentId !== mapping[repo.url], ) const toSoftDeleteRepos = activeIntegrationRepos.filter((repo) => !mappedUrlSet.has(repo.url)) + if (mirroredRepoUrls.size > 0) { + this.options.log.info( + `Skipping ${mirroredRepoUrls.size} mirrored repos from Git integration update`, + ) + } + this.options.log.info( `Repository mapping: ${toInsertUrls.length} to insert, ${toRestoreRepos.length} to restore, ${toSoftDeleteRepos.length} to soft-delete`, ) diff --git a/services/libs/data-access-layer/src/integrations/index.ts b/services/libs/data-access-layer/src/integrations/index.ts index 027b5bba45..6e4bf332e5 100644 --- a/services/libs/data-access-layer/src/integrations/index.ts +++ b/services/libs/data-access-layer/src/integrations/index.ts @@ -550,7 +550,9 @@ export async function syncRepositoriesToGitV2( return } - // Check GitHub repos first, fallback to GitLab repos if none found + const urls = remotes.map((r) => r.url) + + // Check GitHub repos, GitLab repos, AND git.repositories for existing IDs const existingRepos: Array<{ id: string url: string @@ -563,49 +565,50 @@ export async function syncRepositoriesToGitV2( gitlab_repos AS ( SELECT id, url FROM "gitlabRepos" WHERE url IN ($(urls:csv)) AND "deletedAt" IS NULL + ), + git_repos AS ( + SELECT id, url FROM git.repositories + WHERE url IN ($(urls:csv)) AND "deletedAt" IS NULL ) - SELECT id, url FROM github_repos - UNION ALL - SELECT id, url FROM gitlab_repos - WHERE NOT EXISTS (SELECT 1 FROM github_repos) + SELECT DISTINCT ON (url) id, url FROM ( + SELECT id, url FROM github_repos + UNION ALL + SELECT id, url FROM gitlab_repos + UNION ALL + SELECT id, url FROM git_repos + ) combined `, - { - urls: remotes.map((r) => r.url), - }, + { urls }, ) // Create a map of url to forkedFrom for quick lookup const forkedFromMap = new Map(remotes.map((r) => [r.url, r.forkedFrom])) - let repositoriesToSync: Array<{ + // Create a map of existing url to id + const existingUrlToId = new Map(existingRepos.map((r) => [r.url, r.id])) + + // Build repositoriesToSync, using existing IDs where available + const repositoriesToSync: Array<{ id: string url: string integrationId: string segmentId: string forkedFrom?: string | null - }> = [] - - // Map existing repos with their IDs - if (existingRepos.length > 0) { - repositoriesToSync = existingRepos.map((repo) => ({ - id: repo.id, - url: repo.url, - integrationId: gitIntegrationId, - segmentId, - forkedFrom: forkedFromMap.get(repo.url) || null, - })) - } else { - // If no existing repos found, create new ones with generated UUIDs - log.warn( - 'No existing repos found in githubRepos or gitlabRepos - inserting new to git.repositories with new UUIDs', - ) - repositoriesToSync = remotes.map((remote) => ({ - id: generateUUIDv4(), + }> = remotes.map((remote) => { + const existingId = existingUrlToId.get(remote.url) + return { + id: existingId || generateUUIDv4(), url: remote.url, integrationId: gitIntegrationId, segmentId, forkedFrom: remote.forkedFrom || null, - })) + } + }) + + if (existingRepos.length === 0) { + log.warn( + 'No existing repos found in githubRepos, gitlabRepos, or git.repositories - inserting new with generated UUIDs', + ) } // Build SQL placeholders and parameters diff --git a/services/libs/data-access-layer/src/repositories/index.ts b/services/libs/data-access-layer/src/repositories/index.ts index 3607884eea..dbdcd5108d 100644 --- a/services/libs/data-access-layer/src/repositories/index.ts +++ b/services/libs/data-access-layer/src/repositories/index.ts @@ -327,3 +327,48 @@ export async function restoreRepositories( { urls }, ) } + +/** + * Repository mapping result with segment info + */ +export interface IRepositoryMapping { + url: string + segment: { + id: string + name: string + } + gitIntegrationId: string + sourceIntegrationId: string +} + +/** + * Get repository mappings for a specific integration + * Replaces get{Github/Gitlab}Repos with a unified approach for all code platforms + * Matches repos where gitIntegrationId OR sourceIntegrationId equals the given integrationId + * @param qx - Query executor + * @param integrationId - The integration ID (git or source platform) to filter by + * @returns Array of repositories with segment info and integration IDs + */ +export async function getIntegrationReposMapping( + qx: QueryExecutor, + integrationId: string, +): Promise { + return qx.select( + ` + SELECT + r.url, + jsonb_build_object( + 'id', s.id, + 'name', s.name + ) as segment, + r."gitIntegrationId", + r."sourceIntegrationId" + FROM public.repositories r + JOIN segments s ON s.id = r."segmentId" + WHERE (r."gitIntegrationId" = $(integrationId) OR r."sourceIntegrationId" = $(integrationId)) + AND r."deletedAt" IS NULL + ORDER BY r.url + `, + { integrationId }, + ) +} From 782098cc8c42ed00a940f573847d5006bf2a04d9 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Mon, 5 Jan 2026 17:52:39 +0100 Subject: [PATCH 14/17] feat: ui changes to use unified approach --- .../git/components/git-params.vue | 28 ++++++- .../git/components/git-settings-drawer.vue | 81 +++++++++++++++---- .../integration/integration-service.js | 27 ++++--- frontend/src/shared/form/array-input.vue | 6 ++ 4 files changed, 110 insertions(+), 32 deletions(-) diff --git a/frontend/src/config/integrations/git/components/git-params.vue b/frontend/src/config/integrations/git/components/git-params.vue index 91a0635fdd..0f06207ed3 100644 --- a/frontend/src/config/integrations/git/components/git-params.vue +++ b/frontend/src/config/integrations/git/components/git-params.vue @@ -39,8 +39,9 @@