feat: improve prevent duplication [CM-560] (#3317)

Author: ulemons

backend/package.json

@@ -140,9 +140,9 @@
   },
   "private": true,
   "devDependencies": {
-    "@babel/core": "^7.21.8",
-    "@babel/preset-env": "^7.21.5",
-    "@babel/preset-typescript": "^7.21.5",
+    "@babel/core": "^7.24.4",
+    "@babel/preset-env": "^7.24.4",
+    "@babel/preset-typescript": "^7.24.1",
     "@trivago/prettier-plugin-sort-imports": "^4.3.0",
     "@types/bunyan": "^1.8.8",
     "@types/bunyan-format": "^0.2.5",

backend/src/database/migrations/U1755867534__add_unique_constraint_on_repository.sql

@@ -0,0 +1 @@
+CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS segmentRepositories_repository_uq_idx ON "segmentRepositories" ("repository");

backend/src/database/migrations/V1755867534__add_unique_constraint_on_repository.sql

@@ -976,6 +976,66 @@ class SegmentRepository extends RepositoryBase<
 
     return result
   }
+
+  async getGithubRepoUrlsMappedToOtherSegments(urls: string[], segmentId: string) {
+    if (!urls || urls.length === 0) {
+      return []
+    }
+
+    const transaction = SequelizeRepository.getTransaction(this.options)
+    const tenantId = this.options.currentTenant.id
+
+    const rows = await this.options.database.sequelize.query(
+      `
+      select distinct
+        r."url" as "url"
+      from
+        "githubRepos" r
+      where
+        r."tenantId"  = :tenantId
+        and r."url"   in (:urls)
+        and r."deletedAt" is null
+        and r."segmentId" <> :segmentId
+      `,
+      {
+        replacements: { tenantId, urls, segmentId },
+        type: QueryTypes.SELECT,
+        transaction,
+      },
+    )
+
+    return rows.map((r) => r.url)
+  }
+
+  async getGitlabRepoUrlsMappedToOtherSegments(urls: string[], segmentId: string) {
+    if (!urls || urls.length === 0) {
+      return []
+    }
+
+    const transaction = SequelizeRepository.getTransaction(this.options)
+    const tenantId = this.options.currentTenant.id
+
+    const rows = await this.options.database.sequelize.query(
+      `
+      select distinct
+        r."url" as "url"
+      from
+        "gitlabRepos" r
+      where
+        r."tenantId"  = :tenantId
+        and r."url"   in (:urls)
+        and r."deletedAt" is null
+        and r."segmentId" <> :segmentId
+      `,
+      {
+        replacements: { tenantId, urls, segmentId },
+        type: QueryTypes.SELECT,
+        transaction,
+      },
+    )
+
+    return rows.map((r) => r.url)
+  }
 }
 
 export default SegmentRepository

backend/src/database/repositories/segmentRepository.ts

@@ -23,7 +23,11 @@ import {
   updateCollection,
   updateInsightsProject,
 } from '@crowd/data-access-layer/src/collections'
-import { fetchIntegrationsForSegment } from '@crowd/data-access-layer/src/integrations'
+import {
+  fetchIntegrationById,
+  fetchIntegrationsForSegment,
+  removePlainGitHubRepoMapping,
+} from '@crowd/data-access-layer/src/integrations'
 import { OrganizationField, findOrgById, queryOrgs } from '@crowd/data-access-layer/src/orgs'
 import { QueryFilter } from '@crowd/data-access-layer/src/query'
 import { findSegmentById } from '@crowd/data-access-layer/src/segments'
@@ -426,7 +430,7 @@ export class CollectionService extends LoggerBase {
       }
 
       // Add mapped repositories to GitHub platform
-      const segmentRepository = new SegmentRepository(this.options)
+      const segmentRepository = new SegmentRepository({ ...this.options, transaction: tx })
       const githubMappedRepos = await segmentRepository.getGithubMappedRepos(segmentId)
       const gitlabMappedRepos = await segmentRepository.getGitlabMappedRepos(segmentId)
 
@@ -634,4 +638,74 @@ export class CollectionService extends LoggerBase {
 
     return result
   }
+
+  static extractGithubRepoSlug(url: string): any {
+    const parsedUrl = new URL(url)
+    const pathname = parsedUrl.pathname
+    const parts = pathname.split('/').filter(Boolean)
+
+    if (parts.length >= 2) {
+      return `${parts[0]}/${parts[1]}`
+    }
+
+    throw new Error('Invalid GitHub URL format')
+  }
+
+  async findNangoRepositoriesToBeRemoved(integrationId: string): Promise<string[]> {
+    return SequelizeRepository.withTx(this.options, async (tx) => {
+      const qx = SequelizeRepository.getQueryExecutor({ ...this.options, transaction: tx })
+      const integration = await fetchIntegrationById(qx, integrationId)
+
+      if (!integration || integration.platform !== PlatformType.GITHUB_NANGO) {
+        return []
+      }
+
+      const repoSlugs = new Set<string>()
+      const settings = integration.settings as any
+      const reposToBeRemoved = []
+
+      if (!settings.nangoMapping) {
+        return []
+      }
+
+      if (settings.orgs) {
+        for (const org of settings.orgs) {
+          for (const repo of org.repos ?? []) {
+            repoSlugs.add(CollectionService.extractGithubRepoSlug(repo.url))
+          }
+        }
+      }
+
+      if (settings.repos) {
+        for (const repo of settings.repos) {
+          repoSlugs.add(CollectionService.extractGithubRepoSlug(repo.url))
+        }
+      }
+      // determine which connections to delete if needed
+      for (const mappedRepo of Object.values(settings.nangoMapping) as {
+        owner: string
+        repoName: string
+      }[]) {
+        if (!repoSlugs.has(`${mappedRepo.owner}/${mappedRepo.repoName}`)) {
+          reposToBeRemoved.push(`https://github.com/${mappedRepo.owner}/${mappedRepo.repoName}`)
+        }
+      }
+
+      return reposToBeRemoved
+    })
+  }
+
+  async unmapGithubRepo(integrationId: string, repo: string): Promise<void> {
+    return SequelizeRepository.withTx(this.options, async (tx) => {
+      const qx = SequelizeRepository.getQueryExecutor({ ...this.options, transaction: tx })
+      await removePlainGitHubRepoMapping(qx, this.options.redis, integrationId, repo)
+    })
+  }
+
+  async unmapGitlabRepo(integrationId: string, repo: string): Promise<void> {
+    return SequelizeRepository.withTx(this.options, async (tx) => {
+      const qx = SequelizeRepository.getQueryExecutor({ ...this.options, transaction: tx })
+      await removePlainGitHubRepoMapping(qx, this.options.redis, integrationId, repo)
+    })
+  }
 }