Merge branch 'main' into script/CM-774

Author: skwowet

backend/src/database/migrations/U1769438813__addEnabledToRepositories.sql

@@ -0,0 +1 @@
+ALTER TABLE public.repositories DROP COLUMN enabled;

backend/src/database/migrations/V1769438813__addEnabledToRepositories.sql

@@ -0,0 +1,2 @@
+ALTER TABLE public.repositories ADD COLUMN enabled BOOLEAN NOT NULL DEFAULT TRUE;
+COMMENT ON COLUMN public.repositories.enabled IS 'Used to enable/disable repository on insights';

backend/src/services/collectionService.ts

@@ -384,16 +384,6 @@ export class CollectionService extends LoggerBase {
     }
   }
 
-  static normalizeRepositories(
-    repositories?: string[] | { platform: string; url: string }[],
-  ): string[] {
-    if (!repositories || repositories.length === 0) return []
-
-    return typeof repositories[0] === 'string'
-      ? (repositories as string[])
-      : (repositories as { platform: string; url: string }[]).map((r) => r.url)
-  }
-
   async updateInsightsProject(insightsProjectId: string, project: Partial<ICreateInsightsProject>) {
     return SequelizeRepository.withTx(this.options, async (tx) => {
       const qx = SequelizeRepository.getQueryExecutor({ ...this.options, transaction: tx })

services/libs/data-access-layer/src/collections/index.ts

@@ -1,5 +1,6 @@
 import { QueryFilter } from '../query'
 import { QueryExecutor } from '../queryExecutor'
+import { syncRepositoriesEnabledStatus } from '../repositories'
 import { ICreateRepositoryGroup } from '../repositoryGroups'
 import {
   QueryResult,
@@ -310,9 +311,28 @@ export async function updateInsightsProject(
     throw new Error(`Update failed or project with id ${id} not found`)
   }
 
+  // Sync repositories.enabled status when repositories field is updated
+  // Disables repos not in the new list (new repos are enabled by default on insert)
+  if (project.repositories !== undefined) {
+    const enabledUrls = normalizeRepositoriesToUrls(project.repositories)
+    await syncRepositoriesEnabledStatus(qx, id, enabledUrls)
+  }
+
   return updated as IInsightsProject
 }
 
+function normalizeRepositoriesToUrls(
+  repositories: string[] | { platform: string; url: string }[] | undefined,
+): string[] {
+  if (!repositories || repositories.length === 0) return []
+
+  if (typeof repositories[0] === 'string') {
+    return repositories as string[]
+  }
+
+  return (repositories as { platform: string; url: string }[]).map((r) => r.url)
+}
+
 function prepareProject(project: Partial<ICreateInsightsProject>) {
   const toUpdate: Record<string, unknown> = {
     ...project,

services/libs/data-access-layer/src/integrations/index.ts

@@ -2,10 +2,62 @@ import { getServiceChildLogger } from '@crowd/logging'
 import { IIntegration, PlatformType } from '@crowd/types'
 
 import { QueryExecutor } from '../queryExecutor'
-import { getMappedRepos } from '../segments'
+import { getReposBySegmentGroupedByPlatform } from '../segments'
 
 const log = getServiceChildLogger('db.integrations')
 
+export function normalizeRepoUrl(url: string): string {
+  try {
+    const parsed = new URL(url)
+
+    // Normalize protocol to https
+    parsed.protocol = 'https:'
+
+    // Remove www. prefix and lowercase hostname
+    parsed.hostname = parsed.hostname.replace(/^www\./, '').toLowerCase()
+
+    // Lowercase path for GitHub/GitLab (case-insensitive platforms)
+    if (parsed.hostname === 'github.com' || parsed.hostname === 'gitlab.com') {
+      parsed.pathname = parsed.pathname.toLowerCase()
+    }
+
+    // Remove trailing slashes and .git suffix
+    parsed.pathname = parsed.pathname.replace(/\/+$/, '').replace(/\.git$/, '')
+
+    // Remove query string and hash
+    parsed.search = ''
+    parsed.hash = ''
+
+    return parsed.toString()
+  } catch {
+    return url
+  }
+}
+
+/**
+ * Extracts a human-readable label from a repository URL.
+ * For GitHub/GitLab: returns "owner/repo"
+ * For others: returns the full URL
+ */
+export function extractLabelFromUrl(url: string): string {
+  try {
+    const parsed = new URL(url)
+    if (
+      parsed.hostname === 'github.com' ||
+      parsed.hostname === 'gitlab.com' ||
+      parsed.hostname.endsWith('.gitlab.com')
+    ) {
+      return parsed.pathname
+        .slice(1)
+        .replace(/\.git$/, '')
+        .replace(/\/+$/, '')
+    }
+    return url
+  } catch {
+    return url
+  }
+}
+
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
 /**
@@ -609,82 +661,17 @@ export async function findRepositoriesForSegment(
   qx: QueryExecutor,
   segmentId: string,
 ): Promise<Record<string, Array<{ url: string; label: string }>>> {
-  const integrations = await fetchIntegrationsForSegment(qx, segmentId)
-
-  // Initialize result with platform arrays
-  const result: Record<string, Array<{ url: string; label: string }>> = {
-    git: [],
-    github: [],
-    gitlab: [],
-    gerrit: [],
-  }
-
-  const addToResult = (platform: PlatformType, fullUrl: string, label: string) => {
-    const platformKey = platform.toLowerCase()
-    if (!result[platformKey].some((item) => item.url === fullUrl)) {
-      result[platformKey].push({ url: fullUrl, label })
-    }
-  }
-
-  // Add mapped repositories from public.repositories (GitHub and GitLab platforms)
-  const [githubMappedRepos, githubNangoMappedRepos, gitlabMappedRepos] = await Promise.all([
-    getMappedRepos(qx, segmentId, PlatformType.GITHUB),
-    getMappedRepos(qx, segmentId, PlatformType.GITHUB_NANGO),
-    getMappedRepos(qx, segmentId, PlatformType.GITLAB),
-  ])
-
-  for (const repo of [...githubMappedRepos, ...githubNangoMappedRepos, ...gitlabMappedRepos]) {
-    const url = repo.url
-    try {
-      const parsedUrl = new URL(url)
-      if (parsedUrl.hostname === 'github.com') {
-        const label = parsedUrl.pathname.slice(1) // removes leading '/'
-        addToResult(PlatformType.GITHUB, url, label)
-      }
-      if (parsedUrl.hostname === 'gitlab.com') {
-        const label = parsedUrl.pathname.slice(1) // removes leading '/'
-        addToResult(PlatformType.GITLAB, url, label)
-      }
-    } catch (err) {
-      log.error({ err, repo }, 'Error parsing URL for repository!')
-    }
-  }
-
-  for (const i of integrations) {
-    if (i.platform === PlatformType.GIT) {
-      for (const r of (i.settings as any).remotes) {
-        try {
-          const url = new URL(r)
-          let label = r
-
-          if (url.hostname === 'gitlab.com') {
-            label = url.pathname.slice(1)
-          } else if (url.hostname === 'github.com') {
-            label = url.pathname.slice(1)
-          }
-
-          addToResult(i.platform, r, label)
-        } catch {
-          // Invalid URL, skip
-        }
-      }
-    }
+  // Get all repos grouped by platform (github-nango merged into github)
+  const reposByPlatform = await getReposBySegmentGroupedByPlatform(qx, segmentId, true)
 
-    if (i.platform === PlatformType.GITLAB) {
-      for (const group of Object.values((i.settings as any).groupProjects) as any[]) {
-        for (const r of group) {
-          const label = r.path_with_namespace
-          const fullUrl = `https://gitlab.com/${label}`
-          addToResult(i.platform, fullUrl, label)
-        }
-      }
-    }
+  // Transform to include normalized URLs and labels
+  const result: Record<string, Array<{ url: string; label: string }>> = {}
 
-    if (i.platform === PlatformType.GERRIT) {
-      for (const r of (i.settings as any).remote.repoNames) {
-        addToResult(i.platform, `${(i.settings as any).remote.orgURL}/q/project:${r}`, r)
-      }
-    }
+  for (const [platform, urls] of Object.entries(reposByPlatform)) {
+    result[platform] = urls.map((url) => ({
+      url: normalizeRepoUrl(url),
+      label: extractLabelFromUrl(url),
+    }))
   }
 
   return result

services/libs/data-access-layer/src/repositories/index.ts

@@ -16,6 +16,7 @@ export interface IRepository {
   archived: boolean
   forkedFrom: string | null
   excluded: boolean
+  enabled: boolean
   createdAt: string
   updatedAt: string
   deletedAt: string | null
@@ -268,6 +269,7 @@ export async function restoreRepositories(
       archived = COALESCE(v.archived::boolean, r.archived),
       "forkedFrom" = COALESCE(v."forkedFrom", r."forkedFrom"),
       excluded = COALESCE(v.excluded::boolean, r.excluded),
+      enabled = true,
       "deletedAt" = NULL,
       "updatedAt" = NOW()
     FROM jsonb_to_recordset($(values)::jsonb) AS v(
@@ -551,3 +553,27 @@ export async function findSegmentsForRepos(
 
   return results
 }
+
+/**
+ * Syncs repositories.enabled to match insightsProject.repositories list.
+ */
+export async function syncRepositoriesEnabledStatus(
+  qx: QueryExecutor,
+  insightsProjectId: string,
+  enabledUrls: string[],
+): Promise<void> {
+  const normalizedUrls = enabledUrls.map((url) => url.toLowerCase())
+
+  await qx.result(
+    `
+    UPDATE public.repositories
+    SET 
+      enabled = LOWER(url) = ANY($(normalizedUrls)::text[]),
+      "updatedAt" = NOW()
+    WHERE "insightsProjectId" = $(insightsProjectId)
+      AND "deletedAt" IS NULL
+      AND enabled <> (LOWER(url) = ANY($(normalizedUrls)::text[]))
+    `,
+    { insightsProjectId, normalizedUrls },
+  )
+}

services/libs/data-access-layer/src/segments/index.ts

@@ -209,6 +209,60 @@ export async function getMappedRepos(
   )
 }
 
+export interface IRepoByPlatform {
+  url: string
+  platform: string
+}
+
+/**
+ * Get all repositories for a segment, grouped by platform.
+ * Joins with the integrations table to determine the platform for each repo.
+ *
+ * @param qx - Query executor
+ * @param segmentId - The segment ID to get repos for
+ * @param mergeGithubNango - If true, merges 'github-nango' platform into 'github' (default: true)
+ * @returns Record of platform -> array of repo URLs
+ */
+export async function getReposBySegmentGroupedByPlatform(
+  qx: QueryExecutor,
+  segmentId: string,
+  mergeGithubNango = true,
+): Promise<Record<string, string[]>> {
+  const rows: IRepoByPlatform[] = await qx.select(
+    `
+      SELECT DISTINCT
+        r.url,
+        i.platform
+      FROM public.repositories r
+      JOIN integrations i ON r."sourceIntegrationId" = i.id
+      WHERE r."segmentId" = $(segmentId)
+        AND r."deletedAt" IS NULL
+        AND i."deletedAt" IS NULL
+      ORDER BY i.platform, r.url
+    `,
+    { segmentId },
+  )
+
+  const result: Record<string, string[]> = {}
+
+  for (const row of rows) {
+    let platform = row.platform
+
+    // Merge github-nango into github if requested
+    if (mergeGithubNango && platform === PlatformType.GITHUB_NANGO) {
+      platform = PlatformType.GITHUB
+    }
+
+    if (!result[platform]) {
+      result[platform] = []
+    }
+
+    result[platform].push(row.url)
+  }
+
+  return result
+}
+
 export async function getRepoUrlsMappedToOtherSegments(
   qx: QueryExecutor,
   urls: string[],

services/libs/tinybird/datasources/repositories.datasource

@@ -10,6 +10,7 @@ DESCRIPTION >
     - `archived` indicates whether the repository has been archived.
     - `forkedFrom` contains the URL of the parent repository if this is a fork (empty string if not a fork).
     - `excluded` indicates whether the repository is excluded from processing.
+    - `enabled` indicates whether the repository is enabled on insights.
     - `createdAt` and `updatedAt` are standard timestamp fields for record lifecycle tracking.
     - `deletedAt` is set when the repository is soft-deleted.
     - `lastArchivedCheckAt` is the timestamp of the last check for repository archived status.
@@ -26,6 +27,7 @@ SCHEMA >
     `archived` Bool `json:$.record.archived` DEFAULT false,
     `forkedFrom` String `json:$.record.forkedFrom` DEFAULT '',
     `excluded` Bool `json:$.record.excluded` DEFAULT false,
+    `enabled` Bool `json:$.record.enabled` DEFAULT true,
     `createdAt` DateTime64(3) `json:$.record.createdAt`,
     `updatedAt` DateTime64(3) `json:$.record.updatedAt`,
     `deletedAt` Nullable(DateTime64(3)) `json:$.record.deletedAt`,