chore: refactor integration settings nango mappings

Signed-off-by: Uroš Marolt <uros@marolt.me>

Author: themarolt

backend/src/api/integration/index.ts

@@ -21,7 +21,6 @@ export default (app) => {
   app.post(`/integration/query`, safeWrap(require('./integrationQuery').default))
   app.post(`/integration`, safeWrap(require('./integrationCreate').default))
   app.put(`/integration/:id`, safeWrap(require('./integrationUpdate').default))
-  app.post(`/integration/import`, safeWrap(require('./integrationImport').default))
   app.delete(`/integration`, safeWrap(require('./integrationDestroy').default))
   app.get(`/integration/autocomplete`, safeWrap(require('./integrationAutocomplete').default))
   app.get(`/integration/global`, safeWrap(require('./integrationGlobal').default))

backend/src/api/integration/integrationCreate.ts

@@ -5,8 +5,6 @@ import PermissionChecker from '../../services/user/permissionChecker'
 export default async (req, res) => {
   new PermissionChecker(req).validateHas(Permissions.values.integrationCreate)
 
-  new PermissionChecker(req).validateIntegrationsProtectedFields(req.body)
-
   const payload = await new IntegrationService(req).create(req.body)
 
   await req.responseHandler.success(req, res, payload)

backend/src/api/integration/integrationImport.ts

@@ -4,7 +4,6 @@ import PermissionChecker from '../../services/user/permissionChecker'
 
 export default async (req, res) => {
   new PermissionChecker(req).validateHas(Permissions.values.integrationEdit)
-  new PermissionChecker(req).validateIntegrationsProtectedFields(req.body)
 
   const payload = await new IntegrationService(req).update(req.params.id, req.body)
 

backend/src/api/integration/integrationUpdate.ts

@@ -0,0 +1,54 @@
+-- Part A: Drop deprecated columns from integrations table
+ALTER TABLE public.integrations
+  DROP COLUMN IF EXISTS "limitCount",
+  DROP COLUMN IF EXISTS "limitLastResetAt",
+  DROP COLUMN IF EXISTS "emailSentAt",
+  DROP COLUMN IF EXISTS "importHash";
+
+-- Drop same columns from integrationsBackup table (created as SELECT * FROM integrations WHERE 1=2)
+ALTER TABLE public."integrationsBackup"
+  DROP COLUMN IF EXISTS "limitCount",
+  DROP COLUMN IF EXISTS "limitLastResetAt",
+  DROP COLUMN IF EXISTS "emailSentAt",
+  DROP COLUMN IF EXISTS "importHash";
+
+-- Part B: Create nango_mapping table and migrate data from settings JSONB
+CREATE TABLE integration.nango_mapping (
+    "integrationId"  UUID   NOT NULL,
+    "connectionId"   TEXT   NOT NULL,
+    "repositoryId"   UUID,
+    owner            TEXT   NOT NULL,
+    "repoName"       TEXT   NOT NULL,
+    "createdAt"      TIMESTAMPTZ NOT NULL DEFAULT now(),
+    "updatedAt"      TIMESTAMPTZ NOT NULL DEFAULT now(),
+    PRIMARY KEY ("integrationId", "connectionId"),
+    FOREIGN KEY ("integrationId") REFERENCES integrations(id) ON DELETE CASCADE,
+    FOREIGN KEY ("repositoryId") REFERENCES repositories(id) ON DELETE SET NULL
+);
+
+CREATE INDEX ix_nango_mapping_connectionId ON integration.nango_mapping ("connectionId");
+CREATE INDEX ix_nango_mapping_repositoryId ON integration.nango_mapping ("repositoryId");
+
+-- Migrate existing data from settings.nangoMapping
+INSERT INTO integration.nango_mapping ("integrationId", "connectionId", "repositoryId", owner, "repoName")
+SELECT
+    i.id,
+    nm.key,
+    r.id,
+    nm.value->>'owner',
+    nm.value->>'repoName'
+FROM integrations i
+CROSS JOIN LATERAL jsonb_each(i.settings->'nangoMapping') nm
+LEFT JOIN repositories r
+    ON lower(r.url) = lower('https://github.com/' || (nm.value->>'owner') || '/' || (nm.value->>'repoName'))
+    AND r."sourceIntegrationId" = i.id
+    AND r."deletedAt" IS NULL
+WHERE i.platform = 'github-nango'
+  AND i."deletedAt" IS NULL
+  AND i.settings->'nangoMapping' IS NOT NULL
+ON CONFLICT DO NOTHING;
+
+-- Remove nangoMapping from settings
+UPDATE integrations
+SET settings = settings - 'nangoMapping'
+WHERE settings->'nangoMapping' IS NOT NULL;

backend/src/database/migrations/U1771839722__refactorIntegrationsTable.sql

@@ -15,12 +15,6 @@ export default (sequelize) => {
       status: {
         type: DataTypes.TEXT,
       },
-      limitCount: {
-        type: DataTypes.INTEGER,
-      },
-      limitLastResetAt: {
-        type: DataTypes.DATE,
-      },
       token: {
         type: DataTypes.TEXT,
       },
@@ -34,26 +28,9 @@ export default (sequelize) => {
       integrationIdentifier: {
         type: DataTypes.TEXT,
       },
-      importHash: {
-        type: DataTypes.STRING(255),
-        allowNull: true,
-        validate: {
-          len: [0, 255],
-        },
-      },
-      emailSentAt: {
-        type: DataTypes.DATE,
-      },
     },
     {
       indexes: [
-        {
-          unique: true,
-          fields: ['importHash', 'tenantId'],
-          where: {
-            deletedAt: null,
-          },
-        },
         {
           unique: false,
           fields: ['integrationIdentifier'],

backend/src/database/migrations/V1771839722__refactorIntegrationsTable.sql

@@ -35,14 +35,10 @@ class IntegrationRepository {
       ...lodash.pick(data, [
         'platform',
         'status',
-        'limitCount',
-        'limitLastResetAt',
         'token',
         'refreshToken',
         'settings',
         'integrationIdentifier',
-        'importHash',
-        'emailSentAt',
       ]),
       segmentId: segment.id,
       tenantId: DEFAULT_TENANT_ID,
@@ -88,14 +84,10 @@ class IntegrationRepository {
         ...lodash.pick(data, [
           'platform',
           'status',
-          'limitCount',
-          'limitLastResetAt',
           'token',
           'refreshToken',
           'settings',
           'integrationIdentifier',
-          'importHash',
-          'emailSentAt',
         ]),
 
         updatedById: currentUser.id,
@@ -431,46 +423,6 @@ class IntegrationRepository {
         })
       }
 
-      if (filter.limitCountRange) {
-        const [start, end] = filter.limitCountRange
-
-        if (start !== undefined && start !== null && start !== '') {
-          advancedFilter.and.push({
-            limitCount: {
-              gte: start,
-            },
-          })
-        }
-
-        if (end !== undefined && end !== null && end !== '') {
-          advancedFilter.and.push({
-            limitCount: {
-              lte: end,
-            },
-          })
-        }
-      }
-
-      if (filter.limitLastResetAtRange) {
-        const [start, end] = filter.limitLastResetAtRange
-
-        if (start !== undefined && start !== null && start !== '') {
-          advancedFilter.and.push({
-            limitLastResetAt: {
-              gte: start,
-            },
-          })
-        }
-
-        if (end !== undefined && end !== null && end !== '') {
-          advancedFilter.and.push({
-            limitLastResetAt: {
-              lte: end,
-            },
-          })
-        }
-      }
-
       if (filter.integrationIdentifier) {
         advancedFilter.and.push({
           integrationIdentifier: filter.integrationIdentifier,
@@ -663,6 +615,26 @@ class IntegrationRepository {
 
     const output = record.get({ plain: true })
 
+    // For github-nango integrations, populate settings.nangoMapping from the dedicated table
+    // so the API contract remains unchanged for frontend consumers
+    if (output.platform === PlatformType.GITHUB_NANGO) {
+      const rows = await record.sequelize.query(
+        `SELECT "connectionId", owner, "repoName" FROM integration.nango_mapping WHERE "integrationId" = :integrationId`,
+        {
+          replacements: { integrationId: output.id },
+          type: QueryTypes.SELECT,
+        },
+      )
+
+      if (rows.length > 0) {
+        const nangoMapping: Record<string, { owner: string; repoName: string }> = {}
+        for (const row of rows as { connectionId: string; owner: string; repoName: string }[]) {
+          nangoMapping[row.connectionId] = { owner: row.owner, repoName: row.repoName }
+        }
+        output.settings = { ...output.settings, nangoMapping }
+      }
+    }
+
     return output
   }
 }

backend/src/database/models/integration.ts

@@ -165,14 +165,6 @@ class Permissions {
         id: 'organizationAutocomplete',
         allowedRoles: [roles.admin, roles.projectAdmin, roles.readonly],
       },
-      integrationImport: {
-        id: 'integrationImport',
-        allowedRoles: [roles.admin, roles.projectAdmin],
-      },
-      integrationControlLimit: {
-        id: 'integrationControlLimit',
-        allowedRoles: [],
-      },
       integrationCreate: {
         id: 'integrationCreate',
         allowedRoles: [roles.admin, roles.projectAdmin],

backend/src/database/repositories/integrationRepository.ts

@@ -556,46 +556,6 @@ export default class IntegrationService {
     return result
   }
 
-  async import(data, importHash) {
-    const transaction = await SequelizeRepository.createTransaction(this.options)
-
-    try {
-      if (!importHash) {
-        throw new Error400(this.options.language, 'importer.errors.importHashRequired')
-      }
-
-      if (await this._isImportHashExistent(importHash)) {
-        throw new Error400(this.options.language, 'importer.errors.importHashExistent')
-      }
-
-      const dataToCreate = {
-        ...data,
-        importHash,
-      }
-
-      const result = this.create(dataToCreate, transaction)
-
-      await SequelizeRepository.commitTransaction(transaction)
-
-      return await result
-    } catch (err) {
-      await SequelizeRepository.rollbackTransaction(transaction)
-
-      throw err
-    }
-  }
-
-  async _isImportHashExistent(importHash) {
-    const count = await IntegrationRepository.count(
-      {
-        importHash,
-      },
-      this.options,
-    )
-
-    return count > 0
-  }
-
   /**
    * Returns installation access token for a Github App installation
    * @param installId Install id of the Github app
@@ -867,14 +827,7 @@ export default class IntegrationService {
           {
             id: integrationId,
             platform: PlatformType.GITHUB_NANGO,
-            settings: {
-              ...settings,
-              ...(integration.settings.nangoMapping
-                ? {
-                    nangoMapping: integration.settings.nangoMapping,
-                  }
-                : {}),
-            },
+            settings,
           },
           transaction,
         )
@@ -1925,8 +1878,6 @@ export default class IntegrationService {
           integrationIdentifier: profileId,
           token,
           refreshToken,
-          limitCount: 0,
-          limitLastResetAt: moment().format('YYYY-MM-DD HH:mm:ss'),
           status: 'in-progress',
           settings: {
             followers: [],