feat: org dashboard metrics endpoints and copy pipe (#3344)

Signed-off-by: Gašper Grom <[email protected]>

Author: gaspergrom

services/libs/tinybird/datasources/org_dash_metric_copy_ds.datasource

@@ -0,0 +1,18 @@
+DESCRIPTION >
+    Organization-project metrics for organization dashboard
+
+SCHEMA >
+    `slug` String,
+    `accountName` String,
+    `commits` UInt64,
+    `orgCommits` UInt64,
+    `prsOpened` UInt64,
+    `orgPrsOpened` UInt64,
+    `contributorCount` UInt64,
+    `orgContributorCount` UInt64,
+    `averageMergeTimeSeconds` Nullable(Float64),
+    `orgAverageMergeTimeSeconds` Nullable(Float64),
+    `maintainersCount` UInt64
+
+ENGINE MergeTree
+ENGINE_SORTING_KEY accountName, slug

services/libs/tinybird/pipes/org_dash_metric_copy_pipe.pipe

@@ -0,0 +1,91 @@
+NODE org_dash_metric_copy_pipe_memberships_deduplicated
+SQL >
+    SELECT accountName, organizationId FROM lfxMemberships FINAL WHERE organizationId != ''
+
+NODE org_dash_metric_copy_pipe_projects_deduplicated
+SQL >
+    SELECT id, slug, segmentId
+    FROM insights_projects_populated_ds
+    WHERE contributorCount > 0 AND organizationCount > 0
+    GROUP BY id, slug, segmentId
+
+NODE org_dash_metric_copy_pipe_org_project_options
+SQL >
+    SELECT *
+    FROM org_dash_metric_copy_pipe_memberships_deduplicated
+    CROSS JOIN org_dash_metric_copy_pipe_projects_deduplicated
+
+NODE org_dash_metric_copy_pipe_project_activities
+SQL >
+    SELECT
+        segmentId,
+        countIf(
+            case when activityId != '' then activityId else null end, type = 'authored-commit'
+        ) as commits,
+        countIf(
+            case when activityId != '' then activityId else null end, type = 'pull_request-opened'
+        ) as prsOpened,
+        uniq(memberId) AS contributorCount
+    FROM activityRelations_deduplicated_cleaned_ds
+    GROUP BY segmentId
+
+NODE org_dash_metric_copy_pipe_org_activities
+SQL >
+    SELECT
+        segmentId,
+        organizationId,
+        countIf(
+            case when activityId != '' then activityId else null end, type = 'authored-commit'
+        ) as orgCommits,
+        countIf(
+            case when activityId != '' then activityId else null end, type = 'pull_request-opened'
+        ) as orgPrsOpened,
+        uniq(memberId) AS orgContributorCount
+    FROM activityRelations_deduplicated_cleaned_ds
+    GROUP BY segmentId, organizationId
+
+NODE org_dash_metric_copy_pipe_project_avg_merge_time
+SQL >
+    SELECT segmentId, round(avg(mergedInSeconds)) as averageMergeTimeSeconds
+    FROM pull_requests_analyzed
+    GROUP BY segmentId
+
+NODE org_dash_metric_copy_pipe_org_avg_merge_time
+SQL >
+    SELECT segmentId, organizationId, round(avg(mergedInSeconds)) as orgAverageMergeTimeSeconds
+    FROM pull_requests_analyzed
+    GROUP BY segmentId, organizationId
+
+NODE org_dash_metric_copy_pipe_org_maintainers
+SQL >
+    SELECT insightsProjectId as id, organizationId, uniq(memberId) as maintainersCount
+    FROM maintainers_roles_copy_ds
+    WHERE role = 'maintainer' AND toYear(endDate) <= 1970 AND organizationId != ''
+    GROUP BY insightsProjectId, organizationId
+
+NODE org_dash_metric_copy_pipe_metrics
+SQL >
+    SELECT
+        slug,
+        accountName,
+        sum(commits) as commits,
+        sum(orgCommits) as orgCommits,
+        sum(prsOpened) as prsOpened,
+        sum(orgPrsOpened) as orgPrsOpened,
+        sum(contributorCount) as contributorCount,
+        sum(orgContributorCount) as orgContributorCount,
+        avg(averageMergeTimeSeconds) as averageMergeTimeSeconds,
+        avg(orgAverageMergeTimeSeconds) as orgAverageMergeTimeSeconds,
+        sum(maintainersCount) as maintainersCount
+    FROM org_dash_metric_copy_pipe_org_project_options
+    LEFT JOIN org_dash_metric_copy_pipe_project_activities USING (segmentId)
+    LEFT JOIN org_dash_metric_copy_pipe_org_activities USING (segmentId, organizationId)
+    LEFT JOIN org_dash_metric_copy_pipe_project_avg_merge_time USING (segmentId)
+    LEFT JOIN org_dash_metric_copy_pipe_org_avg_merge_time USING (segmentId, organizationId)
+    LEFT JOIN org_dash_metric_copy_pipe_org_maintainers USING (id, organizationId)
+    GROUP BY slug, accountName
+
+TYPE COPY
+TARGET_DATASOURCE org_dash_metric_copy_ds
+COPY_MODE replace
+COPY_SCHEDULE 50 0 * * *

services/libs/tinybird/pipes/org_dash_metrics.pipe

@@ -0,0 +1,21 @@
+TOKEN "org_dash_metrics_endpoint_read_9856" READ
+
+NODE org_dash_metrics_results
+DESCRIPTION >
+    Filters all pr opened activities in last year for given project
+
+SQL >
+    %
+    SELECT *
+    FROM org_dash_metric_copy_ds
+    WHERE
+        1 = 1
+        {% if defined(accountName) %}
+            AND accountName
+            = {{ String(accountName, description="Filter accountName", required=False) }}
+        {% end %}
+        {% if defined(slugs) %}
+            AND slug in {{ Array(slugs, 'String', description="Filter slugs", required=False) }}
+        {% end %}
+    LIMIT {{ Int32(pageSize, 20) }}
+    OFFSET {{ Int32(page, 0) * Int32(pageSize, 20) }}