Merge branch 'main' into busy-lobster

max-ostapenko · web-flow · commit 08242e6b1c7c · 2025-09-10T11:22:42.000+02:00
diff --git a/.github/linters/.trivyignore b/.github/linters/.trivyignore
@@ -0,0 +1,2 @@
+# Ignore the dataplexAdmin role issue
+AVD-GCP-0007
diff --git a/.github/linters/eslint.config.mjs b/.github/linters/eslint.config.mjs
@@ -21,7 +21,8 @@ export default [
         operate: 'readonly',
         ctx: 'readonly',
         constants: 'readonly',
-        reports: 'readonly'
+        reports: 'readonly',
+        reservations: 'readonly'
       }
     },
     rules: {
diff --git a/.github/linters/trivy.yaml b/.github/linters/trivy.yaml
@@ -0,0 +1 @@
+ignorefile: ".github/linters/.trivyignore"
diff --git a/.github/linters/zizmor.yaml b/.github/linters/zizmor.yaml
@@ -0,0 +1,4 @@
+rules:
+  unpinned-uses:
+    ignore:
+      - ci.yaml
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -24,13 +24,13 @@ jobs:
         uses: actions/checkout@v5
         with:
           fetch-depth: 0
+          persist-credentials: false
 
       - name: Lint Code Base
-        uses: super-linter/super-linter/slim@v8.0.0
+        uses: super-linter/super-linter/slim@v8.1.0
         env:
           DEFAULT_BRANCH: main
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          LINTER_RULES_PATH: .
           VALIDATE_JSCPD: false
           VALIDATE_JAVASCRIPT_PRETTIER: false
           VALIDATE_MARKDOWN_PRETTIER: false
diff --git a/definitions/output/crawl/pages.js b/definitions/output/crawl/pages.js
@@ -2,6 +2,8 @@
 assert('corrupted_technology_values')
   .tags(['crawl_complete'])
   .query(ctx => `
+${reservations.reservation_setter(ctx)}
+
 SELECT
   /*
     date,
@@ -30,6 +32,20 @@ ORDER BY cnt_pages DESC
 HAVING cnt_pages > 200
   `)
 
+assert('pages_per_client')
+  .tags(['crawl_complete'])
+  .query(ctx => `
+SELECT
+  client,
+  COUNT(DISTINCT page) AS cnt_pages
+FROM ${ctx.ref('crawl_staging', 'pages')}
+WHERE
+  date = '${constants.currentMonth}'
+GROUP BY
+  client
+HAVING cnt_pages < 20000000
+  `)
+
 publish('pages', {
   type: 'incremental',
   protected: true,
@@ -82,7 +98,7 @@ publish('pages', {
   tags: ['crawl_complete'],
   dependOnDependencyAssertions: true
 }).preOps(ctx => `
-SET @@RESERVATION='${constants.reservation_id}';
+${reservations.reservation_setter(ctx)}
 
 DELETE FROM ${ctx.self()}
 WHERE date = '${constants.currentMonth}' AND
diff --git a/definitions/output/crawl/parsed_css.js b/definitions/output/crawl/parsed_css.js
@@ -7,15 +7,26 @@ publish('parsed_css', {
     clusterBy: ['client', 'is_root_page', 'rank', 'page'],
     requirePartitionFilter: true
   },
+  columns: {
+    date: 'YYYY-MM-DD format of the HTTP Archive monthly crawl',
+    client: 'Test environment: desktop or mobile',
+    page: 'The URL of the page being tested',
+    is_root_page: 'Whether the page is the root of the origin.',
+    root_page: 'The URL of the root page being tested',
+    rank: 'Site popularity rank, from CrUX',
+    url: 'The URL of the request',
+    css: 'The parsed CSS, in JSON format'
+  },
   tags: ['crawl_complete']
 }).preOps(ctx => `
-SET @@RESERVATION='${constants.reservation_id}';
+${reservations.reservation_setter(ctx)}
 
 DELETE FROM ${ctx.self()}
 WHERE date = '${constants.currentMonth}'
   AND client = 'desktop';
 `).query(ctx => `
-SELECT *
+SELECT
+  *
 FROM ${ctx.ref('crawl_staging', 'parsed_css')}
 WHERE date = '${constants.currentMonth}'
   AND client = 'desktop'
@@ -26,7 +37,8 @@ WHERE date = '${constants.currentMonth}'
   AND client = 'mobile';
 
 INSERT INTO ${ctx.self()}
-SELECT *
+SELECT
+  *
 FROM ${ctx.ref('crawl_staging', 'parsed_css')}
 WHERE date = '${constants.currentMonth}'
   AND client = 'mobile'
diff --git a/definitions/output/crawl/requests.js b/definitions/output/crawl/requests.js
@@ -38,7 +38,7 @@ publish('requests', {
   },
   tags: ['crawl_complete']
 }).preOps(ctx => `
-SET @@RESERVATION='${constants.reservation_id}';
+${reservations.reservation_setter(ctx)}
 
 FOR client_var IN (SELECT * FROM UNNEST(['desktop', 'mobile']) AS value) DO
   FOR is_root_page_var IN (SELECT * FROM UNNEST([TRUE, FALSE]) AS value) DO
diff --git a/definitions/output/f1/pages_latest.js b/definitions/output/f1/pages_latest.js
@@ -7,8 +7,8 @@ publish('pages_latest', {
     clusterBy: ['client', 'is_root_page', 'rank', 'page']
   },
   tags: ['crawl_complete']
-}).preOps(`
-SET @@RESERVATION='${constants.reservation_id}';
+}).preOps(ctx => `
+${reservations.reservation_setter(ctx)}
 `).query(ctx => `
 SELECT
   date,
diff --git a/definitions/output/f1/requests_latest.js b/definitions/output/f1/requests_latest.js
@@ -7,8 +7,8 @@ publish('requests_latest', {
     clusterBy: ['client', 'is_root_page', 'rank', 'type']
   },
   tags: ['crawl_complete']
-}).preOps(`
-SET @@RESERVATION='${constants.reservation_id}';
+}).preOps(ctx => `
+${reservations.reservation_setter(ctx)}
 `).query(ctx => `
 SELECT
   date,
diff --git a/definitions/output/reports/reports_html_elements_popularity.js b/definitions/output/reports/reports_html_elements_popularity.js
@@ -1,5 +1,3 @@
-const pastMonth = constants.fnPastMonth(constants.currentMonth)
-
 publish('html_elements_popularity', {
   schema: 'reports',
   type: 'incremental',
@@ -18,7 +16,7 @@ try {
 ''';
 
 DELETE FROM ${ctx.self()}
-WHERE date = '${pastMonth}';
+WHERE date = '${constants.currentMonth}';
 `).query(ctx => `
 WITH pages_data AS (
   SELECT
@@ -29,7 +27,7 @@ WITH pages_data AS (
     custom_metrics.element_count
   FROM ${ctx.ref('crawl', 'pages')}
   WHERE
-    date = '${pastMonth}' ${constants.devRankFilter}
+    date = '${constants.currentMonth}' ${constants.devRankFilter}
 ),
 
 totals AS (
@@ -69,9 +67,9 @@ SELECT
       "destination": "cloud_storage",
       "config": {
         "bucket": "${constants.bucket}",
-        "name": "${constants.storagePath}${pastMonth.replaceAll('-', '_')}/htmlElementPopularity.json"
+        "name": "${constants.storagePath}${constants.currentMonth.replaceAll('-', '_')}/htmlElementPopularity.json"
       },
-      "query": "SELECT * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'"
+      "query": "SELECT * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${constants.currentMonth}'"
     }'''
   );
 `)
diff --git a/includes/constants.js b/includes/constants.js
@@ -22,49 +22,6 @@ const [
   : ['', '']
 const bucket = 'httparchive'
 const storagePath = 'reports/'
-const reservation_id = 'projects/httparchive/locations/US/reservations/pipeline'
-
-class DataformTemplateBuilder {
-  /**
-   * Create a Dataform SQL template that can be dynamically interpolated
-   * @param {function} templateFn - A function that returns the SQL template string
-   * @returns {function} A function that can be called with a context to resolve the template
-   */
-  static create (templateFn) {
-    return (ctx, params) => {
-      // Custom replacer function to handle nested variables
-      const resolveVariable = (path, scope) => {
-        // Split the path into parts (handles nested objects like 'constants.devRankFilter')
-        const parts = path.split('.')
-
-        // Traverse the provided scope (ctx or global) to find the value
-        let value = scope
-        for (const part of parts) {
-          if (value === undefined || value === null) break
-          value = value[part]
-        }
-
-        // Convert value to appropriate string representation
-        if (value === undefined || value === null) return ''
-        if (typeof value === 'string') return `'${value}'`
-        if (typeof value === 'number') return value.toString()
-        if (typeof value === 'boolean') return value.toString()
-        if (typeof value === 'function') return value.toString()
-
-        // For objects or arrays, use JSON.stringify
-        return JSON.stringify(value)
-      }
-
-      // Generate the template with the provided context and global context
-      return templateFn(ctx, params).replace(/\${(.*?)}/g, (match, p1) => {
-        const [scope, path] = p1.includes(':') ? p1.split(':') : ['params', p1.trim()]
-        return scope === 'ctx'
-          ? resolveVariable(path.trim(), ctx)
-          : resolveVariable(path.trim(), params)
-      })
-    }
-  }
-}
 
 module.exports = {
   today,
@@ -76,8 +33,6 @@ module.exports = {
   environment,
   devTABLESAMPLE,
   devRankFilter,
-  DataformTemplateBuilder,
   bucket,
-  storagePath,
-  reservation_id
+  storagePath
 }
diff --git a/includes/reports.js b/includes/reports.js
@@ -1,4 +1,44 @@
-const { DataformTemplateBuilder } = require('./constants')
+class DataformTemplateBuilder {
+  /**
+   * Create a Dataform SQL template that can be dynamically interpolated
+   * @param {function} templateFn - A function that returns the SQL template string
+   * @returns {function} A function that can be called with a context to resolve the template
+   */
+  static create (templateFn) {
+    return (ctx, params) => {
+      // Custom replacer function to handle nested variables
+      const resolveVariable = (path, scope) => {
+        // Split the path into parts (handles nested objects like 'constants.devRankFilter')
+        const parts = path.split('.')
+
+        // Traverse the provided scope (ctx or global) to find the value
+        let value = scope
+        for (const part of parts) {
+          if (value === undefined || value === null) break
+          value = value[part]
+        }
+
+        // Convert value to appropriate string representation
+        if (value === undefined || value === null) return ''
+        if (typeof value === 'string') return `'${value}'`
+        if (typeof value === 'number') return value.toString()
+        if (typeof value === 'boolean') return value.toString()
+        if (typeof value === 'function') return value.toString()
+
+        // For objects or arrays, use JSON.stringify
+        return JSON.stringify(value)
+      }
+
+      // Generate the template with the provided context and global context
+      return templateFn(ctx, params).replace(/\${(.*?)}/g, (match, p1) => {
+        const [scope, path] = p1.includes(':') ? p1.split(':') : ['params', p1.trim()]
+        return scope === 'ctx'
+          ? resolveVariable(path.trim(), ctx)
+          : resolveVariable(path.trim(), params)
+      })
+    }
+  }
+}
 
 const config = {
   _metrics: {
diff --git a/includes/reservations.js b/includes/reservations.js
@@ -0,0 +1,86 @@
+const default_reservation = null
+const RESERVATION_CONFIG = [
+  {
+    tag: 'high_slots',
+    reservation: 'projects/httparchive/locations/US/reservations/pipeline',
+    actions: [
+      'httparchive.crawl.pages',
+      'httparchive.crawl.requests',
+      'httparchive.crawl.parsed_css',
+      'httparchive.f1.pages_latest',
+      'httparchive.f1.requests_latest'
+    ]
+  },
+  {
+    tag: 'low_slots',
+    reservation: null,
+    actions: []
+  },
+  {
+    tag: 'on_demand',
+    reservation: 'none',
+    actions: [
+      'httparchive.dataform_assertions.corrupted_technology_values'
+    ]
+  }
+]
+
+const RESERVATION_SETS = RESERVATION_CONFIG.map(cfg => ({
+  ...cfg,
+  set: new Set(cfg.actions)
+}))
+
+
+/**
+ * Extracts action name from Dataform context object
+ * @param {Object} ctx - Dataform context object
+ * @returns {string|null} The extracted action name or null if not found
+ */
+function getActionName(ctx) {
+  if (!ctx) return null
+
+  if (typeof ctx.self === 'function' && !ctx?.operation) {
+    // Try primary method: ctx.self()
+    const selfName = ctx.self()
+    if (selfName) return selfName.replace(/`/g, '').trim()
+  } else if (ctx?.operation?.proto?.target) {
+    // Fallback: construct from proto target
+    const t = ctx?.operation?.proto?.target
+    return t ? [t.database, t.name].join('.') : null
+  }
+
+  return null
+}
+
+/**
+ * Determines the appropriate reservation for a given action name
+ * @param {string} actionName - The action name (without backticks)
+ * @returns {string|null} The reservation identifier or null if no reservation assignment needed
+ */
+function getReservation(actionName) {
+  if (!actionName || typeof actionName !== 'string') return default_reservation
+
+  for (const { reservation, set } of Object.values(RESERVATION_SETS)) {
+    if (set.has(actionName)) {
+      return reservation
+    }
+  }
+
+  return default_reservation
+}
+
+
+/**
+ * Generates the reservation SQL statement for a given Dataform context
+ * @param {Object} ctx - Dataform context object with self() method and/or proto.target
+ * @returns {string} The SQL statement to set reservation or empty string
+ */
+function reservation_setter(ctx) {
+  const actionName = getActionName(ctx)
+  const reservation = getReservation(actionName)
+  return reservation ? `SET @@reservation='${reservation}';` : ''
+}
+
+module.exports = {
+  reservation_setter
+}
diff --git a/infra/bigquery-export/Dockerfile b/infra/bigquery-export/Dockerfile
@@ -4,6 +4,9 @@ FROM node:22-slim
 # Set the working directory
 WORKDIR /app
 
+# Create a non-root user
+RUN groupadd -r appuser && useradd -r -g appuser appuser
+
 # Copy package files first for better layer caching
 COPY package*.json ./
 
@@ -15,4 +18,13 @@ ENV EXPORT_CONFIG=""
 # Copy source code
 COPY . .
 
+# Change ownership of the app directory to the non-root user
+RUN chown -R appuser:appuser /app
+
+# Switch to non-root user
+USER appuser
+
+# No healthcheck needed for one-time job containers
+HEALTHCHECK NONE
+
 CMD ["node", "index.js"]
diff --git a/infra/dataform-service/Dockerfile b/infra/dataform-service/Dockerfile
diff --git a/infra/dataform-service/index.js b/infra/dataform-service/index.js
diff --git a/infra/dataform-service/package-lock.json b/infra/dataform-service/package-lock.json
diff --git a/infra/dataform-service/package.json b/infra/dataform-service/package.json
diff --git a/infra/tf/.terraform.lock.hcl b/infra/tf/.terraform.lock.hcl
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+# Ignore the dataplexAdmin role issue`
	`2`	`+AVD-GCP-0007`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+ignorefile: ".github/linters/.trivyignore"`
-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
 +rules:
 +  unpinned-uses:
 +    ignore:
 +      - ci.yaml