Allow pushing user-allocation membership to Keycloak

A Keycloak admin client has been added
When `activate_allocation` is called, the user is added
to a Keycloak group named using a format string defined in the
allocation's resource attribute "Format String for Keystone Group Names"
If the user does not already exist in Keycloak, the case is ignored for now

Keycloak integration is optional, toggled by setting the env var "KEYCLOAK_BASE_URL"
Authentication to Keycloak is done via client credentials grant

When `deactivate_allocation` is called, the user is removed from the Keycloak group

New functional test added for Keycloak integration

A comment in `validate_allocations` has been updated to
reflect the more restrictive validation behavior, where users on cluster projects
will be removed if they are not part of the Coldfront allocation (rather
than if they are not registered on Coldfront at all).

Author: QuanMPhm

.github/workflows/test-functional-keycloak.yaml

@@ -0,0 +1,35 @@
+name: test-functional-keycloak
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: 3.12
+
+      - name: Upgrade and install packages
+        run: |
+          bash ./ci/setup-ubuntu.sh
+
+      - name: Install Keycloak
+        run: |
+          bash ./ci/setup-keycloak.sh
+
+      - name: Install ColdFront and plugin
+        run: |
+          ./ci/setup.sh
+
+      - name: Run functional tests
+        run: |
+          ./ci/run_functional_tests_keycloak.sh

ci/run_functional_tests_keycloak.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -xe
+
+if [[ ! "${CI}" == "true" ]]; then
+    source /tmp/coldfront_venv/bin/activate
+fi
+
+export DJANGO_SETTINGS_MODULE="local_settings"
+export PYTHONWARNINGS="ignore:Unverified HTTPS request"
+
+export KEYCLOAK_BASE_URL="http://localhost:8080"
+export KEYCLOAK_REALM="master"
+export KEYCLOAK_CLIENT_ID="coldfront"
+export KEYCLOAK_CLIENT_SECRET="nomoresecret"
+
+coverage run --source="." -m django test coldfront_plugin_cloud.tests.functional.keycloak
+coverage report

ci/setup-keycloak.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+set -xe
+
+sudo docker rm -f keycloak
+
+sudo docker run -d --name keycloak \
+    -e KEYCLOAK_ADMIN=admin \
+    -e KEYCLOAK_ADMIN_PASSWORD=nomoresecret \
+    -p 8080:8080 \
+    -p 8443:8443 \
+    quay.io/keycloak/keycloak:25.0 start-dev
+
+# wait for keycloak to be ready
+until curl -s http://localhost:8080/auth/realms/master; do
+    echo "Waiting for Keycloak to be ready..."
+    sleep 5
+done
+
+# Create client and add admin role to client's service account
+ACCESS_TOKEN=$(curl -X POST "http://localhost:8080/realms/master/protocol/openid-connect/token" \
+     -d "username=admin" \
+     -d "password=nomoresecret" \
+     -d "grant_type=password" \
+     -d "client_id=admin-cli" \
+     -d "scope=openid" \
+| jq -r '.access_token')
+
+
+curl -X POST "http://localhost:8080/admin/realms/master/clients" \
+    -H "Authorization: Bearer $ACCESS_TOKEN" \
+    -H "Content-Type: application/json" \
+    -d '{
+        "clientId": "coldfront",
+        "secret": "nomoresecret",
+        "redirectUris": ["http://localhost:8080/*"],
+        "serviceAccountsEnabled": true
+    }'
+
+COLDFRONT_CLIENT_ID=$(curl -X GET "http://localhost:8080/admin/realms/master/clients?clientId=coldfront" \
+    -H "Authorization: Bearer $ACCESS_TOKEN" | jq -r '.[0].id')
+
+
+COLDFRONT_SERVICE_ACCOUNT_ID=$(curl -X GET "http://localhost:8080/admin/realms/master/clients/$COLDFRONT_CLIENT_ID/service-account-user" \
+    -H "Authorization: Bearer $ACCESS_TOKEN" \
+    -H "Content-Type: application/json" \
+| jq -r '.id')
+
+ADMIN_ROLE_ID=$(curl -X GET "http://localhost:8080/admin/realms/master/roles/admin" \
+    -H "Authorization: Bearer $ACCESS_TOKEN" | jq -r '.id')
+
+# Add admin role to the service account user
+curl -X POST "http://localhost:8080/admin/realms/master/users/$COLDFRONT_SERVICE_ACCOUNT_ID/role-mappings/realm" \
+    -H "Authorization: Bearer $ACCESS_TOKEN" \
+    -H "Content-Type: application/json" \
+    -d '[
+            {
+                "id": "'$ADMIN_ROLE_ID'",
+                "name": "admin"
+            }
+        ]'

requirements.txt

@@ -12,3 +12,4 @@ python-novaclient
 python-neutronclient
 python-swiftclient
 pytz
+requests

src/coldfront_plugin_cloud/attributes.py

@@ -24,7 +24,6 @@ class CloudAllocationAttribute:
 RESOURCE_API_URL = "OpenShift API Endpoint URL"
 RESOURCE_IDENTITY_NAME = "OpenShift Identity Provider Name"
 RESOURCE_ROLE = "Role for User in Project"
-RESOURCE_QUOTA_RESOURCES = "Available Quota Resources"
 
 RESOURCE_FEDERATION_PROTOCOL = "OpenStack Federation Protocol"
 RESOURCE_IDP = "OpenStack Identity Provider"
@@ -35,6 +34,8 @@ class CloudAllocationAttribute:
 
 RESOURCE_EULA_URL = "EULA URL"
 RESOURCE_CLUSTER_NAME = "Internal Cluster Name"
+RESOURCE_QUOTA_RESOURCES = "Available Quota Resources"
+RESOURCE_KEYCLOAK_GROUP_TEMPLATE = "Template String for Keycloak Group Names"
 
 RESOURCE_ATTRIBUTES = [
     CloudResourceAttribute(name=RESOURCE_AUTH_URL),
@@ -45,6 +46,7 @@ class CloudAllocationAttribute:
     CloudResourceAttribute(name=RESOURCE_PROJECT_DOMAIN),
     CloudResourceAttribute(name=RESOURCE_ROLE),
     CloudResourceAttribute(name=RESOURCE_QUOTA_RESOURCES),
+    CloudResourceAttribute(name=RESOURCE_KEYCLOAK_GROUP_TEMPLATE),
     CloudResourceAttribute(name=RESOURCE_USER_DOMAIN),
     CloudResourceAttribute(name=RESOURCE_EULA_URL),
     CloudResourceAttribute(name=RESOURCE_DEFAULT_PUBLIC_NETWORK),

src/coldfront_plugin_cloud/base.py

@@ -1,15 +1,19 @@
 import abc
 import functools
 import json
+import logging
 from typing import NamedTuple
 
 from coldfront.core.allocation import models as allocation_models
 from coldfront.core.resource import models as resource_models
 
-from coldfront_plugin_cloud import attributes
+from coldfront_plugin_cloud import attributes, utils
 from coldfront_plugin_cloud.models.quota_models import QuotaSpecs
 
 
+logger = logging.getLogger(__name__)
+
+
 class ResourceAllocator(abc.ABC):
     resource_type = ""
 
@@ -45,6 +49,94 @@ def get_or_create_federated_user(self, username):
             user = self.create_federated_user(username)
         return user
 
+    def set_default_quota_on_allocation(self, coldfront_attr):
+        resource_quotaspecs = self.resource_quotaspecs
+        value = resource_quotaspecs.root[coldfront_attr].quota_by_su_quantity(
+            self.allocation.quantity
+        )
+        utils.set_attribute_on_allocation(self.allocation, coldfront_attr, value)
+        return value
+
+    def set_users(self, project_id, apply):
+        coldfront_users = allocation_models.AllocationUser.objects.filter(
+            allocation=self.allocation, status__name="Active"
+        )
+        cluster_users = self.get_users(project_id)
+        failed_validation = False
+
+        # Create users that exist in coldfront but not in the resource
+        for coldfront_user in coldfront_users:
+            coldfront_username = coldfront_user.user.username
+            if coldfront_username not in cluster_users:
+                failed_validation = True
+                logger.info(f"{coldfront_username} is not part of {project_id}")
+                if apply:
+                    self.get_or_create_federated_user(coldfront_username)
+                    self.assign_role_on_user(coldfront_username, project_id)
+
+        # remove users that are in the resource but not in coldfront
+        users = set(
+            [coldfront_user.user.username for coldfront_user in coldfront_users]
+        )
+        for allocation_user in cluster_users:
+            if allocation_user not in users:
+                failed_validation = True
+                logger.info(
+                    f"{allocation_user} exists in the resource {project_id} but not in coldfront"
+                )
+                if apply:
+                    self.remove_role_from_user(allocation_user, project_id)
+
+        return failed_validation
+
+    def check_and_apply_quota_attr(
+        self,
+        attr: str,
+        expected_quota: int | None,
+        current_quota: int | None,
+        apply: bool,
+    ) -> bool:
+        failed_validation = False
+        if current_quota is None and expected_quota is None:
+            msg = (
+                f"Value for quota for {attr} is not set anywhere"
+                f" on {self.allocation_str}"
+            )
+            failed_validation = True
+
+            if apply:
+                expected_quota = self.set_default_quota_on_allocation(attr)
+                msg = f"Added default quota for {attr} to {self.allocation_str} to {expected_quota}"
+            logger.info(msg)
+        elif current_quota is not None and expected_quota is None:
+            msg = (
+                f'Attribute "{attr}" expected on {self.allocation_str} but not set.'
+                f" Current quota is {current_quota}."
+            )
+
+            if apply:
+                utils.set_attribute_on_allocation(self.allocation, attr, current_quota)
+
+                # To pass `current_quota != expected_quota` check
+                expected_quota = current_quota
+
+                msg = f"{msg} Attribute set to match current quota."
+            logger.info(msg)
+
+        if current_quota != expected_quota:
+            msg = (
+                f"Value for quota for {attr} = {current_quota} does not match expected"
+                f" value of {expected_quota} on {self.allocation_str}"
+            )
+            logger.info(msg)
+            failed_validation = True
+
+        return failed_validation
+
+    @functools.cached_property
+    def allocation_str(self):
+        return f'allocation {self.allocation.pk} of project "{self.allocation.project.title}"'
+
     @functools.cached_property
     def auth_url(self):
         return self.resource.get_attribute(attributes.RESOURCE_AUTH_URL).rstrip("/")
@@ -54,7 +146,11 @@ def member_role_name(self):
         return self.resource.get_attribute(attributes.RESOURCE_ROLE) or "member"
 
     @abc.abstractmethod
-    def set_project_configuration(self, project_id, dry_run=False):
+    def set_project_configuration(self, project_id, apply=True):
+        pass
+
+    @abc.abstractmethod
+    def get_project(self, project_id):
         pass
 
     @abc.abstractmethod
@@ -85,6 +181,10 @@ def get_quota(self, project_id):
     def create_federated_user(self, unique_id):
         pass
 
+    @abc.abstractmethod
+    def get_users(self, unique_id):
+        pass
+
     @abc.abstractmethod
     def get_federated_user(self, unique_id):
         pass

src/coldfront_plugin_cloud/kc_client.py

@@ -0,0 +1,77 @@
+import os
+
+import requests
+
+
+class KeyCloakAPIClient:
+    def __init__(self):
+        self.base_url = os.getenv("KEYCLOAK_BASE_URL")
+        self.realm = os.getenv("KEYCLOAK_REALM")
+        self.client_id = os.getenv("KEYCLOAK_CLIENT_ID")
+        self.client_secret = os.getenv("KEYCLOAK_CLIENT_SECRET")
+
+        self.token_url = (
+            f"{self.base_url}/realms/{self.realm}/protocol/openid-connect/token"
+        )
+
+    @property
+    def api_client(self):
+        params = {
+            "grant_type": "client_credentials",
+            "client_id": self.client_id,
+            "client_secret": self.client_secret,
+        }
+        r = requests.post(self.token_url, data=params)
+        r.raise_for_status()
+        headers = {
+            "Authorization": ("Bearer %s" % r.json()["access_token"]),
+            "Content-Type": "application/json",
+        }
+        session = requests.session()
+        session.headers.update(headers)
+        return session
+
+    def create_group(self, group_name):
+        url = f"{self.base_url}/admin/realms/{self.realm}/groups"
+        payload = {"name": group_name}
+        response = self.api_client.post(url, json=payload)
+
+        # If group already exists, ignore and move on
+        if response.status_code not in (201, 409):
+            response.raise_for_status()
+
+    def get_group_id(self, group_name) -> str | None:
+        """Return None if group not found"""
+        query = f"search={group_name}&exact=true"
+        url = f"{self.base_url}/admin/realms/{self.realm}/groups?{query}"
+        r = self.api_client.get(url)
+        r.raise_for_status()
+        r_json = r.json()
+        return r_json[0]["id"] if r_json else None
+
+    def get_user_id(self, cf_username) -> str | None:
+        """Return None if user not found"""
+        # (Quan) Coldfront usernames map to Keycloak usernames
+        # https://github.com/nerc-project/coldfront-plugin-cloud/pull/249#discussion_r2953393852
+        query = f"username={cf_username}&exact=true"
+        url = f"{self.base_url}/admin/realms/{self.realm}/users?{query}"
+        r = self.api_client.get(url)
+        r.raise_for_status()
+        r_json = r.json()
+        return r_json[0]["id"] if r_json else None
+
+    def add_user_to_group(self, user_id, group_id):
+        url = f"{self.base_url}/admin/realms/{self.realm}/users/{user_id}/groups/{group_id}"
+        r = self.api_client.put(url)
+        r.raise_for_status()
+
+    def remove_user_from_group(self, user_id, group_id):
+        url = f"{self.base_url}/admin/realms/{self.realm}/users/{user_id}/groups/{group_id}"
+        r = self.api_client.delete(url)
+        r.raise_for_status()
+
+    def get_user_groups(self, user_id) -> list[str]:
+        url = f"{self.base_url}/admin/realms/{self.realm}/users/{user_id}/groups"
+        r = self.api_client.get(url)
+        r.raise_for_status()
+        return [group["name"] for group in r.json()]