Merge branch 'development' into automated-plugin-docs-update

Author: alexandraBara

nodescraper/connection/redfish/redfish_connection.py

@@ -183,6 +183,18 @@ def run_get(self, path: Union[str, RedfishPath]) -> RedfishGetResult:
                 status_code=None,
             )
 
+    def copy(self) -> "RedfishConnection":
+        """Return a new connection with the same config and its own session (for concurrent use)."""
+        return RedfishConnection(
+            base_url=self.base_url,
+            username=self.username,
+            password=self.password,
+            timeout=self.timeout,
+            use_session_auth=self.use_session_auth,
+            verify_ssl=self.verify_ssl,
+            api_root=self.api_root,
+        )
+
     def get_service_root(self) -> dict[str, Any]:
         """GET service root (e.g. /redfish/v1/)."""
         return self.get(RedfishPath(self.api_root))

nodescraper/models/event.py

@@ -28,12 +28,28 @@
 import re
 import uuid
 from enum import Enum
-from typing import Optional, Union
+from typing import Any, Optional, Union
 
 from pydantic import BaseModel, Field, field_serializer, field_validator
 
 from nodescraper.enums import EventPriority
 
+
+def _data_to_json_safe(obj: Any) -> Any:
+    """Recursively convert event data to JSON-serializable form (e.g. exceptions -> str)."""
+    if isinstance(obj, BaseException):
+        return str(obj)
+    if isinstance(obj, dict):
+        return {k: _data_to_json_safe(v) for k, v in obj.items()}
+    if isinstance(obj, (list, tuple)):
+        return [_data_to_json_safe(v) for v in obj]
+    if isinstance(obj, (str, int, float, bool, type(None))):
+        return obj
+    if isinstance(obj, (Enum, datetime.datetime, uuid.UUID)):
+        return str(obj)
+    return str(obj)
+
+
 LOG_LEVEL_MAP = {
     logging.INFO: EventPriority.INFO,
     logging.WARNING: EventPriority.WARNING,
@@ -127,6 +143,11 @@ def serialize_priority(self, priority: EventPriority, _info) -> str:
         """
         return priority.name
 
+    @field_serializer("data")
+    def serialize_data(self, data: dict, _info) -> dict:
+        """Ensure data is JSON-serializable (e.g. convert nested exceptions to str)."""
+        return _data_to_json_safe(data)
+
     @field_validator("data")
     @classmethod
     def validate_data(cls, data: dict) -> dict:

nodescraper/plugins/ooband/redfish_endpoint/collector_args.py

@@ -27,9 +27,34 @@
 
 
 class RedfishEndpointCollectorArgs(BaseModel):
-    """Collection args: uris to GET."""
+    """Collection args: uris to GET (or discover from tree), optional concurrency and tree discovery."""
 
-    uris: list[str] = Field(default_factory=list)
+    uris: list[str] = Field(
+        default_factory=list,
+        description="Redfish URIs to GET. Ignored when discover_tree is True.",
+    )
+    discover_tree: bool = Field(
+        default=False,
+        description="If True, discover endpoints from the BMC Redfish tree (service root and links) instead of using uris.",
+    )
+    tree_max_depth: int = Field(
+        default=2,
+        ge=1,
+        le=10,
+        description="When discover_tree is True: max traversal depth (1=service root only, 2=root + collections, 3=+ members).",
+    )
+    tree_max_endpoints: int = Field(
+        default=0,
+        ge=0,
+        le=10_000,
+        description="When discover_tree is True: max endpoints to discover (0=no limit).",
+    )
+    max_workers: int = Field(
+        default=1,
+        ge=1,
+        le=32,
+        description="Max concurrent GETs (1=sequential). Use >1 for async endpoint fetches.",
+    )
 
     @field_validator("uris", mode="before")
     @classmethod

nodescraper/plugins/ooband/redfish_endpoint/endpoint_analyzer.py

@@ -129,10 +129,10 @@ def analyze_data(
                         )
 
         if failed:
-            details = "; ".join(f"{f['uri']} {f['path']}: {f['reason']}" for f in failed)
+            description = f"Redfish endpoint checks failed: {len(failed)} failure(s)"
             self._log_event(
                 category=EventCategory.TELEMETRY,
-                description=f"Redfish endpoint checks failed: {len(failed)} failure(s) — {details}",
+                description=description,
                 data={"failures": failed},
                 priority=EventPriority.WARNING,
                 console_log=True,

nodescraper/plugins/ooband/redfish_endpoint/endpoint_collector.py

@@ -23,15 +23,99 @@
 # SOFTWARE.
 #
 ###############################################################################
-from typing import Optional
+from collections import deque
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Any, Optional
+from urllib.parse import urlparse
 
 from nodescraper.base import RedfishDataCollector
+from nodescraper.connection.redfish import RedfishConnection, RedfishGetResult
 from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus
 from nodescraper.models import TaskResult
 
 from .collector_args import RedfishEndpointCollectorArgs
 from .endpoint_data import RedfishEndpointDataModel
 
+ODATA_ID = "@odata.id"
+MEMBERS = "Members"
+
+
+def _normalize_path(odata_id: str, api_root: str) -> str:
+    """Convert @odata.id value (URL or path) to a normalized path under api_root."""
+    if not odata_id or not isinstance(odata_id, str):
+        return ""
+    s = odata_id.strip()
+    if s.startswith(("http://", "https://")):
+        parsed = urlparse(s)
+        s = parsed.path or "/"
+    if not s.startswith("/"):
+        s = "/" + s
+    s = s.rstrip("/") or "/"
+    api_root_norm = api_root.strip("/")
+    if api_root_norm and not s.startswith("/" + api_root_norm):
+        return ""
+    return s
+
+
+def _extract_odata_ids(obj: Any) -> list[str]:
+    """Recursively extract all @odata.id values from a Redfish JSON body."""
+    out: list[str] = []
+    if isinstance(obj, dict):
+        if ODATA_ID in obj and isinstance(obj[ODATA_ID], str):
+            out.append(obj[ODATA_ID])
+        for k, v in obj.items():
+            if k == MEMBERS and isinstance(v, list):
+                for item in v:
+                    if (
+                        isinstance(item, dict)
+                        and ODATA_ID in item
+                        and isinstance(item[ODATA_ID], str)
+                    ):
+                        out.append(item[ODATA_ID])
+            elif isinstance(v, dict):
+                out.extend(_extract_odata_ids(v))
+            elif isinstance(v, list):
+                for item in v:
+                    if isinstance(item, dict):
+                        out.extend(_extract_odata_ids(item))
+    return out
+
+
+def _discover_tree(
+    connection: RedfishConnection,
+    api_root: str,
+    max_depth: int,
+    max_endpoints: int,
+) -> tuple[list[str], dict[str, dict], list[RedfishGetResult]]:
+    """
+    Traverse the Redfish resource tree from the service root.
+
+    max_depth matches collection_args.tree_max_depth: 1 = service root only; 2 = root + one link
+    level; child links are only enqueued when depth + 1 < max_depth (root is depth 0).
+    """
+    root_path = _normalize_path(api_root, api_root) or ("/" + api_root.strip("/"))
+    seen: set[str] = set()
+    to_visit: deque[tuple[str, int]] = deque([(root_path, 0)])
+    responses: dict[str, dict] = {}
+    results: list[RedfishGetResult] = []
+    while to_visit:
+        if max_endpoints and len(seen) >= max_endpoints:
+            break
+        path, depth = to_visit.popleft()
+        if path in seen or depth > max_depth:
+            continue
+        seen.add(path)
+        res = connection.run_get(path)
+        results.append(res)
+        if res.success and res.data is not None:
+            responses[path] = res.data
+            for odata_id in _extract_odata_ids(res.data):
+                link_path = _normalize_path(odata_id, api_root)
+                # Follow only if the child depth stays strictly below max_depth (1 = root only).
+                if link_path and link_path not in seen and depth + 1 < max_depth:
+                    to_visit.append((link_path, depth + 1))
+    return sorted(seen), responses, results
+
 
 def _uris_from_args(args: Optional[RedfishEndpointCollectorArgs]) -> list[str]:
     """Return list of URIs from collector args.uris."""
@@ -40,6 +124,18 @@ def _uris_from_args(args: Optional[RedfishEndpointCollectorArgs]) -> list[str]:
     return list(args.uris) if args.uris else []
 
 
+def _discover_tree_enabled(args: Optional[RedfishEndpointCollectorArgs]) -> bool:
+    """True only when tree discovery is explicitly enabled (avoids string/other truthy junk)."""
+    if args is None:
+        return False
+    return getattr(args, "discover_tree", False) is True
+
+
+def _fetch_one(connection_copy: RedfishConnection, path: str) -> RedfishGetResult:
+    """Run a single GET on a connection copy (used from worker threads)."""
+    return connection_copy.run_get(path)
+
+
 class RedfishEndpointCollector(
     RedfishDataCollector[RedfishEndpointDataModel, RedfishEndpointCollectorArgs]
 ):
@@ -50,30 +146,106 @@ class RedfishEndpointCollector(
     def collect_data(
         self, args: Optional[RedfishEndpointCollectorArgs] = None
     ) -> tuple[TaskResult, Optional[RedfishEndpointDataModel]]:
-        """GET each configured Redfish URI via _run_redfish_get() and store the JSON response."""
+        """Collect via tree discovery, or via explicit URIs, or skip if neither is configured."""
         uris = _uris_from_args(args)
+        use_tree = _discover_tree_enabled(args)
+
+        # 1) Tree discovery: only when discover_tree is explicitly true
+        if use_tree:
+            api_root = getattr(self.connection, "api_root", "redfish/v1")
+            max_depth = getattr(args, "tree_max_depth", 2) if args else 2
+            max_endpoints = (getattr(args, "tree_max_endpoints", 0) or 0) if args else 0
+            _paths, responses, results = _discover_tree(
+                self.connection,
+                api_root,
+                max_depth=max_depth,
+                max_endpoints=max_endpoints,
+            )
+            for res in results:
+                self.result.artifacts.append(res)
+                if not res.success and res.error:
+                    self._log_event(
+                        category=EventCategory.RUNTIME,
+                        description=f"Redfish GET failed during tree discovery for {res.path}: {res.error}",
+                        priority=EventPriority.WARNING,
+                        console_log=True,
+                    )
+            if not responses:
+                self.result.message = "No Redfish endpoints discovered from tree"
+                self.result.status = ExecutionStatus.ERROR
+                return self.result, None
+            data = RedfishEndpointDataModel(responses=responses)
+            self.result.message = f"Collected {len(responses)} Redfish endpoint(s) from tree"
+            self.result.status = ExecutionStatus.OK
+            return self.result, data
+
+        # 2) URI list: when discover_tree is false/absent and uris are provided
         if not uris:
-            self.result.message = "No Redfish URIs configured"
+            self.result.message = (
+                "No collection mode configured: set collection_args.discover_tree to true "
+                "or provide collection_args.uris"
+            )
             self.result.status = ExecutionStatus.NOT_RAN
             return self.result, None
 
-        responses: dict[str, dict] = {}
+        paths = []
         for uri in uris:
-            path = uri
+            path = uri.strip() if uri else ""
             if not path:
                 continue
             if not path.startswith("/"):
                 path = "/" + path
-            res = self._run_redfish_get(path, log_artifact=True)
-            if res.success and res.data is not None:
-                responses[res.path] = res.data
-            else:
-                self._log_event(
-                    category=EventCategory.RUNTIME,
-                    description=f"Redfish GET failed for {path}: {res.error or 'unknown'}",
-                    priority=EventPriority.WARNING,
-                    console_log=True,
-                )
+            paths.append(path)
+
+        max_workers = getattr(args, "max_workers", 1) if args else 1
+        max_workers = min(max_workers, len(paths))
+
+        if max_workers <= 1:
+            # Sequential
+            responses = {}
+            for path in paths:
+                res = self._run_redfish_get(path, log_artifact=True)
+                if res.success and res.data is not None:
+                    responses[res.path] = res.data
+                else:
+                    self._log_event(
+                        category=EventCategory.RUNTIME,
+                        description=f"Redfish GET failed for {path}: {res.error or 'unknown'}",
+                        priority=EventPriority.WARNING,
+                        console_log=True,
+                    )
+        else:
+            # Concurrent: one connection copy per worker, collect results in main thread
+            responses = {}
+            results = []
+            with ThreadPoolExecutor(max_workers=max_workers) as executor:
+                futures = {}
+                for path in paths:
+                    conn = self.connection.copy()
+                    futures[executor.submit(_fetch_one, conn, path)] = path
+                for future in as_completed(futures):
+                    path = futures[future]
+                    try:
+                        res = future.result()
+                        results.append(res)
+                        if res.success and res.data is not None:
+                            responses[res.path] = res.data
+                        else:
+                            self._log_event(
+                                category=EventCategory.RUNTIME,
+                                description=f"Redfish GET failed for {path}: {res.error or 'unknown'}",
+                                priority=EventPriority.WARNING,
+                                console_log=True,
+                            )
+                    except Exception as e:
+                        self._log_event(
+                            category=EventCategory.RUNTIME,
+                            description=f"Redfish GET failed for {path}: {e!s}",
+                            priority=EventPriority.WARNING,
+                            console_log=True,
+                        )
+            for res in results:
+                self.result.artifacts.append(res)
 
         if not responses:
             self.result.message = "No Redfish endpoints could be read"

test/functional/conftest.py

@@ -28,9 +28,24 @@
 import subprocess
 import sys
 from typing import List
+from unittest.mock import MagicMock
 
 import pytest
 
+from nodescraper.models.systeminfo import OSFamily, SystemInfo
+
+
+@pytest.fixture
+def system_info():
+    """Minimal SystemInfo for collectors that require it (same shape as test/unit/conftest)."""
+    return SystemInfo(name="test_host", platform="X", os_family=OSFamily.LINUX, sku="TEST")
+
+
+@pytest.fixture
+def redfish_conn_mock():
+    """MagicMock Redfish connection for Redfish plugin tests."""
+    return MagicMock()
+
 
 @pytest.fixture
 def run_cli_command():