diff --git a/docs/index.rst b/docs/index.rst index 1ed65f08..8e38f9a5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,6 +13,7 @@ A library for accessing push items from various sources. sources/koji sources/registry sources/errata + sources/konflux sources/pub model/base model/files diff --git a/docs/sources/konflux.rst b/docs/sources/konflux.rst new file mode 100644 index 00000000..40cde036 --- /dev/null +++ b/docs/sources/konflux.rst @@ -0,0 +1,99 @@ +Source: konflux +================ + +The ``konflux`` push source allows the loading of content from local JSON files +organized by advisory. This source is designed for use with Konflux-generated +advisory metadata and does not require network access or external API calls. + +Supported content types: + +* RPMs +* Advisories + +The source is designed to be extensible and can support additional content types +(such as modules, container images, etc.) in the future as needed. + +konflux source URLs +------------------- + +The base form of a konflux source URL is: + +``konflux:base-directory?advisories=RHXA-XXXX:0001[,RHXA-XXXX:0002[,...]]`` + +For example, referencing a single advisory would look like: + +``konflux:/path/to/konflux/data?advisories=RHSA-2020:0509`` + +Multiple advisories can be specified with a comma-separated list: + +``konflux:/path/to/konflux/data?advisories=RHSA-2020:0509,RHSA-2020:0510`` + +The base directory should contain subdirectories named after each advisory ID. +Each advisory subdirectory must contain: + +* ``advisory_cdn_metadata.json`` - Advisory metadata (title, severity, references, packages, etc.) +* ``advisory_cdn_filelist.json`` - RPM file list with checksums, signing keys, and repository destinations + +Directory structure +................... + +Example directory structure:: + + /path/to/konflux/data/ + ├── RHSA-2020:0509/ + │ ├── advisory_cdn_metadata.json + │ └── advisory_cdn_filelist.json + └── RHSA-2020:0510/ + ├── advisory_cdn_metadata.json + └── advisory_cdn_filelist.json + +File format +........... + +**advisory_cdn_metadata.json** + +This file contains advisory metadata in the standard Errata Tool format, including: + +* Advisory ID, title, description, severity +* Package list with checksums +* References (CVEs, Bugzilla links, etc.) +* Release information + +**advisory_cdn_filelist.json** + +This file contains build and RPM information:: + + { + "build-nvr": { + "rpms": { + "rpm-filename.rpm": ["repo1", "repo2", ...] + }, + "checksums": { + "md5": { + "rpm-filename.rpm": "checksum-value" + }, + "sha256": { + "rpm-filename.rpm": "checksum-value" + } + }, + "sig_key": "signing-key-id" + } + } + +Differences from `ErrataSource` +............................... + +Unlike the `ErrataSource`, the `KonfluxSource`: + +* Reads from local JSON files rather than querying the Errata API +* Does not require Koji integration +* Does not currently support filtering by architecture (this use case may be supported in the future) +* Currently produces RPMs and advisories (additional content types such as modules and container images can be supported in the future) +* RPM push items have ``src=None`` (no local RPM files, only metadata) + +Python API reference +-------------------- + +.. autoclass:: pushsource.KonfluxSource + :members: + :special-members: __init__ diff --git a/docs/userguide.rst b/docs/userguide.rst index 728492c4..039fe09a 100644 --- a/docs/userguide.rst +++ b/docs/userguide.rst @@ -91,6 +91,9 @@ For detailed information, see the API reference of the associated class. | errata | ``errata:https://errata.example.com?errata=RHBA-2020:1234`` | :class:`~pushsource.ErrataSource` | Obtain RPMs, container images and advisory | | | | | metadata from Errata Tool | +--------------+-----------------------------------------------------------------------------+-------------------------------------+----------------------------------------------------+ +| konflux | ``konflux:/path/to/data?advisories=RHSA-2020:0509`` | :class:`~pushsource.KonfluxSource` | Obtain RPMs and advisory metadata from local | +| | | | JSON files organized by advisory | ++--------------+-----------------------------------------------------------------------------+-------------------------------------+----------------------------------------------------+ | file | ``file:/tmp/file-to-push`` | n/a | Obtain a single file-backed item of various types. | +--------------+-----------------------------------------------------------------------------+ | | | cgw | ``cgw:/tmp/yaml-file-to-push`` | | | diff --git a/src/pushsource/__init__.py b/src/pushsource/__init__.py index a4da7c14..e52ee1f8 100644 --- a/src/pushsource/__init__.py +++ b/src/pushsource/__init__.py @@ -38,6 +38,7 @@ from pushsource._impl.backend import ( ErrataSource, KojiSource, + KonfluxSource, StagedSource, RegistrySource, PubSource, diff --git a/src/pushsource/_impl/backend/__init__.py b/src/pushsource/_impl/backend/__init__.py index 651c29ee..fdc68a87 100644 --- a/src/pushsource/_impl/backend/__init__.py +++ b/src/pushsource/_impl/backend/__init__.py @@ -1,5 +1,6 @@ from .errata_source import ErrataSource from .koji_source import KojiSource +from .konflux_source import KonfluxSource from .staged import StagedSource from .registry_source import RegistrySource from .direct import DirectSource diff --git a/src/pushsource/_impl/backend/konflux_source/__init__.py b/src/pushsource/_impl/backend/konflux_source/__init__.py new file mode 100644 index 00000000..8e38a0ae --- /dev/null +++ b/src/pushsource/_impl/backend/konflux_source/__init__.py @@ -0,0 +1,3 @@ +from .konflux_source import KonfluxSource + +__all__ = ["KonfluxSource"] diff --git a/src/pushsource/_impl/backend/konflux_source/konflux_loader.py b/src/pushsource/_impl/backend/konflux_source/konflux_loader.py new file mode 100644 index 00000000..b8c6eb26 --- /dev/null +++ b/src/pushsource/_impl/backend/konflux_source/konflux_loader.py @@ -0,0 +1,85 @@ +import json +import logging +import os + +from ...compat_attr import attr + +LOG = logging.getLogger("pushsource.konflux") + + +@attr.s() +class KonfluxAdvisoryData(object): + """Stores advisory data loaded from JSON files.""" + + advisory_id = attr.ib(type=str) + metadata = attr.ib(type=dict) # advisory_cdn_metadata content + filelist = attr.ib(type=dict) # advisory_cdn_filelist content + + +class KonfluxLoader(object): + """Loads advisory data from local JSON files.""" + + def __init__(self, base_dir): + """Initialize loader with base directory. + + Parameters: + base_dir (str): + Base directory containing advisory subdirectories. + """ + self._base_dir = base_dir + + def load_advisory_data(self, advisory_id): + """Load both advisory_cdn_metadata and advisory_cdn_filelist. + + Parameters: + advisory_id (str): + Advisory ID (e.g., "RHSA-2020-0509") + + Returns: + KonfluxAdvisoryData: Named tuple with metadata and filelist + + Raises: + FileNotFoundError: If JSON files don't exist + ValueError: If JSON is invalid or malformed + """ + advisory_dir = os.path.join(self._base_dir, advisory_id) + + LOG.debug("Loading advisory data for %s from %s", advisory_id, advisory_dir) + + metadata = self._load_json( + os.path.join(advisory_dir, "advisory_cdn_metadata.json") + ) + filelist = self._load_json( + os.path.join(advisory_dir, "advisory_cdn_filelist.json") + ) + + LOG.info("Successfully loaded advisory data for %s", advisory_id) + + return KonfluxAdvisoryData( + advisory_id=advisory_id, metadata=metadata, filelist=filelist + ) + + def _load_json(self, filepath): + """Load and parse a JSON file with error handling. + + Parameters: + filepath (str): + Path to JSON file to load + + Returns: + dict: Parsed JSON data + + Raises: + FileNotFoundError: If file doesn't exist + ValueError: If JSON is invalid + """ + if not os.path.exists(filepath): + raise FileNotFoundError("Required file not found: %s" % filepath) + + LOG.debug("Loading JSON file: %s", filepath) + + with open(filepath, "r") as f: + try: + return json.load(f) + except json.JSONDecodeError as e: + raise ValueError("Invalid JSON in %s: %s" % (filepath, str(e))) from e diff --git a/src/pushsource/_impl/backend/konflux_source/konflux_source.py b/src/pushsource/_impl/backend/konflux_source/konflux_source.py new file mode 100644 index 00000000..ed52f456 --- /dev/null +++ b/src/pushsource/_impl/backend/konflux_source/konflux_source.py @@ -0,0 +1,223 @@ +import logging + +from more_executors import Executors + +from ...source import Source +from ...model import ErratumPushItem, RpmPushItem +from ... import compat_attr as attr +from ...helpers import list_argument, as_completed_with_timeout_reset + +from .konflux_loader import KonfluxLoader + +LOG = logging.getLogger("pushsource.konflux") + + +class KonfluxSource(Source): + """Source for push items loaded from Konflux JSON files. + + This source reads advisory metadata from local JSON files organized + in subdirectories per advisory, and generates push items for RPMs + and erratum metadata. + + The source is designed to be extensible and can support additional + content types (such as modules, container images, etc.) in the future. + + Note: This source does not currently support filtering by architecture, + though such filtering could be added if needed. + """ + + def __init__( + self, + url, + advisories, + threads=4, + timeout=60 * 60, + ): + """Create a new Konflux source. + + Parameters: + url (str): + Base directory containing advisory subdirectories. + Each subdirectory should be named after an advisory ID + and contain advisory_cdn_metadata.json and + advisory_cdn_filelist.json files. + + advisories (str, list[str]): + Advisory ID(s) to process. Can be a single string or list. + Multiple IDs can be comma-separated. + + threads (int): + Number of threads for concurrent processing. + + timeout (int): + Timeout in seconds for operations. + """ + self._base_dir = url + self._advisories = list_argument(advisories) + self._threads = threads + self._timeout = timeout + + self._loader = KonfluxLoader(url) + self._executor = Executors.thread_pool( + name="pushsource-konflux", max_workers=threads + ).with_cancel_on_shutdown() + + LOG.info( + "Initialized KonfluxSource with base_dir=%s, advisories=%s", + url, + self._advisories, + ) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self._executor.shutdown(True) + + def __iter__(self): + """Yield push items for all advisories.""" + # Process advisories in parallel + futures_list = [ + self._executor.submit(self._process_advisory, adv_id) + for adv_id in self._advisories + ] + + for future in as_completed_with_timeout_reset( + futures_list, timeout=self._timeout + ): + # Each future returns a list of push items + for item in future.result(): + yield item + + def _process_advisory(self, advisory_id): + """Process a single advisory and return all its push items. + + Parameters: + advisory_id (str): + Advisory ID to process + + Returns: + list[PushItem]: List of push items for this advisory + """ + LOG.info("Processing advisory: %s", advisory_id) + + items = [] + + # Load data from JSON files + data = self._loader.load_advisory_data(advisory_id) + + # Generate erratum push item + items.append(self._create_erratum_item(data)) + + # Generate RPM push items + items.extend(self._create_rpm_items(data)) + + LOG.info("Generated %d push items for %s", len(items), advisory_id) + + return items + + def _create_erratum_item(self, data): + """Create ErratumPushItem from advisory metadata. + + Parameters: + data (KonfluxAdvisoryData): + Advisory data containing metadata and filelist + + Returns: + ErratumPushItem: Erratum push item + """ + # advisory_cdn_metadata already has the right structure + # Just need to ensure it has the expected fields + metadata = data.metadata.copy() + + # Collect all destinations from RPMs + all_destinations = set() + for build_data in data.filelist.values(): + if "rpms" in build_data: + for _, repos in build_data["rpms"].items(): + all_destinations.update(repos) + + # ErratumPushItem._from_data expects metadata with cdn_repo for destinations + metadata["cdn_repo"] = sorted(all_destinations) + + # Use the same factory method as ErrataSource + erratum = ErratumPushItem._from_data(metadata) + + # Set origin to the advisory ID + return attr.evolve(erratum, origin=data.advisory_id) + + def _create_rpm_items(self, data): + """Create RpmPushItem instances from filelist data. + + Since we don't use koji, we construct RPM push items directly + from the information in advisory_cdn_filelist.json. + + Parameters: + data (KonfluxAdvisoryData): + Advisory data containing metadata and filelist + + Returns: + list[RpmPushItem]: List of RPM push items + """ + items = [] + + for build_nvr, build_data in data.filelist.items(): + if "rpms" in build_data: + checksums = build_data.get("checksums", {}) + sig_key = build_data.get("sig_key") + + for rpm_filename, destinations in build_data["rpms"].items(): + # Construct RPM push item + item = self._create_rpm_item( + filename=rpm_filename, + build_nvr=build_nvr, + destinations=destinations, + checksums=checksums, + signing_key=sig_key, + origin=data.advisory_id, + ) + items.append(item) + + return items + + def _create_rpm_item( + self, filename, build_nvr, destinations, checksums, signing_key, origin + ): + """Create a single RpmPushItem from filelist data. + + Parameters: + filename (str): + RPM filename + build_nvr (str): + Build NVR + destinations (list[str]): + List of repository destinations + checksums (dict): + Dict with 'md5' and 'sha256' checksum mappings + signing_key (str): + Signing key ID + origin (str): + Advisory ID + + Returns: + RpmPushItem: RPM push item + """ + # Extract checksums for this specific RPM + md5sum = checksums.get("md5", {}).get(filename) + sha256sum = checksums.get("sha256", {}).get(filename) + + return RpmPushItem( + name=filename, + state="PENDING", + src=None, # RPMs are stored in artifact storage + dest=sorted(destinations), + md5sum=md5sum, + sha256sum=sha256sum, + origin=origin, + build=build_nvr, + signing_key=signing_key, + ) + + +# Register the backend +Source.register_backend("konflux", KonfluxSource) diff --git a/tests/konflux/__init__.py b/tests/konflux/__init__.py new file mode 100644 index 00000000..8a2dbaaa --- /dev/null +++ b/tests/konflux/__init__.py @@ -0,0 +1 @@ +# Test module for Konflux source diff --git a/tests/konflux/data/RHSA-2020:0509/advisory_cdn_filelist.json b/tests/konflux/data/RHSA-2020:0509/advisory_cdn_filelist.json new file mode 100644 index 00000000..b2e914e9 --- /dev/null +++ b/tests/konflux/data/RHSA-2020:0509/advisory_cdn_filelist.json @@ -0,0 +1,61 @@ +{ + "sudo-1.8.25p1-4.el8_0.3": { + "checksums": { + "md5": { + "sudo-1.8.25p1-4.el8_0.3.ppc64le.rpm": "0d56f302617696d3511e71e1669e62c0", + "sudo-1.8.25p1-4.el8_0.3.src.rpm": "f94ab3724b498e3faeab643fe2a67c9c", + "sudo-1.8.25p1-4.el8_0.3.x86_64.rpm": "25e9470c4fe96034fe1d7525c04b5d8e", + "sudo-debuginfo-1.8.25p1-4.el8_0.3.ppc64le.rpm": "e242826fb38f487502cdc1f1a06991d2", + "sudo-debuginfo-1.8.25p1-4.el8_0.3.x86_64.rpm": "91126f02975c06015880d6ea99cb2760", + "sudo-debugsource-1.8.25p1-4.el8_0.3.ppc64le.rpm": "d6da7e2e3d9efe050fef2e8d047682be", + "sudo-debugsource-1.8.25p1-4.el8_0.3.x86_64.rpm": "6b0967941c0caf626c073dc7da0272b6" + }, + "sha256": { + "sudo-1.8.25p1-4.el8_0.3.ppc64le.rpm": "31c4f73af90c6d267cc5281c59e4a93ae3557b2253d9a8e3fef55f3cafca6e54", + "sudo-1.8.25p1-4.el8_0.3.src.rpm": "10d7724302a60d0d2ca890fc7834b8143df55ba1ce0176469ea634ac4ab7aa28", + "sudo-1.8.25p1-4.el8_0.3.x86_64.rpm": "593f872c1869f7beb963c8df2945fc691a1d999945c8c45c6bc7e02731fa016f", + "sudo-debuginfo-1.8.25p1-4.el8_0.3.ppc64le.rpm": "04db0c39efb31518ff79bf98d1c27256d46cdc72b967a5b2094a6efec3166df2", + "sudo-debuginfo-1.8.25p1-4.el8_0.3.x86_64.rpm": "1b7d3a7613236ffea7c4553eb9dea69fc19557005ac3a059d7e83efc08c5b754", + "sudo-debugsource-1.8.25p1-4.el8_0.3.ppc64le.rpm": "355cbb9dc348b17782cff57120391685d6a1f6884facc54fac4b7fb54abeffba", + "sudo-debugsource-1.8.25p1-4.el8_0.3.x86_64.rpm": "43e318fa49e4df685ea0d5f0925a00a336236b2e20f27f9365c39a48102c2cf6" + } + }, + "rpms": { + "sudo-1.8.25p1-4.el8_0.3.ppc64le.rpm": [ + "rhel-8-for-ppc64le-baseos-rpms__8", + "rhel-8-for-ppc64le-baseos-rpms__8_DOT_0" + ], + "sudo-1.8.25p1-4.el8_0.3.src.rpm": [ + "rhel-8-for-ppc64le-baseos-source-rpms__8", + "rhel-8-for-ppc64le-baseos-source-rpms__8_DOT_0", + "rhel-8-for-x86_64-baseos-source-rpms__8", + "rhel-8-for-x86_64-baseos-source-rpms__8_DOT_0", + "rhel-8-for-x86_64-baseos-e4s-source-rpms__8_DOT_0" + ], + "sudo-1.8.25p1-4.el8_0.3.x86_64.rpm": [ + "rhel-8-for-x86_64-baseos-rpms__8", + "rhel-8-for-x86_64-baseos-rpms__8_DOT_0", + "rhel-8-for-x86_64-baseos-e4s-rpms__8_DOT_0" + ], + "sudo-debuginfo-1.8.25p1-4.el8_0.3.ppc64le.rpm": [ + "rhel-8-for-ppc64le-baseos-debug-rpms__8", + "rhel-8-for-ppc64le-baseos-debug-rpms__8_DOT_0" + ], + "sudo-debuginfo-1.8.25p1-4.el8_0.3.x86_64.rpm": [ + "rhel-8-for-x86_64-baseos-debug-rpms__8", + "rhel-8-for-x86_64-baseos-debug-rpms__8_DOT_0", + "rhel-8-for-x86_64-baseos-e4s-debug-rpms__8_DOT_0" + ], + "sudo-debugsource-1.8.25p1-4.el8_0.3.ppc64le.rpm": [ + "rhel-8-for-ppc64le-baseos-debug-rpms__8", + "rhel-8-for-ppc64le-baseos-debug-rpms__8_DOT_0" + ], + "sudo-debugsource-1.8.25p1-4.el8_0.3.x86_64.rpm": [ + "rhel-8-for-x86_64-baseos-debug-rpms__8", + "rhel-8-for-x86_64-baseos-debug-rpms__8_DOT_0", + "rhel-8-for-x86_64-baseos-e4s-debug-rpms__8_DOT_0" + ] + }, + "sig_key": "fd431d51" + } +} diff --git a/tests/konflux/data/RHSA-2020:0509/advisory_cdn_metadata.json b/tests/konflux/data/RHSA-2020:0509/advisory_cdn_metadata.json new file mode 100644 index 00000000..cf4d0b51 --- /dev/null +++ b/tests/konflux/data/RHSA-2020:0509/advisory_cdn_metadata.json @@ -0,0 +1,161 @@ +{ + "description": "The sudo packages contain the sudo utility which allows system administrators to provide certain users with the permission to execute privileged commands, which are used for system management purposes, without having to log in as root.\n\n\nSecurity Fix(es):\n\n\n* sudo: Stack based buffer overflow when pwfeedback is enabled (CVE-2019-18634)\n\n\nFor more details about the security issue(s), including the impact, a CVSS score, acknowledgments, and other related information, refer to the CVE page(s) listed in the References section.", + "from": "release-engineering@redhat.com", + "id": "RHSA-2020:0509", + "issued": "2020-02-13 19:00:11 UTC", + "pkglist": [ + { + "packages": [ + { + "arch": "ppc64le", + "epoch": "0", + "filename": "sudo-1.8.25p1-4.el8_0.3.ppc64le.rpm", + "name": "sudo", + "release": "4.el8_0.3", + "src": "sudo-1.8.25p1-4.el8_0.3.src.rpm", + "sum": [ + "md5", + "0d56f302617696d3511e71e1669e62c0", + "sha256", + "31c4f73af90c6d267cc5281c59e4a93ae3557b2253d9a8e3fef55f3cafca6e54" + ], + "version": "1.8.25p1" + }, + { + "arch": "SRPMS", + "epoch": "0", + "filename": "sudo-1.8.25p1-4.el8_0.3.src.rpm", + "name": "sudo", + "release": "4.el8_0.3", + "src": "sudo-1.8.25p1-4.el8_0.3.src.rpm", + "sum": [ + "md5", + "f94ab3724b498e3faeab643fe2a67c9c", + "sha256", + "10d7724302a60d0d2ca890fc7834b8143df55ba1ce0176469ea634ac4ab7aa28" + ], + "version": "1.8.25p1" + }, + { + "arch": "x86_64", + "epoch": "0", + "filename": "sudo-1.8.25p1-4.el8_0.3.x86_64.rpm", + "name": "sudo", + "release": "4.el8_0.3", + "src": "sudo-1.8.25p1-4.el8_0.3.src.rpm", + "sum": [ + "md5", + "25e9470c4fe96034fe1d7525c04b5d8e", + "sha256", + "593f872c1869f7beb963c8df2945fc691a1d999945c8c45c6bc7e02731fa016f" + ], + "version": "1.8.25p1" + }, + { + "arch": "ppc64le", + "epoch": "0", + "filename": "sudo-debuginfo-1.8.25p1-4.el8_0.3.ppc64le.rpm", + "name": "sudo-debuginfo", + "release": "4.el8_0.3", + "src": "sudo-1.8.25p1-4.el8_0.3.src.rpm", + "sum": [ + "md5", + "e242826fb38f487502cdc1f1a06991d2", + "sha256", + "04db0c39efb31518ff79bf98d1c27256d46cdc72b967a5b2094a6efec3166df2" + ], + "version": "1.8.25p1" + }, + { + "arch": "x86_64", + "epoch": "0", + "filename": "sudo-debuginfo-1.8.25p1-4.el8_0.3.x86_64.rpm", + "name": "sudo-debuginfo", + "release": "4.el8_0.3", + "src": "sudo-1.8.25p1-4.el8_0.3.src.rpm", + "sum": [ + "md5", + "91126f02975c06015880d6ea99cb2760", + "sha256", + "1b7d3a7613236ffea7c4553eb9dea69fc19557005ac3a059d7e83efc08c5b754" + ], + "version": "1.8.25p1" + }, + { + "arch": "ppc64le", + "epoch": "0", + "filename": "sudo-debugsource-1.8.25p1-4.el8_0.3.ppc64le.rpm", + "name": "sudo-debugsource", + "release": "4.el8_0.3", + "src": "sudo-1.8.25p1-4.el8_0.3.src.rpm", + "reboot_suggested": true, + "sum": [ + "md5", + "d6da7e2e3d9efe050fef2e8d047682be", + "sha256", + "355cbb9dc348b17782cff57120391685d6a1f6884facc54fac4b7fb54abeffba" + ], + "version": "1.8.25p1" + }, + { + "arch": "x86_64", + "epoch": "0", + "filename": "sudo-debugsource-1.8.25p1-4.el8_0.3.x86_64.rpm", + "name": "sudo-debugsource", + "release": "4.el8_0.3", + "src": "sudo-1.8.25p1-4.el8_0.3.src.rpm", + "sum": [ + "md5", + "6b0967941c0caf626c073dc7da0272b6", + "sha256", + "43e318fa49e4df685ea0d5f0925a00a336236b2e20f27f9365c39a48102c2cf6" + ], + "version": "1.8.25p1" + } + ] + } + ], + "pulp_user_metadata": { + "content_types": [ + "rpm" + ] + }, + "pushcount": "3", + "reboot_suggested": false, + "references": [ + { + "href": "https://access.redhat.com/errata/RHSA-2020:0509", + "id": null, + "title": "RHSA-2020:0509", + "type": "self" + }, + { + "href": "https://bugzilla.redhat.com/show_bug.cgi?id=1796944", + "id": "1796944", + "title": "CVE-2019-18634 sudo: Stack based buffer overflow when pwfeedback is enabled", + "type": "bugzilla" + }, + { + "href": "https://www.redhat.com/security/data/cve/CVE-2019-18634.html", + "id": "CVE-2019-18634", + "title": "CVE-2019-18634", + "type": "cve" + }, + { + "href": "https://access.redhat.com/security/updates/classification/#important", + "id": "classification", + "title": "important", + "type": "other" + } + ], + "release": "0", + "rights": "Copyright 2020 Red Hat Inc", + "severity": "Important", + "solution": "For details on how to apply this update, which includes the changes described in this advisory, refer to:\n\n\nhttps://access.redhat.com/articles/11258", + "status": "final", + "summary": "An update for sudo is now available for Red Hat Enterprise Linux 8.0 Update Services for SAP Solutions.\n\n\nRed Hat Product Security has rated this update as having a security impact of Important. A Common Vulnerability Scoring System (CVSS) base score, which gives a detailed severity rating, is available for each vulnerability from the CVE link(s) in the References section.", + "title": "Important: sudo security update", + "type": "security", + "updated": "2020-02-13 19:00:11 UTC", + "version": "3" +} diff --git a/tests/konflux/test_konflux_loader.py b/tests/konflux/test_konflux_loader.py new file mode 100644 index 00000000..d832ca54 --- /dev/null +++ b/tests/konflux/test_konflux_loader.py @@ -0,0 +1,182 @@ +import os +import json +import tempfile +import pytest + +from pushsource._impl.backend.konflux_source.konflux_loader import ( + KonfluxLoader, + KonfluxAdvisoryData, +) + + +DATADIR = os.path.join(os.path.dirname(__file__), "data") + + +def test_load_advisory_data(): + """Test loading advisory data successfully.""" + loader = KonfluxLoader(DATADIR) + data = loader.load_advisory_data("RHSA-2020:0509") + + assert isinstance(data, KonfluxAdvisoryData) + assert data.advisory_id == "RHSA-2020:0509" + assert isinstance(data.metadata, dict) + assert isinstance(data.filelist, dict) + + # Check metadata has expected keys + assert "id" in data.metadata + assert "title" in data.metadata + assert "severity" in data.metadata + + # Check filelist has expected structure + assert "sudo-1.8.25p1-4.el8_0.3" in data.filelist + assert "rpms" in data.filelist["sudo-1.8.25p1-4.el8_0.3"] + assert "checksums" in data.filelist["sudo-1.8.25p1-4.el8_0.3"] + + +def test_missing_advisory_directory(): + """Test error when advisory directory doesn't exist.""" + loader = KonfluxLoader(DATADIR) + + with pytest.raises(FileNotFoundError) as exc_info: + loader.load_advisory_data("NONEXISTENT-2020:9999") + + assert "Required file not found" in str(exc_info.value) + assert "advisory_cdn_metadata.json" in str(exc_info.value) + + +def test_missing_metadata_file(): + """Test error when metadata file is missing.""" + with tempfile.TemporaryDirectory() as tmpdir: + # Create advisory directory with only filelist + adv_dir = os.path.join(tmpdir, "TEST-2020:0001") + os.makedirs(adv_dir) + + with open(os.path.join(adv_dir, "advisory_cdn_filelist.json"), "w") as f: + json.dump({}, f) + + loader = KonfluxLoader(tmpdir) + + with pytest.raises(FileNotFoundError) as exc_info: + loader.load_advisory_data("TEST-2020:0001") + + assert "advisory_cdn_metadata.json" in str(exc_info.value) + + +def test_missing_filelist_file(): + """Test error when filelist file is missing.""" + with tempfile.TemporaryDirectory() as tmpdir: + # Create advisory directory with only metadata + adv_dir = os.path.join(tmpdir, "TEST-2020:0001") + os.makedirs(adv_dir) + + with open(os.path.join(adv_dir, "advisory_cdn_metadata.json"), "w") as f: + json.dump({"id": "TEST-2020:0001"}, f) + + loader = KonfluxLoader(tmpdir) + + with pytest.raises(FileNotFoundError) as exc_info: + loader.load_advisory_data("TEST-2020:0001") + + assert "advisory_cdn_filelist.json" in str(exc_info.value) + + +def test_invalid_json_metadata(): + """Test error when metadata JSON is invalid.""" + with tempfile.TemporaryDirectory() as tmpdir: + adv_dir = os.path.join(tmpdir, "TEST-2020:0001") + os.makedirs(adv_dir) + + # Create invalid JSON + with open(os.path.join(adv_dir, "advisory_cdn_metadata.json"), "w") as f: + f.write("{ this is not valid json }") + + with open(os.path.join(adv_dir, "advisory_cdn_filelist.json"), "w") as f: + json.dump({}, f) + + loader = KonfluxLoader(tmpdir) + + with pytest.raises(ValueError) as exc_info: + loader.load_advisory_data("TEST-2020:0001") + + assert "Invalid JSON" in str(exc_info.value) + assert "advisory_cdn_metadata.json" in str(exc_info.value) + + +def test_invalid_json_filelist(): + """Test error when filelist JSON is invalid.""" + with tempfile.TemporaryDirectory() as tmpdir: + adv_dir = os.path.join(tmpdir, "TEST-2020:0001") + os.makedirs(adv_dir) + + with open(os.path.join(adv_dir, "advisory_cdn_metadata.json"), "w") as f: + json.dump({"id": "TEST-2020:0001"}, f) + + # Create invalid JSON + with open(os.path.join(adv_dir, "advisory_cdn_filelist.json"), "w") as f: + f.write("{ not valid json }") + + loader = KonfluxLoader(tmpdir) + + with pytest.raises(ValueError) as exc_info: + loader.load_advisory_data("TEST-2020:0001") + + assert "Invalid JSON" in str(exc_info.value) + assert "advisory_cdn_filelist.json" in str(exc_info.value) + + +def test_metadata_content(): + """Test that metadata content is correctly loaded and structured.""" + loader = KonfluxLoader(DATADIR) + data = loader.load_advisory_data("RHSA-2020:0509") + + # Verify specific metadata fields + assert data.metadata["id"] == "RHSA-2020:0509" + assert data.metadata["severity"] == "Important" + assert data.metadata["type"] == "security" + assert "sudo" in data.metadata["title"] + + # Verify additional metadata structure + assert "pkglist" in data.metadata + assert "references" in data.metadata + assert "description" in data.metadata + + +def test_filelist_content(): + """Test that filelist content is correctly loaded and structured.""" + loader = KonfluxLoader(DATADIR) + data = loader.load_advisory_data("RHSA-2020:0509") + + # Verify build exists in filelist + build_data = data.filelist["sudo-1.8.25p1-4.el8_0.3"] + + # Verify top-level structure + assert "rpms" in build_data + assert "checksums" in build_data + assert "sig_key" in build_data + assert build_data["sig_key"] == "fd431d51" + + # Verify a specific RPM and its repositories + assert "sudo-1.8.25p1-4.el8_0.3.x86_64.rpm" in build_data["rpms"] + rpm_repos = build_data["rpms"]["sudo-1.8.25p1-4.el8_0.3.x86_64.rpm"] + assert "rhel-8-for-x86_64-baseos-e4s-rpms__8_DOT_0" in rpm_repos + + # Verify checksums structure + checksums = build_data["checksums"] + + # Should have md5 and sha256 + assert "md5" in checksums + assert "sha256" in checksums + + # Each should be a dict of filename -> checksum + assert isinstance(checksums["md5"], dict) + assert isinstance(checksums["sha256"], dict) + + # Verify specific checksums for the x86_64 RPM + assert ( + checksums["md5"]["sudo-1.8.25p1-4.el8_0.3.x86_64.rpm"] + == "25e9470c4fe96034fe1d7525c04b5d8e" + ) + assert ( + checksums["sha256"]["sudo-1.8.25p1-4.el8_0.3.x86_64.rpm"] + == "593f872c1869f7beb963c8df2945fc691a1d999945c8c45c6bc7e02731fa016f" + ) diff --git a/tests/konflux/test_konflux_source.py b/tests/konflux/test_konflux_source.py new file mode 100644 index 00000000..ccea3a80 --- /dev/null +++ b/tests/konflux/test_konflux_source.py @@ -0,0 +1,315 @@ +import os +import pytest +import tempfile +import json + +from pushsource import Source +from pushsource._impl.backend.konflux_source import KonfluxSource +from pushsource._impl.model import ( + ErratumPushItem, + ErratumPackage, + ErratumPackageCollection, + ErratumReference, + RpmPushItem, +) + + +DATADIR = os.path.join(os.path.dirname(__file__), "data") + + +def test_load_single_advisory(): + """Test loading a single advisory and generating push items.""" + with Source.get("konflux:%s?advisories=RHSA-2020:0509" % DATADIR) as source: + items = list(source) + + # Should have 1 erratum + 7 RPMs + assert len(items) == 8 + + erratum_items = [i for i in items if isinstance(i, ErratumPushItem)] + rpm_items = [i for i in items if isinstance(i, RpmPushItem)] + + # Should have exactly one erratum + assert len(erratum_items) == 1 + erratum = erratum_items[0] + + # Should have 7 RPMs total + assert len(rpm_items) == 7 + + # Validate erratum string representation + assert str(erratum) == "RHSA-2020:0509: Important: sudo security update" + + # Validate basic erratum fields + assert erratum.name == "RHSA-2020:0509" + assert erratum.origin == "RHSA-2020:0509" + assert erratum.severity == "Important" + + # Validate pkglist structure with complete object comparison + assert erratum.pkglist == [ + ErratumPackageCollection( + name="", + packages=[ + ErratumPackage( + arch="ppc64le", + filename="sudo-1.8.25p1-4.el8_0.3.ppc64le.rpm", + epoch="0", + name="sudo", + version="1.8.25p1", + release="4.el8_0.3", + src="sudo-1.8.25p1-4.el8_0.3.src.rpm", + md5sum="0d56f302617696d3511e71e1669e62c0", + sha1sum=None, + sha256sum="31c4f73af90c6d267cc5281c59e4a93ae3557b2253d9a8e3fef55f3cafca6e54", + ), + ErratumPackage( + arch="SRPMS", + filename="sudo-1.8.25p1-4.el8_0.3.src.rpm", + epoch="0", + name="sudo", + version="1.8.25p1", + release="4.el8_0.3", + src="sudo-1.8.25p1-4.el8_0.3.src.rpm", + md5sum="f94ab3724b498e3faeab643fe2a67c9c", + sha1sum=None, + sha256sum="10d7724302a60d0d2ca890fc7834b8143df55ba1ce0176469ea634ac4ab7aa28", + ), + ErratumPackage( + arch="x86_64", + filename="sudo-1.8.25p1-4.el8_0.3.x86_64.rpm", + epoch="0", + name="sudo", + version="1.8.25p1", + release="4.el8_0.3", + src="sudo-1.8.25p1-4.el8_0.3.src.rpm", + md5sum="25e9470c4fe96034fe1d7525c04b5d8e", + sha1sum=None, + sha256sum="593f872c1869f7beb963c8df2945fc691a1d999945c8c45c6bc7e02731fa016f", + ), + ErratumPackage( + arch="ppc64le", + filename="sudo-debuginfo-1.8.25p1-4.el8_0.3.ppc64le.rpm", + epoch="0", + name="sudo-debuginfo", + version="1.8.25p1", + release="4.el8_0.3", + src="sudo-1.8.25p1-4.el8_0.3.src.rpm", + md5sum="e242826fb38f487502cdc1f1a06991d2", + sha1sum=None, + sha256sum="04db0c39efb31518ff79bf98d1c27256d46cdc72b967a5b2094a6efec3166df2", + ), + ErratumPackage( + arch="x86_64", + filename="sudo-debuginfo-1.8.25p1-4.el8_0.3.x86_64.rpm", + epoch="0", + name="sudo-debuginfo", + version="1.8.25p1", + release="4.el8_0.3", + src="sudo-1.8.25p1-4.el8_0.3.src.rpm", + md5sum="91126f02975c06015880d6ea99cb2760", + sha1sum=None, + sha256sum="1b7d3a7613236ffea7c4553eb9dea69fc19557005ac3a059d7e83efc08c5b754", + ), + ErratumPackage( + arch="ppc64le", + filename="sudo-debugsource-1.8.25p1-4.el8_0.3.ppc64le.rpm", + epoch="0", + name="sudo-debugsource", + version="1.8.25p1", + release="4.el8_0.3", + src="sudo-1.8.25p1-4.el8_0.3.src.rpm", + reboot_suggested=True, + md5sum="d6da7e2e3d9efe050fef2e8d047682be", + sha1sum=None, + sha256sum="355cbb9dc348b17782cff57120391685d6a1f6884facc54fac4b7fb54abeffba", + ), + ErratumPackage( + arch="x86_64", + filename="sudo-debugsource-1.8.25p1-4.el8_0.3.x86_64.rpm", + epoch="0", + name="sudo-debugsource", + version="1.8.25p1", + release="4.el8_0.3", + src="sudo-1.8.25p1-4.el8_0.3.src.rpm", + md5sum="6b0967941c0caf626c073dc7da0272b6", + sha1sum=None, + sha256sum="43e318fa49e4df685ea0d5f0925a00a336236b2e20f27f9365c39a48102c2cf6", + ), + ], + short="", + module=None, + ) + ] + + # Erratum destinations should match all unique RPM destinations + rpm_dests = [] + for rpm in rpm_items: + # Each RPM should be shipped to at least one repository + assert len(rpm.dest) >= 1 + rpm_dests.extend(rpm.dest) + assert set(erratum.dest) == set(rpm_dests) + + # Validate all RPM items with complete object comparison + assert sorted(rpm_items, key=lambda rpm: rpm.name) == [ + RpmPushItem( + name="sudo-1.8.25p1-4.el8_0.3.ppc64le.rpm", + state="PENDING", + src=None, + dest=[ + "rhel-8-for-ppc64le-baseos-rpms__8", + "rhel-8-for-ppc64le-baseos-rpms__8_DOT_0", + ], + md5sum="0d56f302617696d3511e71e1669e62c0", + sha256sum="31c4f73af90c6d267cc5281c59e4a93ae3557b2253d9a8e3fef55f3cafca6e54", + origin="RHSA-2020:0509", + build="sudo-1.8.25p1-4.el8_0.3", + signing_key="FD431D51", + ), + RpmPushItem( + name="sudo-1.8.25p1-4.el8_0.3.src.rpm", + state="PENDING", + src=None, + dest=[ + "rhel-8-for-ppc64le-baseos-source-rpms__8", + "rhel-8-for-ppc64le-baseos-source-rpms__8_DOT_0", + "rhel-8-for-x86_64-baseos-e4s-source-rpms__8_DOT_0", + "rhel-8-for-x86_64-baseos-source-rpms__8", + "rhel-8-for-x86_64-baseos-source-rpms__8_DOT_0", + ], + md5sum="f94ab3724b498e3faeab643fe2a67c9c", + sha256sum="10d7724302a60d0d2ca890fc7834b8143df55ba1ce0176469ea634ac4ab7aa28", + origin="RHSA-2020:0509", + build="sudo-1.8.25p1-4.el8_0.3", + signing_key="FD431D51", + ), + RpmPushItem( + name="sudo-1.8.25p1-4.el8_0.3.x86_64.rpm", + state="PENDING", + src=None, + dest=[ + "rhel-8-for-x86_64-baseos-e4s-rpms__8_DOT_0", + "rhel-8-for-x86_64-baseos-rpms__8", + "rhel-8-for-x86_64-baseos-rpms__8_DOT_0", + ], + md5sum="25e9470c4fe96034fe1d7525c04b5d8e", + sha256sum="593f872c1869f7beb963c8df2945fc691a1d999945c8c45c6bc7e02731fa016f", + origin="RHSA-2020:0509", + build="sudo-1.8.25p1-4.el8_0.3", + signing_key="FD431D51", + ), + RpmPushItem( + name="sudo-debuginfo-1.8.25p1-4.el8_0.3.ppc64le.rpm", + state="PENDING", + src=None, + dest=[ + "rhel-8-for-ppc64le-baseos-debug-rpms__8", + "rhel-8-for-ppc64le-baseos-debug-rpms__8_DOT_0", + ], + md5sum="e242826fb38f487502cdc1f1a06991d2", + sha256sum="04db0c39efb31518ff79bf98d1c27256d46cdc72b967a5b2094a6efec3166df2", + origin="RHSA-2020:0509", + build="sudo-1.8.25p1-4.el8_0.3", + signing_key="FD431D51", + ), + RpmPushItem( + name="sudo-debuginfo-1.8.25p1-4.el8_0.3.x86_64.rpm", + state="PENDING", + src=None, + dest=[ + "rhel-8-for-x86_64-baseos-debug-rpms__8", + "rhel-8-for-x86_64-baseos-debug-rpms__8_DOT_0", + "rhel-8-for-x86_64-baseos-e4s-debug-rpms__8_DOT_0", + ], + md5sum="91126f02975c06015880d6ea99cb2760", + sha256sum="1b7d3a7613236ffea7c4553eb9dea69fc19557005ac3a059d7e83efc08c5b754", + origin="RHSA-2020:0509", + build="sudo-1.8.25p1-4.el8_0.3", + signing_key="FD431D51", + ), + RpmPushItem( + name="sudo-debugsource-1.8.25p1-4.el8_0.3.ppc64le.rpm", + state="PENDING", + src=None, + dest=[ + "rhel-8-for-ppc64le-baseos-debug-rpms__8", + "rhel-8-for-ppc64le-baseos-debug-rpms__8_DOT_0", + ], + md5sum="d6da7e2e3d9efe050fef2e8d047682be", + sha256sum="355cbb9dc348b17782cff57120391685d6a1f6884facc54fac4b7fb54abeffba", + origin="RHSA-2020:0509", + build="sudo-1.8.25p1-4.el8_0.3", + signing_key="FD431D51", + ), + RpmPushItem( + name="sudo-debugsource-1.8.25p1-4.el8_0.3.x86_64.rpm", + state="PENDING", + src=None, + dest=[ + "rhel-8-for-x86_64-baseos-debug-rpms__8", + "rhel-8-for-x86_64-baseos-debug-rpms__8_DOT_0", + "rhel-8-for-x86_64-baseos-e4s-debug-rpms__8_DOT_0", + ], + md5sum="6b0967941c0caf626c073dc7da0272b6", + sha256sum="43e318fa49e4df685ea0d5f0925a00a336236b2e20f27f9365c39a48102c2cf6", + origin="RHSA-2020:0509", + build="sudo-1.8.25p1-4.el8_0.3", + signing_key="FD431D51", + ), + ] + + +def test_comma_separated_advisories(): + """Test handling of comma-separated advisory IDs.""" + # Note: We only have one advisory in test data, but we can test the parsing + source = KonfluxSource(url=DATADIR, advisories="RHSA-2020:0509,RHSA-2020:0510") + + # Should parse into list + assert len(source._advisories) == 2 + assert "RHSA-2020:0509" in source._advisories + assert "RHSA-2020:0510" in source._advisories + + +def test_missing_advisory_directory(): + """Test error handling for missing advisory directory.""" + with pytest.raises(FileNotFoundError) as exc_info: + source = KonfluxSource(url=DATADIR, advisories="RHSA-9999:9999") + with source: + list(source) + + assert "advisory_cdn_metadata.json" in str(exc_info.value) + + +def test_invalid_json(): + """Test error handling for invalid JSON files.""" + with tempfile.TemporaryDirectory() as tmpdir: + # Create advisory directory + adv_dir = os.path.join(tmpdir, "TEST-2020:0001") + os.makedirs(adv_dir) + + # Create invalid JSON file + with open(os.path.join(adv_dir, "advisory_cdn_metadata.json"), "w") as f: + f.write("{ invalid json }") + + # Create valid filelist (to get past the first file) + with open(os.path.join(adv_dir, "advisory_cdn_filelist.json"), "w") as f: + json.dump({}, f) + + with pytest.raises(ValueError) as exc_info: + source = KonfluxSource(url=tmpdir, advisories="TEST-2020:0001") + with source: + list(source) + + assert "Invalid JSON" in str(exc_info.value) + + +def test_context_manager(): + """Test context manager behavior.""" + source = KonfluxSource(url=DATADIR, advisories="RHSA-2020:0509") + + # Executor should be running + assert source._executor is not None + + with source: + items = list(source) + assert len(items) > 0 + + # After exit, executor should be shutdown + # (We can't easily test this without accessing private state)