Add management command to analyze Kubernetes cluster (#1950)

Signed-off-by: tdruez <tdruez@aboutcode.org>

Author: tdruez

docs/command-line-interface.rst

@@ -56,6 +56,8 @@ ScanPipe's own commands are listed under the ``[scanpipe]`` section::
     [scanpipe]
       add-input
       add-pipeline
+      add-webhook
+      analyze-kubernetes
       archive-project
       batch-create
       check-compliance
@@ -391,6 +393,119 @@ Example usage:
    $ scanpipe add-webhook my_project https://example.com/webhook --inactive
 
 
+.. _cli_analyze_kubernetes:
+
+`$ scanpipe analyze-kubernetes <name>`
+--------------------------------------
+
+Analyzes all Docker images from a Kubernetes cluster by extracting image references
+using ``kubectl`` and creating projects to scan them.
+
+This command connects to your Kubernetes cluster, retrieves all container images
+(including init containers) from running pods, and creates projects to analyze each
+image for packages, dependencies, and optionally vulnerabilities.
+
+Required arguments:
+
+- ``name`` Project name or prefix for the created projects.
+
+Optional arguments:
+
+- ``--multi`` Create multiple projects (one per image) instead of a single project
+  containing all images. When used, each project is named ``<name>: <image-reference>``.
+
+- ``--find-vulnerabilities`` Run the ``find_vulnerabilities`` pipeline during the
+  analysis to detect known security vulnerabilities in discovered packages.
+
+- ``--execute`` Execute the pipelines right after project creation.
+
+- ``--async`` Add the pipeline run to the tasks queue for execution by a worker instead
+  of running in the current thread.
+  Applies only when ``--execute`` is provided.
+
+- ``--namespace NAMESPACE`` Limit the image extraction to a specific Kubernetes
+  namespace. If not provided, images from all namespaces are collected.
+
+- ``--context CONTEXT`` Use a specific Kubernetes context. If not provided, the
+  current context is used.
+
+- ``--notes NOTES`` Optional notes about the project(s).
+
+- ``--label LABELS`` Optional labels for the project(s). Multiple labels can be
+  provided by using this argument multiple times.
+
+- ``--dry-run`` Do not create any projects; just print the images and projects that
+  would be created.
+
+- ``--no-global-webhook`` Skip the creation of the global webhook. This option is
+  only useful if a global webhook is defined in the settings.
+
+.. note::
+    This command requires ``kubectl`` to be installed and configured with access to
+    your Kubernetes cluster.
+
+Example: Analyze All Cluster Images
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To analyze all images from all namespaces in your current Kubernetes cluster::
+
+    $ scanpipe analyze-kubernetes cluster-audit --multi --execute
+
+This creates separate projects for each unique image found in the cluster.
+
+Example: Analyze Production Namespace with Vulnerability Scanning
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To scan all images in the ``production`` namespace and check for vulnerabilities::
+
+    $ scanpipe analyze-kubernetes prod-security-scan \
+        --namespace production \
+        --find-vulnerabilities \
+        --multi \
+        --label "production" \
+        --label "security-audit" \
+        --execute
+
+Example: Dry Run Before Creating Projects
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To preview which images would be analyzed without creating any projects::
+
+    $ scanpipe analyze-kubernetes cluster-preview \
+        --namespace default \
+        --dry-run
+
+This displays all images that would be scanned, allowing you to verify the scope
+before running the actual analysis.
+
+Example: Analyze Specific Cluster Context
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To analyze images from a specific Kubernetes cluster when you have multiple contexts
+configured::
+
+    $ scanpipe analyze-kubernetes staging-audit \
+        --context staging-cluster \
+        --namespace default \
+        --multi \
+        --execute --async
+
+Example: Single Project for All Images
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To create one project containing all images from the cluster::
+
+    $ scanpipe analyze-kubernetes full-cluster-scan \
+        --find-vulnerabilities \
+        --execute
+
+This creates a single project named ``full-cluster-scan`` that analyzes all discovered
+images together.
+
+.. tip::
+    Use ``--multi`` when analyzing large clusters to create separate projects per image,
+    making it easier to track and review results for individual container images.
+
 `$ scanpipe execute --project PROJECT`
 --------------------------------------
 

scanpipe/management/commands/analyze-kubernetes.py

@@ -0,0 +1,146 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
+
+import sys
+
+from django.core.management.base import BaseCommand
+from django.core.management.base import CommandError
+from django.utils.text import slugify
+
+from scanpipe.management.commands import CreateProjectCommandMixin
+from scanpipe.management.commands import execute_project
+from scanpipe.pipes.kubernetes import get_images_from_kubectl
+
+
+class Command(CreateProjectCommandMixin, BaseCommand):
+    help = "Analyze all images of a Kubernetes cluster."
+
+    def add_arguments(self, parser):
+        super().add_arguments(parser)
+        parser.add_argument("name", help="Project name.")
+        parser.add_argument(
+            "--multi",
+            action="store_true",
+            help="Create multiple projects instead of a single one.",
+        )
+        parser.add_argument(
+            "--find-vulnerabilities",
+            action="store_true",
+            help="Run the find_vulnerabilities pipeline during the analysis.",
+        )
+        parser.add_argument(
+            "--execute",
+            action="store_true",
+            help="Execute the pipelines right after the project creation.",
+        )
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            help=(
+                "Do not create any projects."
+                "Print the images and projects that would be created."
+            ),
+        )
+        # Additional kubectl options
+        parser.add_argument(
+            "--namespace",
+            type=str,
+            help="Kubernetes namespace to query (for --kubectl mode).",
+        )
+        parser.add_argument(
+            "--context",
+            type=str,
+            help="Kubernetes context to use (for --kubectl mode).",
+        )
+
+    def handle(self, *args, **options):
+        self.verbosity = options["verbosity"]
+        project_name = options["name"]
+        pipelines = ["analyze_docker_image"]
+        create_multiple_projects = options["multi"]
+        execute = options["execute"]
+        run_async = options["async"]
+        labels = options["labels"]
+        notes = options["notes"]
+        created_projects = []
+
+        if options["find_vulnerabilities"]:
+            pipelines.append("find_vulnerabilities")
+
+        images = self.get_images(**options)
+        if not images:
+            raise CommandError("No images found.")
+
+        create_project_options = {
+            "pipelines": pipelines,
+            "notes": notes,
+            "labels": labels,
+        }
+
+        if create_multiple_projects:
+            labels.append(f"k8s-{slugify(project_name)}")
+            for reference in images:
+                project = self.create_project(
+                    **create_project_options,
+                    name=f"{project_name}: {reference}",
+                    input_urls=[f"docker://{reference}"],
+                )
+                created_projects.append(project)
+
+        else:
+            project = self.create_project(
+                **create_project_options,
+                name=project_name,
+                input_urls=[f"docker://{reference}" for reference in images],
+            )
+            created_projects.append(project)
+
+        if execute:
+            for project in created_projects:
+                execute_project(project=project, run_async=run_async, command=self)
+
+    def get_images(self, **options):
+        namespace = options.get("namespace")
+        context = options.get("context")
+        dry_run = options.get("dry_run")
+
+        if self.verbosity >= 1:
+            self.stdout.write(
+                "Extracting images from Kubernetes cluster using kubectl..."
+            )
+
+        try:
+            images = get_images_from_kubectl(namespace=namespace, context=context)
+        except Exception as e:
+            raise CommandError(e)
+
+        if self.verbosity >= 1 or dry_run:
+            self.stdout.write(
+                self.style.SUCCESS(f"Found {len(images)} images in the cluster:"),
+            )
+            self.stdout.write("\n".join(images))
+
+        if dry_run:
+            self.stdout.write("Dry run mode, no projects were created.")
+            sys.exit(0)
+
+        return images

scanpipe/models.py

@@ -1533,6 +1533,11 @@ def has_single_resource(self):
         """
         return self.resource_count == 1
 
+    @property
+    def pipelines(self):
+        """Return the list of pipeline names assigned to this Project."""
+        return list(self.runs.values_list("pipeline_name", flat=True))
+
     def get_policies_dict(self):
         """
         Load and return the policies from the following locations in that order:

scanpipe/pipes/__init__.py

@@ -22,6 +22,7 @@
 
 import difflib
 import logging
+import subprocess
 import sys
 import time
 import uuid
@@ -574,3 +575,40 @@ def poll_until_success(check, sleep=10, **kwargs):
             return False
 
         time.sleep(sleep)
+
+
+def run_command_safely(command_args):
+    """
+    Execute the external commands following security best practices.
+
+    This function is using the subprocess.run function which simplifies running external
+    commands. It provides a safer and more straightforward API compared to older methods
+    like subprocess.Popen.
+
+    WARNING: Please note that the `--option=value` syntax is required for args entries,
+    and not the `--option value` format.
+
+    - This does not use the Shell (shell=False) to prevent injection vulnerabilities.
+    - The command should be provided as a list of ``command_args`` arguments.
+    - Only full paths to executable commands should be provided to avoid any ambiguity.
+
+    WARNING: If you're incorporating user input into the command, make
+    sure to sanitize and validate the input to prevent any malicious commands from
+    being executed.
+
+    Raise a SubprocessError if the exit code was non-zero.
+    """
+    completed_process = subprocess.run(  # noqa: S603
+        command_args,
+        capture_output=True,
+        text=True,
+    )
+
+    if completed_process.returncode:
+        error_msg = (
+            f'Error while executing cmd="{completed_process.args}": '
+            f'"{completed_process.stderr.strip()}"'
+        )
+        raise subprocess.SubprocessError(error_msg)
+
+    return completed_process.stdout

scanpipe/pipes/fetch.py

@@ -24,7 +24,6 @@
 import logging
 import os
 import re
-import subprocess
 import tempfile
 from collections import namedtuple
 from pathlib import Path
@@ -44,6 +43,8 @@
 from plugincode.location_provider import get_location
 from requests import auth as request_auth
 
+from scanpipe.pipes import run_command_safely
+
 logger = logging.getLogger("scanpipe.pipes")
 
 Download = namedtuple("Download", "uri directory filename path size sha1 md5")
@@ -60,43 +61,6 @@
 HTTP_REQUEST_TIMEOUT = 30
 
 
-def run_command_safely(command_args):
-    """
-    Execute the external commands following security best practices.
-
-    This function is using the subprocess.run function which simplifies running external
-    commands. It provides a safer and more straightforward API compared to older methods
-    like subprocess.Popen.
-
-    WARNING: Please note that the `--option=value` syntax is required for args entries,
-    and not the `--option value` format.
-
-    - This does not use the Shell (shell=False) to prevent injection vulnerabilities.
-    - The command should be provided as a list of ``command_args`` arguments.
-    - Only full paths to executable commands should be provided to avoid any ambiguity.
-
-    WARNING: If you're incorporating user input into the command, make
-    sure to sanitize and validate the input to prevent any malicious commands from
-    being executed.
-
-    Raise a SubprocessError if the exit code was non-zero.
-    """
-    completed_process = subprocess.run(  # noqa: S603
-        command_args,
-        capture_output=True,
-        text=True,
-    )
-
-    if completed_process.returncode:
-        error_msg = (
-            f'Error while executing cmd="{completed_process.args}": '
-            f'"{completed_process.stderr.strip()}"'
-        )
-        raise subprocess.SubprocessError(error_msg)
-
-    return completed_process.stdout
-
-
 def get_request_session(uri):
     """Return a Requests session setup with authentication and headers."""
     session = requests.Session()