feat(ci): parallelize docs and packge test jobs

.github/run-package-tests.sh

@@ -58,11 +58,18 @@ fi
 export COMPOSER=composer-local.json
 
 FAILED_FILE=$(mktemp -d)/failed
-for DIR in ${DIRS}; do
+
+# Executes the package test logic for a single component directory.
+# This function is responsible for copy/configuring local dependencies,
+# updating Composer, and running PHPUnit tests (unit + optional snippet tests).
+# It returns 0 on success, or 1 on failure.
+run_package_test() {
+    local DIR=$1
     echo "--- Processing ${DIR} ---"
     cp "${DIR}/composer.json" "${DIR}/composer-local.json"
+
     # Update composer to use local packages
-    PACKAGE_DEPENDENCIES=(
+    local PACKAGE_DEPENDENCIES=(
         "Gax,gax"
         "CommonProtos,common-protos,4.100"
         "BigQuery,cloud-bigquery"
@@ -78,6 +85,7 @@ for DIR in ${DIRS}; do
         IFS="," read -r PKG_DIR PKG_NAME PKG_VERSION <<< "$i"
         if grep -q "\"google/${PKG_NAME}\":" "${DIR}/composer.json"; then
             # determine local package version
+            local VERSION
             if [ "${STRICT}" = "true" ]; then
                 VERSION=$(cat "${PKG_DIR}/VERSION")
             elif [ -z "${PKG_VERSION}" ]; then
@@ -87,6 +95,7 @@ for DIR in ${DIRS}; do
             fi
             echo "Use local package ${PKG_DIR} as google/${PKG_NAME}:${VERSION} in ${DIR}"
             # "canonical: false" ensures composer will try to install from packagist when the "--prefer-lowest" flag is set.
+            local JSON_CONFIG
             JSON_CONFIG=$(printf '{"type":"path","url":"../%s","options":{"versions":{"google/%s":"%s"}},"canonical":false}' "${PKG_DIR}" "${PKG_NAME}" "${VERSION}")
             composer config "repositories.${PKG_NAME}" -d "${DIR}" "${JSON_CONFIG}"
         fi
@@ -97,20 +106,60 @@ for DIR in ${DIRS}; do
         echo -n " (with ${PREFER_LOWEST})"
     fi
     echo ""
-    composer -q --no-interaction --no-ansi --no-progress ${PREFER_LOWEST} update -d "${DIR}" || {
+    if ! composer -q --no-interaction --no-ansi --no-progress ${PREFER_LOWEST} update -d "${DIR}"; then
         echo "${DIR}: composer install failed" >> "${FAILED_FILE}"
         # run again but without "-q" so we can see the error
         composer --no-interaction --no-ansi --no-progress ${PREFER_LOWEST} update -d "${DIR}"
-        continue
-    }
+        return 1
+    fi
+
     echo "Running ${DIR} Unit Tests"
-    "${DIR}/vendor/bin/phpunit" -c "${DIR}/phpunit.xml.dist" || echo "${DIR}: failed" >> "${FAILED_FILE}"
+    if ! "${DIR}/vendor/bin/phpunit" -c "${DIR}/phpunit.xml.dist"; then
+        echo "${DIR}: failed" >> "${FAILED_FILE}"
+        return 1
+    fi
 
     if [ -f "${DIR}/phpunit-snippets.xml.dist" ]; then
         echo "Running ${DIR} Snippet Tests"
-        "${DIR}/vendor/bin/phpunit" -c "${DIR}/phpunit-snippets.xml.dist" || echo "${DIR} (snippets): failed" >> "${FAILED_FILE}"
+        if ! "${DIR}/vendor/bin/phpunit" -c "${DIR}/phpunit-snippets.xml.dist"; then
+            echo "${DIR} (snippets): failed" >> "${FAILED_FILE}"
+            return 1
+        fi
     fi
-done
+    return 0
+}
+
+# Wrapper function to run the package test logic for a single component.
+# Buffers all stdout/stderr into a temporary log file to ensure that concurrent
+# executions run completely hermetically and their log outputs are printed
+# contiguously, rather than interleaved at the line level.
+run_package_test_parallel() {
+    local DIR=$1
+    local LOG_FILE
+    LOG_FILE=$(mktemp)
+
+    # Run the logic, capturing all output
+    run_package_test "${DIR}" > "$LOG_FILE" 2>&1
+    local EXIT_CODE=$?
+
+    # Print the captured output atomically to standard streams
+    cat "$LOG_FILE"
+    rm "$LOG_FILE"
+    return $EXIT_CODE
+}
+
+# Export functions and key env vars so they are available within subprocesses spawned by xargs
+export -f run_package_test
+export -f run_package_test_parallel
+export STRICT
+export PREFER_LOWEST
+export FAILED_FILE
+
+# Determine optimal parallelism: default to the number of CPU cores on the host runner
+MAX_JOBS=${MAX_JOBS:-$(nproc 2>/dev/null || echo 8)}
+
+# Run the test suites concurrently using xargs -P
+printf "%s\n" ${DIRS} | xargs -P "${MAX_JOBS}" -I {} bash -c 'run_package_test_parallel "$@"' _ {}
 
 if [ -f "${FAILED_FILE}" ]; then
     echo "--------- Failed tests --------------"

.kokoro/docs/publish.sh

@@ -29,19 +29,68 @@ if [ "$GCLOUD_DEBUG" = "1" ]; then
     echo "Setting verbosity to VERBOSE...";
     VERBOSITY_FLAG=" -v";
 fi
-find $PROJECT_DIR/* -mindepth 1 -maxdepth 1 -name 'composer.json' -not -path '*vendor/*' -regex "$PROJECT_DIR/[A-Z].*" -exec dirname {} \; | while read DIR
-do
-    COMPONENT=$(basename $DIR)
-    VERSION=$(cat $DIR/VERSION)
-    $PROJECT_DIR/dev/google-cloud docfx \
-        --component $COMPONENT \
-        --out $DIR/out \
-        --metadata-version $VERSION \
+run_docfx() {
+    local DIR=$1
+    local COMPONENT
+    COMPONENT=$(basename "$DIR")
+    local VERSION
+    VERSION=$(cat "$DIR/VERSION")
+
+    echo "--- Generating DocFX for ${COMPONENT} ---"
+    if ! "$PROJECT_DIR/dev/google-cloud" docfx \
+        --component "$COMPONENT" \
+        --out "$DIR/out" \
+        --metadata-version "$VERSION" \
         --with-cache \
         $STAGING_FLAG \
-        $VERBOSITY_FLAG
+        $VERBOSITY_FLAG; then
+        echo "Error: DocFX generation failed for ${COMPONENT}"
+        return 1
+    fi
+    return 0
+}
+
+run_docfx_parallel() {
+    local DIR=$1
+    local LOG_FILE
+    LOG_FILE=$(mktemp)
+
+    run_docfx "${DIR}" > "$LOG_FILE" 2>&1
+    local EXIT_CODE=$?
+
+    cat "$LOG_FILE"
+    rm "$LOG_FILE"
+    return $EXIT_CODE
+}
+export -f run_docfx
+export -f run_docfx_parallel
+export PROJECT_DIR
+export STAGING_FLAG
+export VERBOSITY_FLAG
+
+# Get the list of directories
+DIRS=$(find $PROJECT_DIR/* -mindepth 1 -maxdepth 1 -name 'composer.json' -not -path '*vendor/*' -regex "$PROJECT_DIR/[A-Z].*" -exec dirname {} \;)
+
+DIR_ARRAY=()
+for DIR in ${DIRS}; do
+    if [ -d "${DIR}" ]; then
+        DIR_ARRAY+=("${DIR}")
+    fi
 done
 
+# Warm the cache
+echo "--- Warming DocFX Cache ---"
+if ! "$PROJECT_DIR/dev/google-cloud" docfx --warm-cache; then
+    echo "Error: Initial DocFX cache warming failed. Aborting." >&2
+    exit 1
+fi
+
+# Run all in parallel
+if [ ${#DIR_ARRAY[@]} -gt 0 ]; then
+    MAX_JOBS=${MAX_JOBS:-$(nproc 2>/dev/null || echo 8)}
+    printf "%s\n" "${DIR_ARRAY[@]}" | xargs -P "${MAX_JOBS}" -I {} bash -c 'run_docfx_parallel "$@"' _ {}
+fi
+
 # Add Auth repo
 AUTH_DIR=$PROJECT_DIR/dev/vendor/google/auth
 $PROJECT_DIR/dev/google-cloud docfx \

dev/src/Command/DocFxCommand.php

@@ -76,11 +76,19 @@ protected function configure()
             ->addOption('path', '', InputOption::VALUE_OPTIONAL, 'Specify the path to the composer package to generate.')
             ->addOption('--with-cache', '', InputOption::VALUE_NONE, 'Cache expensive proto namespace lookups to a file')
             ->addOption('--generate-product-neutral-guides', '', InputOption::VALUE_NONE, 'Instead of a component, generate product-neutral guides.')
+            ->addOption('--warm-cache', '', InputOption::VALUE_NONE, 'Warm the cache by resolving proto packages to namespaces')
         ;
     }
 
     protected function execute(InputInterface $input, OutputInterface $output)
     {
+        if ($input->getOption('warm-cache')) {
+            $output->writeln('Warming cache... ');
+            $this->getProtoPackageToNamespaceMap(true);
+            $output->writeln('Done.');
+            return 0;
+        }
+
         // YAML dump configuration
         $inline = 11; // The level where you switch to inline YAML
         $indent = 2; // The amount of spaces to use for indentation of nested nodes

dev/tests/Unit/Command/DocFxCommandTest.php

@@ -245,6 +245,17 @@ public function testProductNeutralGuides(string $filepath)
         $this->assertFileEqualsWithDiff($left, $right, '1' === getenv('UPDATE_FIXTURES'));
     }
 
+    public function testWarmCache()
+    {
+        $exitCode = self::getCommandTester()->execute([
+            '--warm-cache' => true,
+        ]);
+
+        $this->assertEquals(0, $exitCode);
+        $this->assertStringContainsString('Warming cache... ', self::getCommandTester()->getDisplay());
+        $this->assertStringContainsString('Done.', self::getCommandTester()->getDisplay());
+    }
+
     private function assertFileEqualsWithDiff(string $left, string $right, bool $updateFixtures = false)
     {
         if (file_get_contents($left) !== file_get_contents($right)) {