diff --git a/.github/run-package-tests.sh b/.github/run-package-tests.sh index c2024bcd2aa8..4dea8623ec37 100644 --- a/.github/run-package-tests.sh +++ b/.github/run-package-tests.sh @@ -58,11 +58,18 @@ fi export COMPOSER=composer-local.json FAILED_FILE=$(mktemp -d)/failed -for DIR in ${DIRS}; do + +# Executes the package test logic for a single component directory. +# This function is responsible for copy/configuring local dependencies, +# updating Composer, and running PHPUnit tests (unit + optional snippet tests). +# It returns 0 on success, or 1 on failure. +run_package_test() { + local DIR=$1 echo "--- Processing ${DIR} ---" cp "${DIR}/composer.json" "${DIR}/composer-local.json" + # Update composer to use local packages - PACKAGE_DEPENDENCIES=( + local PACKAGE_DEPENDENCIES=( "Gax,gax" "CommonProtos,common-protos,4.100" "BigQuery,cloud-bigquery" @@ -78,6 +85,7 @@ for DIR in ${DIRS}; do IFS="," read -r PKG_DIR PKG_NAME PKG_VERSION <<< "$i" if grep -q "\"google/${PKG_NAME}\":" "${DIR}/composer.json"; then # determine local package version + local VERSION if [ "${STRICT}" = "true" ]; then VERSION=$(cat "${PKG_DIR}/VERSION") elif [ -z "${PKG_VERSION}" ]; then @@ -87,6 +95,7 @@ for DIR in ${DIRS}; do fi echo "Use local package ${PKG_DIR} as google/${PKG_NAME}:${VERSION} in ${DIR}" # "canonical: false" ensures composer will try to install from packagist when the "--prefer-lowest" flag is set. + local JSON_CONFIG JSON_CONFIG=$(printf '{"type":"path","url":"../%s","options":{"versions":{"google/%s":"%s"}},"canonical":false}' "${PKG_DIR}" "${PKG_NAME}" "${VERSION}") composer config "repositories.${PKG_NAME}" -d "${DIR}" "${JSON_CONFIG}" fi @@ -97,20 +106,60 @@ for DIR in ${DIRS}; do echo -n " (with ${PREFER_LOWEST})" fi echo "" - composer -q --no-interaction --no-ansi --no-progress ${PREFER_LOWEST} update -d "${DIR}" || { + if ! composer -q --no-interaction --no-ansi --no-progress ${PREFER_LOWEST} update -d "${DIR}"; then echo "${DIR}: composer install failed" >> "${FAILED_FILE}" # run again but without "-q" so we can see the error composer --no-interaction --no-ansi --no-progress ${PREFER_LOWEST} update -d "${DIR}" - continue - } + return 1 + fi + echo "Running ${DIR} Unit Tests" - "${DIR}/vendor/bin/phpunit" -c "${DIR}/phpunit.xml.dist" || echo "${DIR}: failed" >> "${FAILED_FILE}" + if ! "${DIR}/vendor/bin/phpunit" -c "${DIR}/phpunit.xml.dist"; then + echo "${DIR}: failed" >> "${FAILED_FILE}" + return 1 + fi if [ -f "${DIR}/phpunit-snippets.xml.dist" ]; then echo "Running ${DIR} Snippet Tests" - "${DIR}/vendor/bin/phpunit" -c "${DIR}/phpunit-snippets.xml.dist" || echo "${DIR} (snippets): failed" >> "${FAILED_FILE}" + if ! "${DIR}/vendor/bin/phpunit" -c "${DIR}/phpunit-snippets.xml.dist"; then + echo "${DIR} (snippets): failed" >> "${FAILED_FILE}" + return 1 + fi fi -done + return 0 +} + +# Wrapper function to run the package test logic for a single component. +# Buffers all stdout/stderr into a temporary log file to ensure that concurrent +# executions run completely hermetically and their log outputs are printed +# contiguously, rather than interleaved at the line level. +run_package_test_parallel() { + local DIR=$1 + local LOG_FILE + LOG_FILE=$(mktemp) + + # Run the logic, capturing all output + run_package_test "${DIR}" > "$LOG_FILE" 2>&1 + local EXIT_CODE=$? + + # Print the captured output atomically to standard streams + cat "$LOG_FILE" + rm "$LOG_FILE" + return $EXIT_CODE +} + +# Export functions and key env vars so they are available within subprocesses spawned by xargs +export -f run_package_test +export -f run_package_test_parallel +export STRICT +export PREFER_LOWEST +export FAILED_FILE + +# Determine optimal parallelism: default to the number of CPU cores on the host runner +MAX_JOBS=${MAX_JOBS:-$(nproc 2>/dev/null || echo 8)} + +# Run the test suites concurrently using xargs -P +printf "%s\n" ${DIRS} | xargs -P "${MAX_JOBS}" -I {} bash -c 'run_package_test_parallel "$@"' _ {} if [ -f "${FAILED_FILE}" ]; then echo "--------- Failed tests --------------" diff --git a/.kokoro/docs/publish.sh b/.kokoro/docs/publish.sh index 739690fe14c0..1f9d0824abaf 100755 --- a/.kokoro/docs/publish.sh +++ b/.kokoro/docs/publish.sh @@ -29,19 +29,68 @@ if [ "$GCLOUD_DEBUG" = "1" ]; then echo "Setting verbosity to VERBOSE..."; VERBOSITY_FLAG=" -v"; fi -find $PROJECT_DIR/* -mindepth 1 -maxdepth 1 -name 'composer.json' -not -path '*vendor/*' -regex "$PROJECT_DIR/[A-Z].*" -exec dirname {} \; | while read DIR -do - COMPONENT=$(basename $DIR) - VERSION=$(cat $DIR/VERSION) - $PROJECT_DIR/dev/google-cloud docfx \ - --component $COMPONENT \ - --out $DIR/out \ - --metadata-version $VERSION \ +run_docfx() { + local DIR=$1 + local COMPONENT + COMPONENT=$(basename "$DIR") + local VERSION + VERSION=$(cat "$DIR/VERSION") + + echo "--- Generating DocFX for ${COMPONENT} ---" + if ! "$PROJECT_DIR/dev/google-cloud" docfx \ + --component "$COMPONENT" \ + --out "$DIR/out" \ + --metadata-version "$VERSION" \ --with-cache \ $STAGING_FLAG \ - $VERBOSITY_FLAG + $VERBOSITY_FLAG; then + echo "Error: DocFX generation failed for ${COMPONENT}" + return 1 + fi + return 0 +} + +run_docfx_parallel() { + local DIR=$1 + local LOG_FILE + LOG_FILE=$(mktemp) + + run_docfx "${DIR}" > "$LOG_FILE" 2>&1 + local EXIT_CODE=$? + + cat "$LOG_FILE" + rm "$LOG_FILE" + return $EXIT_CODE +} +export -f run_docfx +export -f run_docfx_parallel +export PROJECT_DIR +export STAGING_FLAG +export VERBOSITY_FLAG + +# Get the list of directories +DIRS=$(find $PROJECT_DIR/* -mindepth 1 -maxdepth 1 -name 'composer.json' -not -path '*vendor/*' -regex "$PROJECT_DIR/[A-Z].*" -exec dirname {} \;) + +DIR_ARRAY=() +for DIR in ${DIRS}; do + if [ -d "${DIR}" ]; then + DIR_ARRAY+=("${DIR}") + fi done +# Warm the cache +echo "--- Warming DocFX Cache ---" +if ! "$PROJECT_DIR/dev/google-cloud" docfx --warm-cache; then + echo "Error: Initial DocFX cache warming failed. Aborting." >&2 + exit 1 +fi + +# Run all in parallel +if [ ${#DIR_ARRAY[@]} -gt 0 ]; then + MAX_JOBS=${MAX_JOBS:-$(nproc 2>/dev/null || echo 8)} + printf "%s\n" "${DIR_ARRAY[@]}" | xargs -P "${MAX_JOBS}" -I {} bash -c 'run_docfx_parallel "$@"' _ {} +fi + # Add Auth repo AUTH_DIR=$PROJECT_DIR/dev/vendor/google/auth $PROJECT_DIR/dev/google-cloud docfx \ diff --git a/dev/src/Command/DocFxCommand.php b/dev/src/Command/DocFxCommand.php index 5707a26b3209..9189a7744b59 100644 --- a/dev/src/Command/DocFxCommand.php +++ b/dev/src/Command/DocFxCommand.php @@ -76,11 +76,19 @@ protected function configure() ->addOption('path', '', InputOption::VALUE_OPTIONAL, 'Specify the path to the composer package to generate.') ->addOption('--with-cache', '', InputOption::VALUE_NONE, 'Cache expensive proto namespace lookups to a file') ->addOption('--generate-product-neutral-guides', '', InputOption::VALUE_NONE, 'Instead of a component, generate product-neutral guides.') + ->addOption('--warm-cache', '', InputOption::VALUE_NONE, 'Warm the cache by resolving proto packages to namespaces') ; } protected function execute(InputInterface $input, OutputInterface $output) { + if ($input->getOption('warm-cache')) { + $output->writeln('Warming cache... '); + $this->getProtoPackageToNamespaceMap(true); + $output->writeln('Done.'); + return 0; + } + // YAML dump configuration $inline = 11; // The level where you switch to inline YAML $indent = 2; // The amount of spaces to use for indentation of nested nodes diff --git a/dev/tests/Unit/Command/DocFxCommandTest.php b/dev/tests/Unit/Command/DocFxCommandTest.php index b7f6cccc9353..b06dc71d8047 100644 --- a/dev/tests/Unit/Command/DocFxCommandTest.php +++ b/dev/tests/Unit/Command/DocFxCommandTest.php @@ -245,6 +245,17 @@ public function testProductNeutralGuides(string $filepath) $this->assertFileEqualsWithDiff($left, $right, '1' === getenv('UPDATE_FIXTURES')); } + public function testWarmCache() + { + $exitCode = self::getCommandTester()->execute([ + '--warm-cache' => true, + ]); + + $this->assertEquals(0, $exitCode); + $this->assertStringContainsString('Warming cache... ', self::getCommandTester()->getDisplay()); + $this->assertStringContainsString('Done.', self::getCommandTester()->getDisplay()); + } + private function assertFileEqualsWithDiff(string $left, string $right, bool $updateFixtures = false) { if (file_get_contents($left) !== file_get_contents($right)) {