diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 7d87ab4..892826e 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -26,22 +26,8 @@ concurrency: jobs: seed-build-script: - runs-on: ubuntu-latest name: Pre-seed static binaries' versions - steps: - - env: - GITHUB_TOKEN: ${{ github.token }} - run: >- - gh release download - -R cpp-linter/clang-tools-static-binaries - --pattern versions.json - --output versions.json - - name: Upload versions.json - uses: actions/upload-artifact@v7 - with: - name: static-binary-versions - path: versions.json - if-no-files-found: error + uses: ./.github/workflows/pre-seed-versions.yml build-bin: needs: [seed-build-script] @@ -83,7 +69,7 @@ jobs: if: steps.cache.outputs.cache-hit != 'true' || steps.validate.outputs.cache-valid == 'false' uses: actions/download-artifact@v8 with: - name: static-binary-versions + name: ${{ needs.seed-build-script.outputs.artifact-name }} path: clang-tools-manager - run: cargo build --bin cpp-linter --features bin --release if: steps.cache.outputs.cache-hit != 'true' || steps.validate.outputs.cache-valid == 'false' diff --git a/.github/workflows/binary-builds.yml b/.github/workflows/binary-builds.yml index 3613763..c937039 100644 --- a/.github/workflows/binary-builds.yml +++ b/.github/workflows/binary-builds.yml @@ -35,22 +35,8 @@ concurrency: jobs: seed-build-script: - runs-on: ubuntu-latest name: Pre-seed static binaries' versions - steps: - - env: - GITHUB_TOKEN: ${{ github.token }} - run: >- - gh release download - -R cpp-linter/clang-tools-static-binaries - --pattern versions.json - --output versions.json - - name: Upload versions.json - uses: actions/upload-artifact@v7 - with: - name: static-binary-versions - path: versions.json - if-no-files-found: error + uses: ./.github/workflows/pre-seed-versions.yml create-assets: needs: [seed-build-script] @@ -135,7 +121,7 @@ jobs: - name: Restore build script seed uses: actions/download-artifact@v8 with: - name: static-binary-versions + name: ${{ needs.seed-build-script.outputs.artifact-name }} path: clang-tools-manager - name: Build diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index e64b5ee..cc5a569 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -45,22 +45,8 @@ jobs: - run: cargo fetch seed-build-script: - runs-on: ubuntu-latest name: Pre-seed static binaries' versions - steps: - - env: - GITHUB_TOKEN: ${{ github.token }} - run: >- - gh release download - -R cpp-linter/clang-tools-static-binaries - --pattern versions.json - --output versions.json - - name: Upload versions.json - uses: actions/upload-artifact@v7 - with: - name: static-binary-versions - path: versions.json - if-no-files-found: error + uses: ./.github/workflows/pre-seed-versions.yml build-mkdocs: runs-on: ubuntu-latest @@ -87,7 +73,7 @@ jobs: - name: Restore build script seed uses: actions/download-artifact@v8 with: - name: static-binary-versions + name: ${{ needs.seed-build-script.outputs.artifact-name }} path: clang-tools-manager - name: Build docs run: nur docs --build @@ -123,7 +109,7 @@ jobs: - name: Restore build script seed uses: actions/download-artifact@v8 with: - name: static-binary-versions + name: ${{ needs.seed-build-script.outputs.artifact-name }} path: clang-tools-manager - run: nur docs rs - name: upload rustdoc build as artifact diff --git a/.github/workflows/node-js-packaging.yml b/.github/workflows/node-js-packaging.yml index 317573d..8363935 100644 --- a/.github/workflows/node-js-packaging.yml +++ b/.github/workflows/node-js-packaging.yml @@ -39,22 +39,8 @@ concurrency: jobs: seed-build-script: - runs-on: ubuntu-latest name: Pre-seed static binaries' versions - steps: - - env: - GITHUB_TOKEN: ${{ github.token }} - run: >- - gh release download - -R cpp-linter/clang-tools-static-binaries - --pattern versions.json - --output versions.json - - name: Upload versions.json - uses: actions/upload-artifact@v7 - with: - name: static-binary-versions - path: versions.json - if-no-files-found: error + uses: ./.github/workflows/pre-seed-versions.yml build: needs: [seed-build-script] @@ -143,7 +129,7 @@ jobs: - name: Restore build script seed uses: actions/download-artifact@v8 with: - name: static-binary-versions + name: ${{ needs.seed-build-script.outputs.artifact-name }} path: clang-tools-manager - name: Build run: ${{ matrix.settings.build }} @@ -178,7 +164,7 @@ jobs: - name: Restore build script seed uses: actions/download-artifact@v8 with: - name: static-binary-versions + name: ${{ needs.seed-build-script.outputs.artifact-name }} path: clang-tools-manager - name: Build shell: cpa.sh {0} diff --git a/.github/workflows/pre-seed-versions.yml b/.github/workflows/pre-seed-versions.yml new file mode 100644 index 0000000..6fb1104 --- /dev/null +++ b/.github/workflows/pre-seed-versions.yml @@ -0,0 +1,29 @@ +name: Pre-seed versions + +on: + workflow_call: + outputs: + artifact-name: + description: >- + The name of uploaded artifact containing the versions.json file. + value: static-binary-versions + +permissions: {} + +jobs: + pre-seed-versions: + runs-on: ubuntu-latest + steps: + - env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + gh release download + -R cpp-linter/clang-tools-static-binaries + --pattern versions.json + --output versions.json + - name: Upload versions.json + uses: actions/upload-artifact@v7 + with: + name: static-binary-versions + path: versions.json + if-no-files-found: error diff --git a/.github/workflows/python-packaging.yml b/.github/workflows/python-packaging.yml index 2e9b429..32c6e9e 100644 --- a/.github/workflows/python-packaging.yml +++ b/.github/workflows/python-packaging.yml @@ -44,22 +44,8 @@ concurrency: jobs: seed-build-script: - runs-on: ubuntu-latest name: Pre-seed static binaries' versions - steps: - - env: - GITHUB_TOKEN: ${{ github.token }} - run: >- - gh release download - -R cpp-linter/clang-tools-static-binaries - --pattern versions.json - --output versions.json - - name: Upload versions.json - uses: actions/upload-artifact@v7 - with: - name: static-binary-versions - path: versions.json - if-no-files-found: error + uses: ./.github/workflows/pre-seed-versions.yml linux: needs: [seed-build-script] @@ -93,31 +79,18 @@ jobs: - name: Restore build script seed uses: actions/download-artifact@v8 with: - name: static-binary-versions + name: ${{ needs.seed-build-script.outputs.artifact-name }} path: clang-tools-manager - name: Build wheels uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 with: target: ${{ matrix.platform.target }} - # Use the zig toolchain for all targets except ppc64le. - # Note, this is meant to avoid build errors in aws-lc-sys crate, which compiles C code. + # Use the zig toolchain to avoid build errors in aws-lc-sys crate, which compiles C code. args: >- --release --out dist --find-interpreter --zig # avoid cache-poisoning potential for actually distributed builds sccache: ${{ startsWith(github.ref, 'refs/tags') && 'false' || 'true' }} manylinux: 'auto' - # before-script-linux: | - # case "${{ matrix.platform.target }}" in - # ppc64le) - # apt-get update - # apt-get install -y \ - # pkg-config \ - # gcc-powerpc64le-linux-gnu \ - # g++-powerpc64le-linux-gnu \ - # binutils-powerpc64le-linux-gnu \ - # libc6-dev-ppc64el-cross - # ;; - # esac - name: Upload wheels uses: actions/upload-artifact@v7 with: @@ -145,7 +118,7 @@ jobs: - name: Restore build script seed uses: actions/download-artifact@v8 with: - name: static-binary-versions + name: ${{ needs.seed-build-script.outputs.artifact-name }} path: clang-tools-manager - name: Build wheels uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 @@ -180,7 +153,7 @@ jobs: - name: Restore build script seed uses: actions/download-artifact@v8 with: - name: static-binary-versions + name: ${{ needs.seed-build-script.outputs.artifact-name }} path: clang-tools-manager - name: Build wheels uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 diff --git a/.github/workflows/run-dev-tests.yml b/.github/workflows/run-dev-tests.yml index 39cf838..7ad898e 100644 --- a/.github/workflows/run-dev-tests.yml +++ b/.github/workflows/run-dev-tests.yml @@ -30,22 +30,8 @@ concurrency: jobs: seed-build-script: - runs-on: ubuntu-latest name: Pre-seed static binaries' versions - steps: - - env: - GITHUB_TOKEN: ${{ github.token }} - run: >- - gh release download - -R cpp-linter/clang-tools-static-binaries - --pattern versions.json - --output versions.json - - name: Upload versions.json - uses: actions/upload-artifact@v7 - with: - name: static-binary-versions - path: versions.json - if-no-files-found: error + uses: ./.github/workflows/pre-seed-versions.yml test: needs: [seed-build-script] @@ -120,7 +106,7 @@ jobs: - name: Restore build script seed uses: actions/download-artifact@v8 with: - name: static-binary-versions + name: ${{ needs.seed-build-script.outputs.artifact-name }} path: clang-tools-manager - name: Run test suite diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index bdfb959..ba2ce27 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -1,7 +1,8 @@ name: 'Close stale issues' on: schedule: - - cron: '30 1 * * *' + # Run at 1:30 AM every Monday, Wednesday, and Friday + - cron: '30 1 * * 1,3,5' permissions: {} diff --git a/clang-tools-manager/src/downloader/hashing.rs b/clang-tools-manager/src/downloader/hashing.rs index 5e68426..807081e 100644 --- a/clang-tools-manager/src/downloader/hashing.rs +++ b/clang-tools-manager/src/downloader/hashing.rs @@ -16,9 +16,20 @@ use std::{fs, io::Read, num::NonZero, path::Path}; pub enum HashAlgorithm { /// SHA-256 hash algorithm with the expected checksum value. Sha256(String), + /// BLAKE2b-256 hash algorithm with the expected checksum value. Blake2b256(String), + /// SHA-512 hash algorithm with the expected checksum value. + #[cfg(any( + // Windows support is only for x86_64 architecture (for now) + all(target_os = "windows", target_arch = "x86_64"), + // Linux and macOS support only x86_64 and aarch64 architectures + all( + any(target_os = "linux", target_os = "macos"), + any(target_arch = "x86_64", target_arch = "aarch64") + ), + ))] Sha512(String), } @@ -74,6 +85,15 @@ impl HashAlgorithm { let hasher = Blake2b::::new(); Self::hash_file(hasher, file_path, expected) } + #[cfg(any( + // Windows support is only for x86_64 architecture (for now) + all(target_os = "windows", target_arch = "x86_64"), + // Linux and macOS support only x86_64 and aarch64 architectures + all( + any(target_os = "linux", target_os = "macos"), + any(target_arch = "x86_64", target_arch = "aarch64") + ), + ))] HashAlgorithm::Sha512(expected) => { use sha2::{Digest, Sha512}; diff --git a/clang-tools-manager/src/downloader/static_dist.rs b/clang-tools-manager/src/downloader/static_dist.rs index 256aa62..ceb9b69 100644 --- a/clang-tools-manager/src/downloader/static_dist.rs +++ b/clang-tools-manager/src/downloader/static_dist.rs @@ -2,19 +2,10 @@ use std::{ env::consts::{ARCH, OS}, - fs, ops::RangeInclusive, - path::{Path, PathBuf}, }; -use semver::{Version, VersionReq}; -use url::Url; - -use crate::{ - Cacher, ClangTool, DownloadError, - downloader::{download, hashing::HashAlgorithm}, - utils::lock_path, -}; +use crate::{Cacher, DownloadError}; /// An error that can occur while downloading a static binary. #[derive(Debug, thiserror::Error)] @@ -45,10 +36,9 @@ pub enum StaticDistDownloadError { #[error("Failed to parse the SHA512 sum file")] Sha512Corruption, } + const MIN_CLANG_TOOLS_VERSION: &str = env!("MIN_CLANG_TOOLS_VERSION"); pub(crate) const MAX_CLANG_TOOLS_VERSION: &str = env!("MAX_CLANG_TOOLS_VERSION"); -const CLANG_TOOLS_REPO: &str = "https://github.com/cpp-linter/clang-tools-static-binaries"; -const CLANG_TOOLS_TAG: &str = env!("CLANG_TOOLS_TAG"); /// A downloader that uses statically linked binary distribution files /// provided by the cpp-linter team. @@ -61,131 +51,182 @@ impl StaticDistDownloader { MIN_CLANG_TOOLS_VERSION.parse().unwrap_or(11) ..=MAX_CLANG_TOOLS_VERSION.parse().unwrap_or(22) } +} - /// Finds a suitable version from `req_ver` within the range of available clang tools versions. - /// - /// The available versions are determined by the `MIN_CLANG_TOOLS_VERSION` and - /// `MAX_CLANG_TOOLS_VERSION` environment variables (inclusive) at compile time. - fn find_suitable_version(req_ver: &VersionReq) -> Option { - let clang_tools_versions: RangeInclusive = Self::get_major_version_range(); - clang_tools_versions - .map(|v| Version::new(v as u64, 0, 0)) - .rev() - .find(|ver| req_ver.matches(ver)) +#[cfg(any( + // Windows support is only for x86_64 architecture (for now) + all(target_os = "windows", not(target_arch = "x86_64")), + // Linux and macOS support only x86_64 and aarch64 architectures + all( + any(target_os = "linux", target_os = "macos"), + not(any(target_arch = "x86_64", target_arch = "aarch64")) + ), + // Any OS other than Windows, Linux, or macOS is unsupported + not(any(target_os = "windows", target_os = "linux", target_os = "macos")), +))] +mod unsupported_platform { + use super::{StaticDistDownloadError, StaticDistDownloader}; + use crate::ClangTool; + use semver::VersionReq; + use std::path::PathBuf; + + impl StaticDistDownloader { + pub async fn download_tool( + _tool: &ClangTool, + _requested_version: &VersionReq, + _directory: Option<&PathBuf>, + ) -> Result { + Err(StaticDistDownloadError::UnsupportedArchitecture) + } } +} - /// Verifies the SHA512 checksum of the downloaded file. - /// - /// The expected checksum is extracted from another downloaded `*.sha512sum` file - /// (pointed to by `sha512_path`). - fn verify_sha512(file_path: &Path, sha512_path: &Path) -> Result<(), StaticDistDownloadError> { - let checksum_file_content = fs::read_to_string(sha512_path)?; - let expected = checksum_file_content - .split(' ') - .next() - .ok_or(StaticDistDownloadError::Sha512Corruption)?; - HashAlgorithm::Sha512(expected.to_string()).verify(file_path)?; - Ok(()) - } +#[cfg(any( + // Windows support is only for x86_64 architecture (for now) + all(target_os = "windows", target_arch = "x86_64"), + // Linux and macOS support only x86_64 and aarch64 architectures + all( + any(target_os = "linux", target_os = "macos"), + any(target_arch = "x86_64", target_arch = "aarch64") + ), +))] +mod supported_platform { + use std::{ + fs, + ops::RangeInclusive, + path::{Path, PathBuf}, + }; + + use semver::{Version, VersionReq}; + use url::Url; + + use super::{StaticDistDownloadError, StaticDistDownloader}; + use crate::{ + Cacher, ClangTool, + downloader::{download, hashing::HashAlgorithm}, + utils::lock_path, + }; + + const CLANG_TOOLS_REPO: &str = "https://github.com/cpp-linter/clang-tools-static-binaries"; + const CLANG_TOOLS_TAG: &str = env!("CLANG_TOOLS_TAG"); + + impl StaticDistDownloader { + /// Finds a suitable version from `req_ver` within the range of available clang tools versions. + /// + /// The available versions are determined by the `MIN_CLANG_TOOLS_VERSION` and + /// `MAX_CLANG_TOOLS_VERSION` environment variables (inclusive) at compile time. + fn find_suitable_version(req_ver: &VersionReq) -> Option { + let clang_tools_versions: RangeInclusive = Self::get_major_version_range(); + clang_tools_versions + .map(|v| Version::new(v as u64, 0, 0)) + .rev() + .find(|ver| req_ver.matches(ver)) + } - /// Downloads the `requested_version` of the specified `tool` from a distribution of statically linked binaries. - /// - /// The distribution is maintained at . - /// Supported platforms includes Windows, Linux, and MacOS. - /// Supported architectures is limited to `x86_64` (`amd64`). - pub async fn download_tool( - tool: &ClangTool, - requested_version: &VersionReq, - directory: Option<&PathBuf>, - ) -> Result { - #[cfg(any( - // Windows support is only for x86_64 architecture (for now) - all(target_os = "windows", not(target_arch = "x86_64")), - // Linux and macOS support only x86_64 and aarch64 architectures - all( - any(target_os = "linux", target_os = "macos"), - not(any(target_arch = "x86_64", target_arch = "aarch64")) - ), - // Any OS other than Windows, Linux, or macOS is unsupported - not(any(target_os = "windows", target_os = "linux", target_os = "macos")), - ))] - return Err(StaticDistDownloadError::UnsupportedArchitecture); - - let ver = Self::find_suitable_version(requested_version) - .ok_or(StaticDistDownloadError::UnsupportedVersion)?; - let ver_str = ver.major.to_string(); - // we already gated unsupported architectures above, - // so we can assume it's either x86_64 or aarch64 here - let arch = if cfg!(target_arch = "aarch64") { - "arm64" - } else { - "amd64" - }; - let platform = if cfg!(target_os = "windows") { - "windows" - } else if cfg!(target_os = "macos") { - "macos" - } else { - "linux" - }; - - let base_url = format!( - "{CLANG_TOOLS_REPO}/releases/download/{CLANG_TOOLS_TAG}/{tool}-{ver_str}_{platform}-{arch}", - ); - let suffix = if cfg!(target_os = "windows") { - ".exe" - } else { - "" - }; - let url = Url::parse(format!("{base_url}{suffix}").as_str())?; - let cache_path = Self::get_cache_dir(); - let bin_name = format!("{tool}-{ver_str}{suffix}"); - let download_path = match directory { - None => cache_path.join("bin").join(&bin_name), - Some(dir) => dir.join(&bin_name), - }; - let file_lock = lock_path(&download_path)?; - if download_path.exists() { - log::info!( - "Using cached static binary for {tool} version {ver_str} from {:?}", - download_path.to_string_lossy() - ); - } else { - log::info!("Downloading static binary for {tool} version {ver_str} from {url}"); - download(&url, &download_path, 60 * 2).await?; - #[cfg(unix)] - super::chmod_file(&download_path, None)?; + /// Verifies the SHA512 checksum of the downloaded file. + /// + /// The expected checksum is extracted from another downloaded `*.sha512sum` file + /// (pointed to by `sha512_path`). + fn verify_sha512( + file_path: &Path, + sha512_path: &Path, + ) -> Result<(), StaticDistDownloadError> { + let checksum_file_content = fs::read_to_string(sha512_path)?; + let expected = checksum_file_content + .split(' ') + .next() + .ok_or(StaticDistDownloadError::Sha512Corruption)?; + HashAlgorithm::Sha512(expected.to_string()).verify(file_path)?; + Ok(()) } - let sha512_cache_path = cache_path - .join("static_dist") - .join(format!("{tool}-{ver_str}.sha512")); - if sha512_cache_path.exists() { - log::info!( - "Using cached SHA512 checksum for {tool} version {ver_str} from {:?}", - sha512_cache_path.to_string_lossy() - ); - } else { - let sha512_url = Url::parse(format!("{base_url}{suffix}.sha512sum").as_str())?; - log::info!( - "Downloading SHA512 checksum for {tool} version {ver_str} from {sha512_url}" + + /// Downloads the `requested_version` of the specified `tool` from a distribution of statically linked binaries. + /// + /// The distribution is maintained at . + /// Supported platforms includes Windows, Linux, and MacOS. + /// Supported architectures is limited to `x86_64` (`amd64`) and `aarch64` (`arm64`); + /// Windows only supports x86_64 architecture (for now). + pub async fn download_tool( + tool: &ClangTool, + requested_version: &VersionReq, + directory: Option<&PathBuf>, + ) -> Result { + let ver = Self::find_suitable_version(requested_version) + .ok_or(StaticDistDownloadError::UnsupportedVersion)?; + let ver_str = ver.major.to_string(); + // we already gated unsupported architectures above, + // so we can assume it's either x86_64 or aarch64 here + let arch = if cfg!(target_arch = "aarch64") { + "arm64" + } else { + "amd64" + }; + let platform = if cfg!(target_os = "windows") { + "windows" + } else if cfg!(target_os = "macos") { + "macos" + } else { + "linux" + }; + + let base_url = format!( + "{CLANG_TOOLS_REPO}/releases/download/{CLANG_TOOLS_TAG}/{tool}-{ver_str}_{platform}-{arch}", ); - download(&sha512_url, &sha512_cache_path, 10).await?; + let suffix = if cfg!(target_os = "windows") { + ".exe" + } else { + "" + }; + let url = Url::parse(format!("{base_url}{suffix}").as_str())?; + let cache_path = Self::get_cache_dir(); + let bin_name = format!("{tool}-{ver_str}{suffix}"); + let download_path = match directory { + None => cache_path.join("bin").join(&bin_name), + Some(dir) => dir.join(&bin_name), + }; + let file_lock = lock_path(&download_path)?; + if download_path.exists() { + log::info!( + "Using cached static binary for {tool} version {ver_str} from {:?}", + download_path.to_string_lossy() + ); + } else { + log::info!("Downloading static binary for {tool} version {ver_str} from {url}"); + download(&url, &download_path, 60 * 2).await?; + #[cfg(unix)] + super::super::chmod_file(&download_path, None)?; + } + let sha512_cache_path = cache_path + .join("static_dist") + .join(format!("{tool}-{ver_str}.sha512")); + if sha512_cache_path.exists() { + log::info!( + "Using cached SHA512 checksum for {tool} version {ver_str} from {:?}", + sha512_cache_path.to_string_lossy() + ); + } else { + let sha512_url = Url::parse(format!("{base_url}{suffix}.sha512sum").as_str())?; + log::info!( + "Downloading SHA512 checksum for {tool} version {ver_str} from {sha512_url}" + ); + download(&sha512_url, &sha512_cache_path, 10).await?; + } + Self::verify_sha512(&download_path, &sha512_cache_path)?; + file_lock.unlock()?; + Ok(download_path) } - Self::verify_sha512(&download_path, &sha512_cache_path)?; - file_lock.unlock()?; - Ok(download_path) } -} -#[cfg(test)] -mod tests { - use super::StaticDistDownloader; - use semver::VersionReq; + #[cfg(test)] + mod tests { + use super::StaticDistDownloader; + use semver::VersionReq; - #[test] - fn find_none() { - let req_ver = VersionReq::parse("=8").unwrap(); - let suitable_version = StaticDistDownloader::find_suitable_version(&req_ver); - assert_eq!(suitable_version, None); + #[test] + fn find_none() { + let req_ver = VersionReq::parse("=8").unwrap(); + let suitable_version = StaticDistDownloader::find_suitable_version(&req_ver); + assert_eq!(suitable_version, None); + } } } diff --git a/nurfile b/nurfile index d0be790..41cd72c 100644 --- a/nurfile +++ b/nurfile @@ -20,6 +20,15 @@ def "nur docs rs" [ run-cmd ...$cmd } +const URL = "https://github.com/cpp-linter/clang-tools-static-binaries/releases/latest/download/versions.json" + +# Seed the versions.json file with the latest release from GitHub. +# +# Useful to avoid network requests in local dev builds of clang-tools-manager crate. +# This will instigate a warning, but that is expected. +def "nur seed versions" [] { + let versions = (http get $URL) | save --force clang-tools-manager/versions.json +} # Build mkdocs output. #