From 2818f2d2ab345b95629265a166981f2aa39b2053 Mon Sep 17 00:00:00 2001 From: Chevdor Date: Thu, 15 Jul 2021 18:04:39 +0200 Subject: [PATCH 01/10] wip --- README.md | 19 +++++++++++++++++++ README_src.adoc | 5 +++++ cli/src/main.rs | 4 ++++ cli/src/opts.rs | 14 ++++++++++++++ doc/usage.adoc | 2 ++ doc/usage_verify.adoc | 14 ++++++++++++++ justfile | 1 + 7 files changed, 59 insertions(+) create mode 100644 doc/usage_verify.adoc diff --git a/README.md b/README.md index d4fee2b..b437326 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,8 @@ This alias is likely set in your `.bash_profile` or `.zshrc`, make sure to remov help Prints this message or the help of the given subcommand(s) info Provide information about the srtool container and your repo pull Simply pull the srtool image and do not run anything else + verify Show the versions of the srtool container. Use --version if you want the version + of this executable version Show the versions of the srtool container. Use --version if you want the version of this executable @@ -157,6 +159,23 @@ This alias is likely set in your `.bash_profile` or `.zshrc`, make sure to remov If your runtime is not in the standard location runtime/ you can pass this args to help srtool find it [env: RUNTIME_DIR=] +**verify** + + srtool-verify 0.6.0 + chevdor + Show the versions of the srtool container. Use --version if you want the version of this executable + + USAGE: + srtool verify + + ARGS: + The path of the srtool digest (json) where most of the settings will be fetched + to reproduce the exact same build + + FLAGS: + -h, --help Prints help information + -V, --version Prints version information + ## Contributing If you landed here, you likely want to contribute the project. Let me thank you already. diff --git a/README_src.adoc b/README_src.adoc index 43c42c0..f0c46b2 100644 --- a/README_src.adoc +++ b/README_src.adoc @@ -56,4 +56,9 @@ include::./doc/usage_pull.adoc[] include::./doc/usage_build.adoc[] ---- +.verify +---- +include::./doc/usage_verify.adoc[] +---- + include::CONTRIBUTING.adoc[] diff --git a/cli/src/main.rs b/cli/src/main.rs index c639d2c..aa1aa0f 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -122,6 +122,10 @@ fn main() { SubCommand::Version(_) => { format!("docker run --name srtool --rm {image}:{tag} version", image = image, tag = tag,) } + + SubCommand::Verify(_) => { + todo!() + } }; debug!("command = {:?}", command); diff --git a/cli/src/opts.rs b/cli/src/opts.rs index badebc7..97eda79 100644 --- a/cli/src/opts.rs +++ b/cli/src/opts.rs @@ -48,6 +48,11 @@ pub enum SubCommand { /// the version of this executable. #[clap(version = crate_version!(), author = crate_authors!())] Version(VersionOpts), + + /// Show the versions of the srtool container. Use --version if you want + /// the version of this executable. + #[clap(version = crate_version!(), author = crate_authors!())] + Verify(VerifyOpts), } /// Build opts @@ -127,3 +132,12 @@ pub struct InfoOpts { /// Version opts #[derive(Clap)] pub struct VersionOpts; + +/// Verify opts +#[derive(Clap)] +pub struct VerifyOpts { + /// The path of the srtool digest (json) where most of the settings + /// will be fetched to reproduce the exact same build. + #[clap(index = 1)] + pub digest: PathBuf, +} diff --git a/doc/usage.adoc b/doc/usage.adoc index 1325986..c3e35fb 100644 --- a/doc/usage.adoc +++ b/doc/usage.adoc @@ -24,5 +24,7 @@ SUBCOMMANDS: help Prints this message or the help of the given subcommand(s) info Provide information about the srtool container and your repo pull Simply pull the srtool image and do not run anything else + verify Show the versions of the srtool container. Use --version if you want the version + of this executable version Show the versions of the srtool container. Use --version if you want the version of this executable diff --git a/doc/usage_verify.adoc b/doc/usage_verify.adoc new file mode 100644 index 0000000..5104b53 --- /dev/null +++ b/doc/usage_verify.adoc @@ -0,0 +1,14 @@ +srtool-verify 0.6.0 +chevdor +Show the versions of the srtool container. Use --version if you want the version of this executable + +USAGE: + srtool verify + +ARGS: + The path of the srtool digest (json) where most of the settings will be fetched + to reproduce the exact same build + +FLAGS: + -h, --help Prints help information + -V, --version Prints version information diff --git a/justfile b/justfile index ea60a8e..57b49cc 100644 --- a/justfile +++ b/justfile @@ -14,6 +14,7 @@ usage: cargo run -q -- build --help > doc/usage_build.adoc cargo run -q -- version --help > doc/usage_version.adoc cargo run -q -- info --help > doc/usage_info.adoc + cargo run -q -- verify --help > doc/usage_verify.adoc # When comes the time to release, this will set a new tag tag: From 7df1c55bae6cf17fa29581e0baa8b0f4c2f8d5c4 Mon Sep 17 00:00:00 2001 From: Chevdor Date: Fri, 16 Jul 2021 09:23:08 +0200 Subject: [PATCH 02/10] chore: reordering --- .rustfmt.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.rustfmt.toml b/.rustfmt.toml index c699603..fd0c287 100644 --- a/.rustfmt.toml +++ b/.rustfmt.toml @@ -1,4 +1,4 @@ +edition = "2018" hard_tabs = true max_width = 120 use_small_heuristics = "Max" -edition = "2018" From 48602e5c2ed9c616dc6e03a7ad8e0f91736312be Mon Sep 17 00:00:00 2001 From: Chevdor Date: Fri, 16 Jul 2021 17:03:18 +0200 Subject: [PATCH 03/10] WIP --- Cargo.lock | 27 +++++++++++++- cli/src/main.rs | 3 +- cli/src/opts.rs | 2 +- lib/Cargo.toml | 2 + lib/src/digest.rs | 14 +++++++ lib/src/digest_json.rs | 60 ++++++++++++++++++++++++++++++ lib/src/digest_source.rs | 8 ++++ lib/src/digest_v9.rs | 79 ++++++++++++++++++++++++++++++++++++++++ lib/src/lib.rs | 5 +++ 9 files changed, 197 insertions(+), 3 deletions(-) create mode 100644 lib/src/digest.rs create mode 100644 lib/src/digest_json.rs create mode 100644 lib/src/digest_source.rs create mode 100644 lib/src/digest_v9.rs diff --git a/Cargo.lock b/Cargo.lock index 25a4f43..193ba68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -456,11 +456,34 @@ dependencies = [ "untrusted", ] +[[package]] +name = "semver" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f3aac57ee7f3272d8395c6e4f502f434f0e289fcd62876f70daa008c20dcabe" +dependencies = [ + "serde", +] + [[package]] name = "serde" version = "1.0.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f03b9878abf6d14e6779d3f24f07b2cfa90352cfec4acc5aab8f1ac7f146fae8" +checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "serde_json" @@ -497,6 +520,8 @@ name = "srtool-lib" version = "0.6.0" dependencies = [ "log", + "semver", + "serde", "serde_json", "ureq", ] diff --git a/cli/src/main.rs b/cli/src/main.rs index aa1aa0f..e5da12f 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -123,7 +123,8 @@ fn main() { format!("docker run --name srtool --rm {image}:{tag} version", image = image, tag = tag,) } - SubCommand::Verify(_) => { + SubCommand::Verify(verify_opts) => { + debug!("Digest from: {:?}", verify_opts.digest); todo!() } }; diff --git a/cli/src/opts.rs b/cli/src/opts.rs index 97eda79..6c0c8cf 100644 --- a/cli/src/opts.rs +++ b/cli/src/opts.rs @@ -51,7 +51,7 @@ pub enum SubCommand { /// Show the versions of the srtool container. Use --version if you want /// the version of this executable. - #[clap(version = crate_version!(), author = crate_authors!())] + #[clap(version = crate_version!(), author = crate_authors!(), alias = "check")] Verify(VerifyOpts), } diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 7aad655..b03808f 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -8,5 +8,7 @@ edition = "2018" [dependencies] log = "0.4" +semver = {version = "1.0", features = ["serde"]} +serde = {version = "1.0", features = ["derive"]} serde_json = "1.0" ureq = "2.1" diff --git a/lib/src/digest.rs b/lib/src/digest.rs new file mode 100644 index 0000000..898de07 --- /dev/null +++ b/lib/src/digest.rs @@ -0,0 +1,14 @@ +//! The srtool digests format has evolved over time. The first versions were rather flat +//! and no old schema has been written here, at least not yet. That means the old digests +//! are currently not supported. The first schema that is supported is called V9 and matches the +//! output of srtool v0.9.15. The latest V9 is compatible with the old format and thus contains +//! duplication. V10 will be a chance for a good cleanup. +//! Especially, the split between content in Info and Context should be improved. + +use crate::digest_v9::V9; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub enum Digest { + V9(V9), +} diff --git a/lib/src/digest_json.rs b/lib/src/digest_json.rs new file mode 100644 index 0000000..217c5be --- /dev/null +++ b/lib/src/digest_json.rs @@ -0,0 +1,60 @@ +use serde_json::Result; +use serde_json::Value; + +use crate::{digest::Digest, digest_source::DigestSource}; +type Json = Value; +pub struct DigestJson {} + +impl DigestSource for DigestJson { + fn load(src: Json) -> Result { + let digest: Digest = serde_json::from_str(&src.to_string())?; + Ok(digest) + } +} + +#[cfg(test)] +mod test_super { + use super::*; + use crate::digest_v9; + use serde_json::json; + + #[test] + fn test_() { + let json = json!({ + "V9": { + "context": { + "docker": { + "image": "paritytech/srtool", + "full_tag": "1.53.0-0.9.15", + }, + "runtime_dir": "runtime/polkadot", + "package": "polkadot-runtime", + "profile": "release", + }, + "source": "git", + "info": { + + "generator": { + "name": "srtool", + "version": "0.9.15", + }, + "src": "git", + "version": "0.9.7", + "git": { + "commit": "5d35bac7408a4cb12a578764217d06f3920b36aa", + "tag": "v0.9.7-rc3", + "branch": "heads/v0.9.7-rc3", + }, + "rustc": "rustc 1.53.0 (53cb7b09b 2021-06-17)", + "pkg": "polkadot-runtime", + "profile": "release", + }, + } + }); + let digest = DigestJson::load(json).unwrap(); + + match digest { + Digest::V9(v9) => assert!(v9.source == digest_v9::Source::Git), + } + } +} diff --git a/lib/src/digest_source.rs b/lib/src/digest_source.rs new file mode 100644 index 0000000..5e3baba --- /dev/null +++ b/lib/src/digest_source.rs @@ -0,0 +1,8 @@ +use crate::digest::Digest; +use serde_json::Result; + +/// This trait describes digest sources. Those could be +/// File, Http, Ipfs, etc.... +pub trait DigestSource { + fn load(source: S) -> Result; +} diff --git a/lib/src/digest_v9.rs b/lib/src/digest_v9.rs new file mode 100644 index 0000000..9b80c92 --- /dev/null +++ b/lib/src/digest_v9.rs @@ -0,0 +1,79 @@ +use semver::Version; +use serde::{Serialize, Deserialize}; + +#[derive(Debug, Serialize, Deserialize, PartialEq)] +pub enum Source { + #[serde(alias = "git")] + Git, + Archive, +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct Generator { + name: String, + version: Version, +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct GitInfo { + commit: String, + tag: String, + branch: String, +} + +//TODO: in V9, in order to NOT break compatibility, some fields are duplicated. That must be reworked. The profile for instance should be in the Context only. + +/// The difference between Info and Context is that the content +/// of Info is (should be...) derivated from Context. In other words, once we have +/// the Context, we can extract all the Info. +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct Info { + /// Information about the tooling used for the build. + generator: Generator, + + /// Whether the build from an Archive or from a Git repo. + src: Source, + + /// The version of the crate/package to build + version: Version, + + /// Optionnal Git information if the src was Git + git: Option, + + /// Rust compiler version + rustc: String, + + /// Package + pkg: String, + + /// Profile. Always 'release'. + profile: String, +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct DockerContext { + pub image: String, + pub full_tag: String, +} + +/// This struct describes all the information required +/// by srtool to build a runtime. +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct Context { + pub docker: DockerContext, + pub runtime_dir: String, + pub package: String, + pub profile: String, +} + +/// A srtool digest. The schema of the data srtool produces may +/// change over time. This struct can load all version and make +/// the common and relevant data available. +// TODO: This is a piece that should be shared with/coming from srtool-cargo. +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct V9 { + pub info: Info, + pub context: Context, + pub source: Source, + +} diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 5e0fcbf..6073808 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -1,3 +1,8 @@ +mod digest; +mod digest_source; +mod digest_json; +mod digest_v9; + use log::{debug, info}; use std::{ env, From d762789c0b3dcac9b77dbdff76352ea3b651e3c0 Mon Sep 17 00:00:00 2001 From: Chevdor Date: Mon, 19 Jul 2021 18:56:27 +0200 Subject: [PATCH 04/10] WIP: ground work --- Cargo.lock | 2 + cli/Cargo.toml | 2 + cli/src/main.rs | 11 +++ cli/src/opts.rs | 6 +- lib/src/digest.rs | 103 ++++++++++++++++++++- lib/src/digest_json.rs | 10 +- lib/src/digest_source.rs | 2 +- lib/src/digest_v1.rs | 26 ++++++ lib/src/{digest_v9.rs => digest_v2.rs} | 71 +++++++------- lib/src/lib.rs | 14 ++- lib/src/run_specs.rs | 25 +++++ lib/src/rustc_version.rs | 37 ++++++++ lib/src/samples.rs | 123 +++++++++++++++++++++++++ lib/src/srtool_tag.rs | 54 +++++++++++ 14 files changed, 436 insertions(+), 50 deletions(-) create mode 100644 lib/src/digest_v1.rs rename lib/src/{digest_v9.rs => digest_v2.rs} (55%) create mode 100644 lib/src/run_specs.rs create mode 100644 lib/src/rustc_version.rs create mode 100644 lib/src/samples.rs create mode 100644 lib/src/srtool_tag.rs diff --git a/Cargo.lock b/Cargo.lock index 193ba68..4778499 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -512,6 +512,8 @@ dependencies = [ "ctrlc", "env_logger", "log", + "serde", + "serde_json", "srtool-lib", ] diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 74418cf..973d927 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -25,6 +25,8 @@ ctrlc = "3.2" env_logger = "0.9" log = "0.4" srtool-lib = {path = "../lib"} +serde = {version = "1.0", features = ["derive"]} +serde_json = "1.0" [package.metadata.deb] assets = [ diff --git a/cli/src/main.rs b/cli/src/main.rs index e5da12f..ca7e5a8 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -2,7 +2,10 @@ mod opts; use clap::{crate_version, Clap}; use log::{debug, info}; use opts::*; +use serde_json::json; use srtool_lib::*; +use std::fs::File; +use std::io::BufReader; use std::path::PathBuf; use std::process::Command; use std::{env, fs}; @@ -125,6 +128,14 @@ fn main() { SubCommand::Verify(verify_opts) => { debug!("Digest from: {:?}", verify_opts.digest); + let file = File::open(verify_opts.digest).unwrap(); + let reader = BufReader::new(file); + let content: V2 = serde_json::from_reader(reader).unwrap(); + let digest_json = json!({ "V2": content }); + // let digest = Digest::from + // debug!("digest = {:#?}", digest); + // let specs = digest.get_run_specs(); + // debug!("specs = {:#?}", specs); todo!() } }; diff --git a/cli/src/opts.rs b/cli/src/opts.rs index 6c0c8cf..70baffe 100644 --- a/cli/src/opts.rs +++ b/cli/src/opts.rs @@ -49,8 +49,10 @@ pub enum SubCommand { #[clap(version = crate_version!(), author = crate_authors!())] Version(VersionOpts), - /// Show the versions of the srtool container. Use --version if you want - /// the version of this executable. + /// Run a new build based on the digest of a previous run in order + /// to check/verify the result. Such a check may not use the very latest + /// version of the srtool image but use the same version than used in the + /// reference run. #[clap(version = crate_version!(), author = crate_authors!(), alias = "check")] Verify(VerifyOpts), } diff --git a/lib/src/digest.rs b/lib/src/digest.rs index 898de07..4c16625 100644 --- a/lib/src/digest.rs +++ b/lib/src/digest.rs @@ -1,14 +1,107 @@ //! The srtool digests format has evolved over time. The first versions were rather flat -//! and no old schema has been written here, at least not yet. That means the old digests -//! are currently not supported. The first schema that is supported is called V9 and matches the -//! output of srtool v0.9.15. The latest V9 is compatible with the old format and thus contains +//! and no old schema has been written here, at least not yet. That means the old digests +//! are currently not supported. The first schema that is supported is called V2 and matches the +//! output of srtool v0.9.15. The latest V2 is compatible with the old format and thus contains //! duplication. V10 will be a chance for a good cleanup. //! Especially, the split between content in Info and Context should be improved. -use crate::digest_v9::V9; +// TODO: The code for the srtool digest needs to be moved under srtool-cargo once published. + +use std::str::FromStr; + +use crate::{digest_v2::V2, run_specs::RunSpecs}; +use semver::{Version, VersionReq}; use serde::{Deserialize, Serialize}; +use serde_json::Value; #[derive(Debug, PartialEq, Serialize, Deserialize)] pub enum Digest { - V9(V9), + V2(V2), +} + +impl Digest { + pub fn get_run_specs(&self) -> Result { + match self { + Digest::V2(v2) => Ok(RunSpecs { + package: v2.context.package.to_owned(), + runtime_dir: v2.context.runtime_dir.to_owned(), + profile: v2.context.profile.to_owned(), + image: v2.context.docker.image.to_owned(), + image_sha256: String::new(), // TODO + cargo_build_opts: Vec::new(), // TODO + default_features: Vec::new(), // TODO + tag: v2.context.docker.full_tag.to_owned(), + cache_mount: false, // TODO + }), + } + } + + pub fn get_version(json: Value) -> Option { + let version_v1 = &json["gen"].as_str().unwrap_or_default().split('v').nth(1); + let version_v2 = &json["info"]["generator"]["version"].as_str(); + + if let Some(v) = version_v2 { + return Some(Version::from_str(v).unwrap()); + } + + if let Some(v) = version_v1 { + return Some(Version::from_str(v).unwrap()); + } + + None + } +} + +impl From for Digest { + fn from(v: Value) -> Self { + if !v.is_object() { + panic!("Invalid digest, it should be a JSON Object"); + } + + let version = Digest::get_version(v); + + match version { + // TODO: exact version to double check, not sure when the new format was introduced, 0.9.13 or 0.9.14 + Some(v) if VersionReq::parse("<=0.9.13").unwrap().matches(&v) => { + // V1 + todo!() + } + Some(v) if VersionReq::parse(">0.9.14 <=0.9.15").unwrap().matches(&v) => { + // V2 + todo!() + } + Some(v) => panic!("Version {} is not supported", v), + None => unreachable!(), + } + } +} + +#[cfg(test)] +mod test_digest { + use super::*; + use crate::samples::*; + + #[test] + fn test_version_from_json_v1() { + let v1: Value = serde_json::from_str(SAMPLE_V1).unwrap(); + assert_eq!(Digest::get_version(v1), Some(Version::from_str("0.9.14").unwrap())); + } + + #[test] + fn test_version_from_json_v2() { + let v2: Value = serde_json::from_str(SAMPLE_V2).unwrap(); + assert_eq!(Digest::get_version(v2), Some(Version::from_str("0.9.15").unwrap())); + } + + #[test] + fn test_version_from_json_v3() { + let v3: Value = serde_json::from_str(SAMPLE_V3).unwrap(); + assert_eq!(Digest::get_version(v3), Some(Version::from_str("0.9.17").unwrap())); + } + + #[test] + fn test_version_from_json_unknown() { + let v4: Value = serde_json::from_str(SAMPLE_V4).unwrap(); + assert_eq!(Digest::get_version(v4), None); + } } diff --git a/lib/src/digest_json.rs b/lib/src/digest_json.rs index 217c5be..7bd1d1b 100644 --- a/lib/src/digest_json.rs +++ b/lib/src/digest_json.rs @@ -1,8 +1,8 @@ +use crate::{digest::Digest, digest_source::DigestSource}; use serde_json::Result; use serde_json::Value; - -use crate::{digest::Digest, digest_source::DigestSource}; type Json = Value; + pub struct DigestJson {} impl DigestSource for DigestJson { @@ -15,13 +15,13 @@ impl DigestSource for DigestJson { #[cfg(test)] mod test_super { use super::*; - use crate::digest_v9; + use crate::digest_v2; use serde_json::json; #[test] fn test_() { let json = json!({ - "V9": { + "V2": { "context": { "docker": { "image": "paritytech/srtool", @@ -54,7 +54,7 @@ mod test_super { let digest = DigestJson::load(json).unwrap(); match digest { - Digest::V9(v9) => assert!(v9.source == digest_v9::Source::Git), + Digest::V2(v2) => assert!(v2.source == digest_v2::Source::Git), } } } diff --git a/lib/src/digest_source.rs b/lib/src/digest_source.rs index 5e3baba..dcda013 100644 --- a/lib/src/digest_source.rs +++ b/lib/src/digest_source.rs @@ -4,5 +4,5 @@ use serde_json::Result; /// This trait describes digest sources. Those could be /// File, Http, Ipfs, etc.... pub trait DigestSource { - fn load(source: S) -> Result; + fn load(source: S) -> Result; } diff --git a/lib/src/digest_v1.rs b/lib/src/digest_v1.rs new file mode 100644 index 0000000..f4c33ce --- /dev/null +++ b/lib/src/digest_v1.rs @@ -0,0 +1,26 @@ +use semver::Version; +use serde::{Deserialize, Serialize}; + +//TODO: in V2, in order to NOT break compatibility, some fields are duplicated. That must be reworked. The profile for instance should be in the Context only. + +/// A srtool digest. The schema of the data srtool produces may +/// change over time. This struct can load all version and make +/// the common and relevant data available. +// TODO: This is a piece that should be shared with/coming from srtool-cargo. +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct V1 { + pub(crate) gen: String, + pub(crate) src: String, + pub(crate) version: Version, + pub(crate) commit: String, + pub(crate) tag: String, + pub(crate) branch: String, + pub(crate) rustc: String, + pub(crate) pkg: String, + pub(crate) tmsp: String, + pub(crate) size: uisze, + pub(crate) prop: String, + pub(crate) ipfs: String, + pub(crate) sha256: String, + pub(crate) wasm: String, +} diff --git a/lib/src/digest_v9.rs b/lib/src/digest_v2.rs similarity index 55% rename from lib/src/digest_v9.rs rename to lib/src/digest_v2.rs index 9b80c92..1e205f0 100644 --- a/lib/src/digest_v9.rs +++ b/lib/src/digest_v2.rs @@ -1,69 +1,70 @@ use semver::Version; -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; #[derive(Debug, Serialize, Deserialize, PartialEq)] pub enum Source { - #[serde(alias = "git")] - Git, - Archive, + #[serde(alias = "git")] + Git, + Archive, } #[derive(Debug, PartialEq, Serialize, Deserialize)] pub struct Generator { - name: String, - version: Version, + name: String, + version: Version, } #[derive(Debug, PartialEq, Serialize, Deserialize)] pub struct GitInfo { - commit: String, - tag: String, - branch: String, + commit: String, + tag: String, + branch: String, } -//TODO: in V9, in order to NOT break compatibility, some fields are duplicated. That must be reworked. The profile for instance should be in the Context only. +//TODO: in V2, in order to NOT break compatibility, some fields are duplicated. That must be reworked. The profile for instance should be in the Context only. /// The difference between Info and Context is that the content /// of Info is (should be...) derivated from Context. In other words, once we have /// the Context, we can extract all the Info. #[derive(Debug, PartialEq, Serialize, Deserialize)] pub struct Info { - /// Information about the tooling used for the build. - generator: Generator, + /// Information about the tooling used for the build. + generator: Generator, - /// Whether the build from an Archive or from a Git repo. - src: Source, + /// Whether the build from an Archive or from a Git repo. + src: Source, - /// The version of the crate/package to build - version: Version, + /// The version of the crate/package to build + version: Version, - /// Optionnal Git information if the src was Git - git: Option, + /// Optionnal Git information if the src was Git + git: Option, - /// Rust compiler version - rustc: String, + /// Rust compiler version + rustc: String, - /// Package - pkg: String, + /// Package + pkg: String, - /// Profile. Always 'release'. - profile: String, + /// Profile. Always 'release'. + profile: String, } #[derive(Debug, PartialEq, Serialize, Deserialize)] pub struct DockerContext { - pub image: String, - pub full_tag: String, + pub image: String, + #[serde(alias = "tag")] + pub full_tag: String, } /// This struct describes all the information required /// by srtool to build a runtime. #[derive(Debug, PartialEq, Serialize, Deserialize)] pub struct Context { - pub docker: DockerContext, - pub runtime_dir: String, - pub package: String, - pub profile: String, + pub(crate) docker: DockerContext, + pub(crate) runtime_dir: String, + pub(crate) package: String, + pub(crate) profile: String, } /// A srtool digest. The schema of the data srtool produces may @@ -71,9 +72,9 @@ pub struct Context { /// the common and relevant data available. // TODO: This is a piece that should be shared with/coming from srtool-cargo. #[derive(Debug, PartialEq, Serialize, Deserialize)] -pub struct V9 { - pub info: Info, - pub context: Context, - pub source: Source, - +pub struct V2 { + pub(crate) info: Info, + pub(crate) context: Context, + #[serde(alias = "src")] + pub(crate) source: Source, } diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 6073808..445a1e5 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -1,7 +1,17 @@ mod digest; -mod digest_source; mod digest_json; -mod digest_v9; +mod digest_source; +mod digest_v2; +mod run_specs; +mod rustc_version; +mod samples; +mod srtool_tag; + +pub use digest::*; +pub use digest_json::*; +pub use digest_source::*; +pub use digest_v2::*; +pub use srtool_tag::*; use log::{debug, info}; use std::{ diff --git a/lib/src/run_specs.rs b/lib/src/run_specs.rs new file mode 100644 index 0000000..c71e8f4 --- /dev/null +++ b/lib/src/run_specs.rs @@ -0,0 +1,25 @@ +//! Ideally, we don't need this struct and everything is available under the `Context`. +//! However, V2 does not contain enough in the Context and has some of the information under +//! the `Info` key. So we bring everything together as `RunSpecs`. + +pub struct RunSpecs { + /// Name of the crate of the runtime + pub(crate) package: String, + + /// Path to the runtime crate relative to the root of the repository + pub(crate) runtime_dir: String, + + /// Usually `release` + pub(crate) profile: String, + + /// The docker image, ie: paritytech/srtool + pub(crate) image: String, + + /// The digest of the docker image + pub(crate) image_sha256: String, + pub(crate) cargo_build_opts: Vec, + pub(crate) default_features: Vec, + + pub(crate) tag: String, + pub(crate) cache_mount: bool, +} diff --git a/lib/src/rustc_version.rs b/lib/src/rustc_version.rs new file mode 100644 index 0000000..ad71e33 --- /dev/null +++ b/lib/src/rustc_version.rs @@ -0,0 +1,37 @@ +use std::str::FromStr; + +use semver::Version; + +#[derive(Debug, PartialEq)] +pub enum RustcVersion { + Stable(Version), + // Beta(String), + Nightly(String), + // Dev(String), +} + +impl FromStr for RustcVersion { + type Err = String; + + // TODO: The checks below are very light ... See https://docs.rs/rustc_version/0.2.3/rustc_version/ + fn from_str(s: &str) -> Result { + match semver::Version::from_str(s) { + Ok(version) => Ok(RustcVersion::Stable(version)), + _ => Ok(RustcVersion::Nightly(s.to_string())), + } + } +} + +#[cfg(test)] +mod test_rustc_version { + use super::*; + + #[test] + fn test_from_str() { + println!("v = {:?}", RustcVersion::from_str("1.53.0")); + println!("v = {:?}", RustcVersion::from_str("nightly-2021-03-15")); + + // TODO: the following should error + // println!("v = {:?}", RustcVersion::from_str("junk")); + } +} diff --git a/lib/src/samples.rs b/lib/src/samples.rs new file mode 100644 index 0000000..4d6441c --- /dev/null +++ b/lib/src/samples.rs @@ -0,0 +1,123 @@ +#[cfg(test)] +pub const SAMPLE_V1: &str = r#"{ + "gen": "srtool v0.9.14", + "src": "git", + "version": "0.9.7", + "commit": "5d35bac7408a4cb12a578764217d06f3920b36aa", + "tag": "v0.9.7-rc3", + "branch": "heads/v0.9.7-rc3", + "rustc": "rustc 1.53.0 (53cb7b09b 2021-06-17)", + "pkg": "polkadot-runtime", + "tmsp": "2021-06-29T16:12:59Z", + "size": "2093380", + "prop": "0x424ac5063ce844b878cd418e7d4c0e5518a6323ec0c54f744b1fb44a2ab24dcd", + "ipfs": "QmeBgekBhZHNCkrayDgQaLXfAoLibS5Eq2fyEv4rzbttTo", + "sha256": "0x5f31cd25a9de645f278f18b008f38edad5b3253c1b94dc71a12da48c27dd1581", + "wasm": "runtime/polkadot/target/srtool/release/wbuild/polkadot-runtime/polkadot_runtime.compact.wasm" +}"#; + +#[cfg(test)] +pub const SAMPLE_V2: &str = r#"{ + "info": { + "generator": { + "name": "srtool", + "version": "0.9.15" + }, + "src": "git", + "version": "0.9.7", + "git": { + "commit": "5d35bac7408a4cb12a578764217d06f3920b36aa", + "tag": "v0.9.7-rc3", + "branch": "heads/v0.9.7-rc3" + }, + "rustc": "rustc 1.53.0 (53cb7b09b 2021-06-17)", + "pkg": "polkadot-runtime", + "profile": "release" + }, + "context": { + "package": "polkadot-runtime", + "runtime_dir": "runtime/polkadot", + "docker": { + "image": "chevdor/srtool", + "tag": "1.53.0" + }, + "profile": "release" + }, + "runtimes": { + "compact": { + "tmsp": "2021-06-29T16:12:24Z", + "sha256": "0x5f31cd25a9de645f278f18b008f38edad5b3253c1b94dc71a12da48c27dd1581", + "wasm": "runtime/polkadot/target/srtool/release/wbuild/polkadot-runtime/polkadot_runtime.compact.wasm", + "subwasm": { + "size": 2093380, + "compression": { + "size_compressed": 2093380, + "size_decompressed": 2093380, + "compressed": false + }, + "reserved_meta_valid": true, + "metadata_version": 13, + "core_version": "polkadot-9070 (parity-polkadot-0.tx7.au0)", + "proposal_hash": "0x424ac5063ce844b878cd418e7d4c0e5518a6323ec0c54f744b1fb44a2ab24dcd", + "ipfs_hash": "QmeBgekBhZHNCkrayDgQaLXfAoLibS5Eq2fyEv4rzbttTo", + "blake2_256": "0xc5daf28ebf7f23c8de92a99a6c15b84abeaf12d226542e7504febaf0d1484e05" + } + }, + "compressed": {} + } +}"#; + +#[cfg(test)] +pub const SAMPLE_V3: &str = r#"{ + "version": "0.9.17", + "info": { + "generator": { + "name": "srtool", + "version": "0.9.17" + }, + "src": "git", + "version": "0.9.7", + "git": { + "commit": "5d35bac7408a4cb12a578764217d06f3920b36aa", + "tag": "v0.9.7-rc3", + "branch": "heads/v0.9.7-rc3" + }, + "rustc": "rustc 1.53.0 (53cb7b09b 2021-06-17)", + "pkg": "polkadot-runtime", + "profile": "release" + }, + "context": { + "package": "polkadot-runtime", + "runtime_dir": "runtime/polkadot", + "docker": { + "image": "chevdor/srtool", + "tag": "1.53.0" + }, + "profile": "release" + }, + "runtimes": { + "compact": { + "tmsp": "2021-06-29T16:12:24Z", + "sha256": "0x5f31cd25a9de645f278f18b008f38edad5b3253c1b94dc71a12da48c27dd1581", + "wasm": "runtime/polkadot/target/srtool/release/wbuild/polkadot-runtime/polkadot_runtime.compact.wasm", + "subwasm": { + "size": 2093380, + "compression": { + "size_compressed": 2093380, + "size_decompressed": 2093380, + "compressed": false + }, + "reserved_meta_valid": true, + "metadata_version": 13, + "core_version": "polkadot-9070 (parity-polkadot-0.tx7.au0)", + "proposal_hash": "0x424ac5063ce844b878cd418e7d4c0e5518a6323ec0c54f744b1fb44a2ab24dcd", + "ipfs_hash": "QmeBgekBhZHNCkrayDgQaLXfAoLibS5Eq2fyEv4rzbttTo", + "blake2_256": "0xc5daf28ebf7f23c8de92a99a6c15b84abeaf12d226542e7504febaf0d1484e05" + } + }, + "compressed": {} + } +}"#; + +#[cfg(test)] +pub const SAMPLE_V4: &str = r#"{}"#; diff --git a/lib/src/srtool_tag.rs b/lib/src/srtool_tag.rs new file mode 100644 index 0000000..70155b0 --- /dev/null +++ b/lib/src/srtool_tag.rs @@ -0,0 +1,54 @@ +use crate::rustc_version::RustcVersion; +use semver::Version; +use std::str::FromStr; + +#[derive(Debug, PartialEq)] +pub struct SrtoolTag { + /// This is the srtool version: nightly-2021-03-15 or 1.53.0 for instance. + pub rustc: RustcVersion, + + /// This is the version of srtool itself. Typicaly something like 0.9.25 + pub srtool: Option, +} + +impl std::fmt::Display for SrtoolTag { + fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + todo!() + } +} + +impl FromStr for SrtoolTag { + type Err = String; + + fn from_str(s: &str) -> Result { + let mut splitter = s.split('-'); + let rustc = splitter.next().unwrap_or(""); + let srtool = splitter.next().unwrap_or(""); + + Ok(Self { rustc: RustcVersion::from_str(rustc).unwrap(), srtool: Version::from_str(srtool).ok() }) + } +} + +impl SrtoolTag { + /// whether the tag is fully qualified (such as 1.53.0-0.9.15) or not (such as 1.53.0) + pub fn is_fully_qualified(&self) -> bool { + self.srtool.is_some() + } +} + +#[cfg(test)] +mod test_srtooltah { + use super::*; + + #[test] + fn test_from() { + let tag = SrtoolTag::from_str("1.53.0-0.9.15").unwrap(); + println!("tag = {:?}", tag); + assert!(tag.is_fully_qualified()); + assert!(tag.rustc == RustcVersion::Stable(Version::from_str("1.53.0").unwrap())); + + let tag = SrtoolTag::from_str("1.53.0").unwrap(); + println!("tag = {:?}", tag); + assert!(!tag.is_fully_qualified()); + } +} From d2bb1cf252e76b87e11b1ef2163a88dfe97d6bff Mon Sep 17 00:00:00 2001 From: Chevdor Date: Tue, 20 Jul 2021 19:09:15 +0200 Subject: [PATCH 05/10] wip: refactor and functionnalities to find the runtime --- Cargo.lock | 10 ++ cli/src/main.rs | 108 ++++++------- lib/Cargo.toml | 1 + lib/src/digest/digest_json.rs | 46 ++++++ lib/src/{ => digest}/digest_source.rs | 0 lib/src/{ => digest}/digest_v1.rs | 18 ++- lib/src/{ => digest}/digest_v2.rs | 20 +-- lib/src/digest/mod.rs | 11 ++ .../versionned_digest.rs} | 32 +++- lib/src/digest_json.rs | 60 ------- lib/src/lib.rs | 12 +- lib/src/run_specs.rs | 39 ++++- lib/src/runner.rs | 153 ++++++++++++++++++ lib/src/runtime_crate.rs | 148 +++++++++++++++++ lib/src/samples.rs | 6 +- 15 files changed, 519 insertions(+), 145 deletions(-) create mode 100644 lib/src/digest/digest_json.rs rename lib/src/{ => digest}/digest_source.rs (100%) rename lib/src/{ => digest}/digest_v1.rs (68%) rename lib/src/{ => digest}/digest_v2.rs (87%) create mode 100644 lib/src/digest/mod.rs rename lib/src/{digest.rs => digest/versionned_digest.rs} (77%) delete mode 100644 lib/src/digest_json.rs create mode 100644 lib/src/runner.rs create mode 100644 lib/src/runtime_crate.rs diff --git a/Cargo.lock b/Cargo.lock index 4778499..7cb21a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -525,6 +525,7 @@ dependencies = [ "semver", "serde", "serde_json", + "toml", "ureq", ] @@ -578,6 +579,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +[[package]] +name = "toml" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +dependencies = [ + "serde", +] + [[package]] name = "treeline" version = "0.1.0" diff --git a/cli/src/main.rs b/cli/src/main.rs index ca7e5a8..b82cebb 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -38,13 +38,11 @@ fn main() { const ONE_HOUR: u64 = 60 * 60; let tag = get_image_tag(Some(ONE_HOUR)).expect("Issue getting the image tag"); - info!("Using {}:{}", image, tag); - let command = match opts.subcmd { + match opts.subcmd { SubCommand::Pull(_) => { - println!("Found {tag}, we will be using {image}:{tag} for the build", tag = tag, image = image); - format!("docker pull {image}:{tag}", image = image, tag = tag,) + Runner::pull(&image, &tag); } SubCommand::Build(build_opts) => { @@ -57,12 +55,18 @@ fn main() { let runtime_dir = build_opts.runtime_dir.unwrap_or_else(|| PathBuf::from(&default_runtime_dir)); let tmpdir = env::temp_dir().join("cargo"); let digest = get_image_digest(&image, &tag).unwrap_or_default(); - let cache_mount = if !build_opts.no_cache { + let package = build_opts.package; + let profile = build_opts.profile; + let workdir = fs::canonicalize(&build_opts.path).unwrap(); + let _cache_mount = if !build_opts.no_cache { format!("-v {tmpdir}:/cargo-home", tmpdir = tmpdir.display()) } else { String::new() }; + let search_info = RuntimeCrateSearchInfo { workdir: workdir.to_owned(), options: None }; + let _rtm_crate = RuntimeCrate::search(&search_info); + debug!("app: '{}'", &app); debug!("json: '{}'", &json); debug!("chain: '{}'", &chain); @@ -72,58 +76,48 @@ fn main() { debug!("digest: '{}'", &digest); debug!("no-cache: '{}'", build_opts.no_cache); - let path = fs::canonicalize(&build_opts.path).unwrap(); - - format!( - "docker run --name srtool --rm \ - -e PACKAGE={package} \ - -e RUNTIME_DIR={runtime_dir} \ - -e BUILD_OPTS={c_build_opts} \ - -e DEFAULT_FEATURES={default_features} \ - -e PROFILE={profile} \ - -e IMAGE={digest} \ - -v {dir}:/build \ - {cache_mount} \ - {image}:{tag} build{app}{json}", - package = build_opts.package, - dir = path.display(), - cache_mount = cache_mount, - image = image, - tag = tag, - runtime_dir = runtime_dir.display(), - c_build_opts = build_opts.build_opts.unwrap_or_else(|| String::from("")), - default_features = build_opts.default_features.unwrap_or_else(|| String::from("")), - profile = build_opts.profile, - json = json, - app = app, - digest = digest, - ) + let specs = RunSpecs::new(&package, &runtime_dir, &profile, &image, &tag); + let opts = srtool_lib::BuildOpts { json: json == "json", app: app == "app", workdir }; + Runner::build(&specs, &opts); + + // format!( + // "docker run --name srtool --rm \ + // -e PACKAGE={package} \ + // -e RUNTIME_DIR={runtime_dir} \ + // -e BUILD_OPTS={c_build_opts} \ + // -e DEFAULT_FEATURES={default_features} \ + // -e PROFILE={profile} \ + // -e IMAGE={digest} \ + // -v {dir}:/build \ + // {cache_mount} \ + // {image}:{tag} build{app}{json}", + // package = build_opts.package, + // dir = path.display(), + // cache_mount = cache_mount, + // image = image, + // tag = tag, + // runtime_dir = runtime_dir.display(), + // c_build_opts = build_opts.build_opts.unwrap_or_else(|| String::from("")), + // default_features = build_opts.default_features.unwrap_or_else(|| String::from("")), + // profile = build_opts.profile, + // json = json, + // app = app, + // digest = digest, + // ) } SubCommand::Info(info_opts) => { - let path = fs::canonicalize(&info_opts.path).unwrap(); + // let path = fs::canonicalize(&info_opts.path).unwrap(); let chain = info_opts.package.replace("-runtime", ""); let default_runtime_dir = format!("runtime/{}", chain); let runtime_dir = info_opts.runtime_dir.unwrap_or_else(|| PathBuf::from(&default_runtime_dir)); - debug!("chain: '{}'", &chain); - debug!("default_runtime_dir: '{}'", &default_runtime_dir); - debug!("runtime_dir: '{}'", &runtime_dir.display()); - - format!( - "docker run --name srtool --rm \ - -v {dir}:/build \ - -e RUNTIME_DIR={runtime_dir} \ - {image}:{tag} info", - dir = path.display(), - runtime_dir = runtime_dir.display(), - image = image, - tag = tag, - ) + let specs = RunSpecs::new("", &runtime_dir, "release", &image, &tag); + Runner::info(&specs, &runtime_dir); } SubCommand::Version(_) => { - format!("docker run --name srtool --rm {image}:{tag} version", image = image, tag = tag,) + Runner::version(&image, &tag); } SubCommand::Verify(verify_opts) => { @@ -132,22 +126,16 @@ fn main() { let reader = BufReader::new(file); let content: V2 = serde_json::from_reader(reader).unwrap(); let digest_json = json!({ "V2": content }); - // let digest = Digest::from - // debug!("digest = {:#?}", digest); - // let specs = digest.get_run_specs(); - // debug!("specs = {:#?}", specs); + let digest = DigestJson::load(json!(digest_json)).unwrap(); + debug!("digest = {:#?}", digest); + let specs = digest.get_run_specs().unwrap(); + debug!("specs = {:#?}", specs); + let build_opts = srtool_lib::BuildOpts { json: true, app: true, workdir: "/projects/polkadot".into() }; + + Runner::build(&specs, &build_opts); todo!() } }; - - debug!("command = {:?}", command); - - if cfg!(target_os = "windows") { - Command::new("cmd").args(&["/C", command.as_str()]).output().expect("failed to execute process"); - } else { - let _ = - Command::new("sh").arg("-c").arg(command).spawn().expect("failed to execute process").wait_with_output(); - } } #[cfg(test)] diff --git a/lib/Cargo.toml b/lib/Cargo.toml index b03808f..b4c8df0 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -11,4 +11,5 @@ log = "0.4" semver = {version = "1.0", features = ["serde"]} serde = {version = "1.0", features = ["derive"]} serde_json = "1.0" +toml = "0.5.8" ureq = "2.1" diff --git a/lib/src/digest/digest_json.rs b/lib/src/digest/digest_json.rs new file mode 100644 index 0000000..5a94efc --- /dev/null +++ b/lib/src/digest/digest_json.rs @@ -0,0 +1,46 @@ +use super::digest_source::DigestSource; +use super::versionned_digest::Digest; +use serde_json::Result; +use serde_json::Value; +type Json = Value; + +pub struct DigestJson {} + +impl DigestSource for DigestJson { + fn load(src: Json) -> Result { + let digest: Digest = serde_json::from_str(&src.to_string())?; + Ok(digest) + } +} + +#[cfg(test)] +mod test_super { + use super::*; + use crate::{ + digest::digest_v2, + samples::{SAMPLE_V1, SAMPLE_V2}, + }; + use serde_json::json; + + #[test] + fn test_v1() { + let v1: Value = serde_json::from_str(SAMPLE_V1).unwrap(); + let digest = DigestJson::load(json!({ "V1": v1 })).unwrap(); + + match digest { + Digest::V1(v1) => assert!(v1.src == "git"), + Digest::V2(v2) => assert!(v2.info.src == digest_v2::Source::Git), + } + } + + #[test] + fn test_v2() { + let v2: Value = serde_json::from_str(SAMPLE_V2).unwrap(); + let digest = DigestJson::load(json!({ "V2": v2 })).unwrap(); + + match digest { + Digest::V1(v1) => assert!(v1.src == "git"), + Digest::V2(v2) => assert!(v2.info.src == digest_v2::Source::Git), + } + } +} diff --git a/lib/src/digest_source.rs b/lib/src/digest/digest_source.rs similarity index 100% rename from lib/src/digest_source.rs rename to lib/src/digest/digest_source.rs diff --git a/lib/src/digest_v1.rs b/lib/src/digest/digest_v1.rs similarity index 68% rename from lib/src/digest_v1.rs rename to lib/src/digest/digest_v1.rs index f4c33ce..85d5b4f 100644 --- a/lib/src/digest_v1.rs +++ b/lib/src/digest/digest_v1.rs @@ -1,8 +1,20 @@ use semver::Version; -use serde::{Deserialize, Serialize}; +use serde::{de, Deserialize, Deserializer, Serialize}; +use std::fmt::Display; +use std::str::FromStr; //TODO: in V2, in order to NOT break compatibility, some fields are duplicated. That must be reworked. The profile for instance should be in the Context only. +fn from_str<'de, T, D>(deserializer: D) -> Result +where + T: FromStr, + T::Err: Display, + D: Deserializer<'de>, +{ + let s = String::deserialize(deserializer)?; + T::from_str(&s).map_err(de::Error::custom) +} + /// A srtool digest. The schema of the data srtool produces may /// change over time. This struct can load all version and make /// the common and relevant data available. @@ -18,7 +30,9 @@ pub struct V1 { pub(crate) rustc: String, pub(crate) pkg: String, pub(crate) tmsp: String, - pub(crate) size: uisze, + + #[serde(deserialize_with = "from_str")] + pub(crate) size: usize, pub(crate) prop: String, pub(crate) ipfs: String, pub(crate) sha256: String, diff --git a/lib/src/digest_v2.rs b/lib/src/digest/digest_v2.rs similarity index 87% rename from lib/src/digest_v2.rs rename to lib/src/digest/digest_v2.rs index 1e205f0..6ae9bd6 100644 --- a/lib/src/digest_v2.rs +++ b/lib/src/digest/digest_v2.rs @@ -1,3 +1,5 @@ +use std::path::PathBuf; + use semver::Version; use serde::{Deserialize, Serialize}; @@ -29,25 +31,25 @@ pub struct GitInfo { #[derive(Debug, PartialEq, Serialize, Deserialize)] pub struct Info { /// Information about the tooling used for the build. - generator: Generator, + pub(crate) generator: Generator, /// Whether the build from an Archive or from a Git repo. - src: Source, + pub(crate) src: Source, /// The version of the crate/package to build - version: Version, + pub(crate) version: Version, /// Optionnal Git information if the src was Git - git: Option, + pub(crate) git: Option, /// Rust compiler version - rustc: String, + pub(crate) rustc: String, /// Package - pkg: String, + pub(crate) pkg: String, /// Profile. Always 'release'. - profile: String, + pub(crate) profile: String, } #[derive(Debug, PartialEq, Serialize, Deserialize)] @@ -62,7 +64,7 @@ pub struct DockerContext { #[derive(Debug, PartialEq, Serialize, Deserialize)] pub struct Context { pub(crate) docker: DockerContext, - pub(crate) runtime_dir: String, + pub(crate) runtime_dir: PathBuf, pub(crate) package: String, pub(crate) profile: String, } @@ -75,6 +77,4 @@ pub struct Context { pub struct V2 { pub(crate) info: Info, pub(crate) context: Context, - #[serde(alias = "src")] - pub(crate) source: Source, } diff --git a/lib/src/digest/mod.rs b/lib/src/digest/mod.rs new file mode 100644 index 0000000..35a5a45 --- /dev/null +++ b/lib/src/digest/mod.rs @@ -0,0 +1,11 @@ +mod digest_json; +mod digest_source; +mod digest_v1; +mod digest_v2; +mod versionned_digest; + +pub use digest_json::*; +pub use digest_source::*; +pub use digest_v1::*; +pub use digest_v2::*; +pub use versionned_digest::*; diff --git a/lib/src/digest.rs b/lib/src/digest/versionned_digest.rs similarity index 77% rename from lib/src/digest.rs rename to lib/src/digest/versionned_digest.rs index 4c16625..836c6cb 100644 --- a/lib/src/digest.rs +++ b/lib/src/digest/versionned_digest.rs @@ -7,21 +7,25 @@ // TODO: The code for the srtool digest needs to be moved under srtool-cargo once published. -use std::str::FromStr; - -use crate::{digest_v2::V2, run_specs::RunSpecs}; +use super::{V1, V2}; +use crate::run_specs::RunSpecs; use semver::{Version, VersionReq}; use serde::{Deserialize, Serialize}; use serde_json::Value; +use std::str::FromStr; #[derive(Debug, PartialEq, Serialize, Deserialize)] pub enum Digest { + V1(V1), V2(V2), } +/// Use a DigestSource such as DigestJson to load a Digest impl Digest { pub fn get_run_specs(&self) -> Result { match self { + // TODO: what we could do for V1 if really needed if to let the user provide the missing information + Digest::V1(_v1) => panic!("Older V1 digests do not contain enough information to generate runspecs"), Digest::V2(v2) => Ok(RunSpecs { package: v2.context.package.to_owned(), runtime_dir: v2.context.runtime_dir.to_owned(), @@ -36,7 +40,7 @@ impl Digest { } } - pub fn get_version(json: Value) -> Option { + fn get_version(json: Value) -> Option { let version_v1 = &json["gen"].as_str().unwrap_or_default().split('v').nth(1); let version_v2 = &json["info"]["generator"]["version"].as_str(); @@ -78,8 +82,10 @@ impl From for Digest { #[cfg(test)] mod test_digest { + use serde_json::json; + use super::*; - use crate::samples::*; + use crate::{samples::*, DigestJson, DigestSource}; #[test] fn test_version_from_json_v1() { @@ -104,4 +110,20 @@ mod test_digest { let v4: Value = serde_json::from_str(SAMPLE_V4).unwrap(); assert_eq!(Digest::get_version(v4), None); } + + #[test] + #[should_panic] + fn test_get_run_specs_v1() { + let v1: Value = serde_json::from_str(SAMPLE_V1).unwrap(); + let digest = DigestJson::load(json!({ "V1": v1 })).unwrap(); + let _rs = digest.get_run_specs(); + } + + #[test] + fn test_get_run_specs_v2() { + let v2: Value = serde_json::from_str(SAMPLE_V2).unwrap(); + let digest = DigestJson::load(json!({ "V2": v2 })).unwrap(); + let rs = digest.get_run_specs(); + println!("rs = {:#?}", rs); + } } diff --git a/lib/src/digest_json.rs b/lib/src/digest_json.rs deleted file mode 100644 index 7bd1d1b..0000000 --- a/lib/src/digest_json.rs +++ /dev/null @@ -1,60 +0,0 @@ -use crate::{digest::Digest, digest_source::DigestSource}; -use serde_json::Result; -use serde_json::Value; -type Json = Value; - -pub struct DigestJson {} - -impl DigestSource for DigestJson { - fn load(src: Json) -> Result { - let digest: Digest = serde_json::from_str(&src.to_string())?; - Ok(digest) - } -} - -#[cfg(test)] -mod test_super { - use super::*; - use crate::digest_v2; - use serde_json::json; - - #[test] - fn test_() { - let json = json!({ - "V2": { - "context": { - "docker": { - "image": "paritytech/srtool", - "full_tag": "1.53.0-0.9.15", - }, - "runtime_dir": "runtime/polkadot", - "package": "polkadot-runtime", - "profile": "release", - }, - "source": "git", - "info": { - - "generator": { - "name": "srtool", - "version": "0.9.15", - }, - "src": "git", - "version": "0.9.7", - "git": { - "commit": "5d35bac7408a4cb12a578764217d06f3920b36aa", - "tag": "v0.9.7-rc3", - "branch": "heads/v0.9.7-rc3", - }, - "rustc": "rustc 1.53.0 (53cb7b09b 2021-06-17)", - "pkg": "polkadot-runtime", - "profile": "release", - }, - } - }); - let digest = DigestJson::load(json).unwrap(); - - match digest { - Digest::V2(v2) => assert!(v2.source == digest_v2::Source::Git), - } - } -} diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 445a1e5..47d118c 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -1,16 +1,16 @@ mod digest; -mod digest_json; -mod digest_source; -mod digest_v2; + mod run_specs; +mod runner; +mod runtime_crate; mod rustc_version; mod samples; mod srtool_tag; pub use digest::*; -pub use digest_json::*; -pub use digest_source::*; -pub use digest_v2::*; +pub use run_specs::*; +pub use runner::*; +pub use runtime_crate::*; pub use srtool_tag::*; use log::{debug, info}; diff --git a/lib/src/run_specs.rs b/lib/src/run_specs.rs index c71e8f4..622a208 100644 --- a/lib/src/run_specs.rs +++ b/lib/src/run_specs.rs @@ -2,12 +2,15 @@ //! However, V2 does not contain enough in the Context and has some of the information under //! the `Info` key. So we bring everything together as `RunSpecs`. +use std::path::{Path, PathBuf}; + +#[derive(Debug)] pub struct RunSpecs { /// Name of the crate of the runtime pub(crate) package: String, /// Path to the runtime crate relative to the root of the repository - pub(crate) runtime_dir: String, + pub(crate) runtime_dir: PathBuf, /// Usually `release` pub(crate) profile: String, @@ -23,3 +26,37 @@ pub struct RunSpecs { pub(crate) tag: String, pub(crate) cache_mount: bool, } + +impl RunSpecs { + pub fn new(package: &str, runtime_dir: &Path, profile: &str, image: &str, tag: &str) -> Self { + Self { + package: package.to_string(), + runtime_dir: runtime_dir.to_owned(), + profile: profile.to_string(), + image: image.to_string(), + tag: tag.to_string(), + image_sha256: String::new(), // TODO + cargo_build_opts: Vec::new(), // TODO, + default_features: Vec::new(), // TODO + cache_mount: true, // TODO + } + } +} + +#[cfg(test)] +/// Default is only used as convenience for the tests. +impl Default for RunSpecs { + fn default() -> Self { + Self { + package: "polkadot-runtime".to_string(), + runtime_dir: PathBuf::from("runtime/polkadot"), + profile: "release".to_string(), + image: "paritytech/srtool".to_string(), + image_sha256: String::new(), + cargo_build_opts: vec![], + default_features: vec![], + tag: "1.53.0-0.9.15".to_string(), + cache_mount: true, + } + } +} diff --git a/lib/src/runner.rs b/lib/src/runner.rs new file mode 100644 index 0000000..b6e64e5 --- /dev/null +++ b/lib/src/runner.rs @@ -0,0 +1,153 @@ +//! The runner is effectively a wrapper around docker + +use crate::{get_image_digest, run_specs::RunSpecs}; +use log::debug; +use std::{ + env, fs, + path::{Path, PathBuf}, + process::Command, +}; + +pub struct BuildOpts { + pub json: bool, + pub app: bool, + pub workdir: PathBuf, +} + +pub struct Runner; + +impl Runner { + /// Pulls the image + pub fn pull(image: &str, tag: &str) { + debug!("Found {tag}, we will be using {image}:{tag} for the build", tag = tag, image = image); + let cmd = format!("docker pull {image}:{tag}", image = image, tag = tag); + Runner::run(cmd); + } + + /// Invoke the build + pub fn build(specs: &RunSpecs, opts: &BuildOpts) { + println!("Found {tag}, we will be using {image}:{tag} for the build", tag = specs.tag, image = specs.image); + + let app = if opts.app { " --app" } else { "" }; + let json = if opts.json { " --json" } else { "" }; + let chain = specs.package.replace("-runtime", ""); + let default_runtime_dir = format!("runtime/{}", chain); + let runtime_dir = specs.runtime_dir.to_owned(); + let tmpdir = env::temp_dir().join("cargo"); + let digest = get_image_digest(&specs.image, &specs.tag).unwrap_or_default(); + let cache_mount = if specs.cache_mount { + format!("-v {tmpdir}:/cargo-home", tmpdir = tmpdir.display()) + } else { + String::new() + }; + + debug!("app: '{}'", &app); + debug!("json: '{}'", &json); + debug!("chain: '{}'", &chain); + debug!("default_runtime_dir: '{}'", &default_runtime_dir); + debug!("runtime_dir: '{}'", &runtime_dir.display()); + debug!("tmpdir: '{}'", &tmpdir.display()); + debug!("digest: '{}'", &digest); + debug!("cache-mount: '{}'", specs.cache_mount); + + let path = fs::canonicalize(&opts.workdir).unwrap(); + + let cmd = format!( + "docker run --name srtool --rm \ + -e PACKAGE={package} \ + -e RUNTIME_DIR={runtime_dir} \ + -e BUILD_OPTS={c_build_opts} \ + -e DEFAULT_FEATURES={default_features} \ + -e PROFILE={profile} \ + -e IMAGE={digest} \ + -v {dir}:/build \ + {cache_mount} \ + {image}:{tag} build{app}{json}", + package = specs.package, + dir = path.display(), + cache_mount = cache_mount, + image = specs.image, + tag = specs.tag, + runtime_dir = runtime_dir.display(), + c_build_opts = specs.cargo_build_opts.join(" "), + default_features = specs.default_features.join(" "), + profile = specs.profile, + json = json, + app = app, + digest = digest, + ); + Runner::run(cmd); + } + + /// Show infos + pub fn info(specs: &RunSpecs, workdir: &Path) { + // let path = fs::canonicalize(&workdir).unwrap(); + let chain = specs.package.replace("-runtime", ""); + // let default_runtime_dir = format!("runtime/{}", chain); + + debug!("specs: '{:#?}'", &specs); + debug!("chain: '{}'", &chain); + + let cmd = format!( + "docker run --name srtool --rm \ + -v {dir}:/build \ + -e RUNTIME_DIR={runtime_dir} \ + {image}:{tag} info", + dir = workdir.display(), + runtime_dir = specs.runtime_dir.display(), + image = specs.image, + tag = specs.tag, + ); + Runner::run(cmd); + } + + /// Get version + pub fn version(image: &str, tag: &str) { + let cmd = format!("docker run --name srtool --rm {image}:{tag} version", image = image, tag = tag); + Runner::run(cmd); + } + + /// Run the docker command that is passed + fn run(cmd: String) { + if cfg!(target_os = "windows") { + Command::new("cmd").args(&["/C", cmd.as_str()]).output().expect("failed to execute process"); + } else { + let _ = + Command::new("sh").arg("-c").arg(cmd).spawn().expect("failed to execute process").wait_with_output(); + } + } +} + +#[cfg(test)] +mod test_runner { + use super::*; + + #[test] + fn test_pull() { + let specs = RunSpecs::default(); + Runner::pull(&specs.image, &specs.tag); + } + + #[test] + fn test_version() { + let specs = RunSpecs::default(); + Runner::version(&specs.image, &specs.tag); + } + + #[test] + #[ignore = "local data"] + fn test_info() { + let specs = RunSpecs::default(); + let workdir = PathBuf::from("/projects/polkadot"); + Runner::info(&specs, &workdir); + } + + #[test] + #[ignore = "local data + long running"] + fn test_build() { + let specs = RunSpecs::default(); + let workdir = PathBuf::from("/projects/polkadot"); + let opts = BuildOpts { json: true, app: true, workdir }; + Runner::build(&specs, &opts); + } +} diff --git a/lib/src/runtime_crate.rs b/lib/src/runtime_crate.rs new file mode 100644 index 0000000..79114e6 --- /dev/null +++ b/lib/src/runtime_crate.rs @@ -0,0 +1,148 @@ +use std::{fs, path::PathBuf}; +use toml::Value; + +use std::error::Error; + +/// This sctruct holds the information required to know which +/// runtime to build. +#[derive(Debug)] +pub struct RuntimeCrate { + workdir: PathBuf, + runtime_dir: PathBuf, + package: String, + chain: String, +} + +#[derive(Debug)] +pub enum RuntimeCrateSearchOption { + RuntimeDir(String), + Package(String), + ChainName(String), +} + +/// This is the data to provide to search for the runtime. +/// Note that the only reliable way is to pass workdir + runtime_dir. +/// All other options and combinations have more chances to fail. +#[derive(Debug)] +pub struct RuntimeCrateSearchInfo { + pub workdir: PathBuf, + pub options: Option, +} + +impl RuntimeCrate { + /// This function helps find the runtime crate based on *some* information. + /// The result will be Ok if and only if the search critera lead to one single + /// result. In any other cases, it will return an error. + pub fn search(input: &RuntimeCrateSearchInfo) -> Result> { + match &input.options { + Some(opts) => match opts { + // This is the less fuzzy option + RuntimeCrateSearchOption::RuntimeDir(runtime_dir) => { + let runtime_dir = PathBuf::from(runtime_dir); + let cargo_toml = input.workdir.join(&runtime_dir).join("Cargo.toml"); + let toml_content: Value = fs::read_to_string(cargo_toml)?.parse()?; + let package = toml_content["package"]["name"].as_str().expect("Failed getting the package name"); + let chain = package.replace("-runtime", ""); + Ok(RuntimeCrate { + workdir: input.workdir.to_owned(), + runtime_dir, + package: package.to_string(), + chain, + }) + } + RuntimeCrateSearchOption::Package(package) => { + let chain = package.replace("-runtime", ""); + let runtime_dir = PathBuf::from("runtime").join(&chain); + let cargo_toml = input.workdir.join(&runtime_dir).join("Cargo.toml"); + let _: Value = fs::read_to_string(cargo_toml)?.parse()?; + Ok(RuntimeCrate { + workdir: input.workdir.to_owned(), + runtime_dir, + package: package.to_string(), + chain, + }) + } + RuntimeCrateSearchOption::ChainName(chain) => { + let package = format!("{}-runtime", chain); + let runtime_dir = PathBuf::from("runtime").join(&chain); + let cargo_toml = input.workdir.join(&runtime_dir).join("Cargo.toml"); + let _: Value = fs::read_to_string(cargo_toml)?.parse()?; + + Ok(RuntimeCrate { + workdir: input.workdir.to_owned(), + runtime_dir, + package, + chain: chain.to_string(), + }) + } + }, + None => todo!("This feature is not implemented yet, please pass one other search criteria"), + } + } +} + +#[cfg(test)] +mod test_runtime_crate { + use super::*; + + #[test] + #[ignore = "local data"] + #[should_panic] // Not implemented yet, this may end up passing later + fn test_search_workdir_only() { + let _ = RuntimeCrate::search(&RuntimeCrateSearchInfo { workdir: "/projects/polkadot".into(), options: None }); + } + + #[test] + #[ignore = "local data"] + #[should_panic] // Not implemented yet + fn test_search_bad_workdir_only() { + // Should fail for now + let _ = RuntimeCrate::search(&RuntimeCrateSearchInfo { workdir: "/tmp".into(), options: None }); + } + + #[test] + #[ignore = "local data"] + fn test_search_workdir_runtime() { + // The best way + let res = RuntimeCrate::search(&RuntimeCrateSearchInfo { + workdir: "/projects/polkadot".into(), + options: Some(RuntimeCrateSearchOption::RuntimeDir("runtime/polkadot".into())), + }); + assert!(res.is_ok()); + println!("res = {:#?}", res); + } + + #[test] + #[ignore = "local data"] + fn test_search_workdir_bad_runtime_dir() { + // Should fail + let res = RuntimeCrate::search(&RuntimeCrateSearchInfo { + workdir: "/projects/polkadot".into(), + options: Some(RuntimeCrateSearchOption::RuntimeDir("foobar".into())), + }); + assert!(res.is_err()); + println!("res = {:#?}", res); + } + + #[test] + #[ignore = "local data"] + fn test_search_workdir_package() { + let res = RuntimeCrate::search(&RuntimeCrateSearchInfo { + workdir: "/projects/polkadot".into(), + options: Some(RuntimeCrateSearchOption::Package("polkadot-runtime".into())), + }); + assert!(res.is_ok()); + println!("res = {:#?}", res); + } + + #[test] + #[ignore = "local data"] + fn test_search_workdir_chain_name() { + let res = RuntimeCrate::search(&RuntimeCrateSearchInfo { + workdir: "/projects/polkadot".into(), + options: Some(RuntimeCrateSearchOption::ChainName("polkadot".into())), + }); + assert!(res.is_ok()); + println!("res = {:#?}", res); + } +} diff --git a/lib/src/samples.rs b/lib/src/samples.rs index 4d6441c..50c7ff4 100644 --- a/lib/src/samples.rs +++ b/lib/src/samples.rs @@ -120,4 +120,8 @@ pub const SAMPLE_V3: &str = r#"{ }"#; #[cfg(test)] -pub const SAMPLE_V4: &str = r#"{}"#; +pub const SAMPLE_V4: &str = r#"{ + "V4": { + "version": "1.2.3" + } +}"#; From c04f2e2870d1c8ba18ed8f7faf04fcb23920bc58 Mon Sep 17 00:00:00 2001 From: Chevdor Date: Wed, 21 Jul 2021 14:59:35 +0200 Subject: [PATCH 06/10] test: add sample digest --- data/test_digest_v2.json | 88 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 data/test_digest_v2.json diff --git a/data/test_digest_v2.json b/data/test_digest_v2.json new file mode 100644 index 0000000..6d50619 --- /dev/null +++ b/data/test_digest_v2.json @@ -0,0 +1,88 @@ +{ + "info": { + "generator": { + "name": "srtool", + "version": "0.9.15" + }, + "src": "git", + "version": "0.9.7", + "git": { + "commit": "5d35bac7408a4cb12a578764217d06f3920b36aa", + "tag": "v0.9.7-rc3", + "branch": "heads/v0.9.7-rc3" + }, + "rustc": "rustc 1.53.0 (53cb7b09b 2021-06-17)", + "pkg": "polkadot-runtime", + "profile": "release" + }, + "context": { + "package": "polkadot-runtime", + "runtime_dir": "runtime/polkadot", + "docker": { + "image": "chevdor/srtool", + "tag": "1.53.0", + "digest": "sha256:31a302da3198ac5d9fe0beb5cb4b456552e8745544172dffa244c439750c0133" + }, + "profile": "release" + }, + "runtimes": { + "compact": { + "tmsp": "2021-06-29T16:12:24Z", + "size": "2093380", + "prop": "0x424ac5063ce844b878cd418e7d4c0e5518a6323ec0c54f744b1fb44a2ab24dcd", + "blake2_256": "0xc5daf28ebf7f23c8de92a99a6c15b84abeaf12d226542e7504febaf0d1484e05", + "ipfs": "QmeBgekBhZHNCkrayDgQaLXfAoLibS5Eq2fyEv4rzbttTo", + "sha256": "0x5f31cd25a9de645f278f18b008f38edad5b3253c1b94dc71a12da48c27dd1581", + "wasm": "runtime/polkadot/target/srtool/release/wbuild/polkadot-runtime/polkadot_runtime.compact.wasm", + "subwasm": { + "size": 2093380, + "compression": { + "size_compressed": 2093380, + "size_decompressed": 2093380, + "compressed": false + }, + "reserved_meta": [ + 109, + 101, + 116, + 97 + ], + "reserved_meta_valid": true, + "metadata_version": 13, + "core_version": "polkadot-9070 (parity-polkadot-0.tx7.au0)", + "proposal_hash": "0x424ac5063ce844b878cd418e7d4c0e5518a6323ec0c54f744b1fb44a2ab24dcd", + "ipfs_hash": "QmeBgekBhZHNCkrayDgQaLXfAoLibS5Eq2fyEv4rzbttTo", + "blake2_256": "0xc5daf28ebf7f23c8de92a99a6c15b84abeaf12d226542e7504febaf0d1484e05" + } + }, + "compressed": { + "tmsp": "2021-06-23T20:22:41Z", + "size": "613213", + "prop": "0x73470f4dcc83d491eac816248ac0c91557087f82d123db2c1a5ee098977b0d41", + "blake2_256": "0xb8708536646e506c95716bbbaa0a51edbaaf19070c8ba401ef4c0002b308224b", + "ipfs": "QmSX3Kho3TWrZUmywLiyWDwDgbuK9uvXixjuFjFREiaUGg", + "sha256": "0x9b30d5a053ce48f0ac5a3909a72fe9881a43c21d22f1462bb54f0de6e6e19288", + "wasm": "runtime/polkadot/target/srtool/release/wbuild/polkadot-runtime/polkadot_runtime.compact.compressed.wasm", + "subwasm": { + "size": 2087945, + "compression": { + "size_compressed": 613213, + "size_decompressed": 2087945, + "compressed": true + }, + "reserved_meta": [ + 109, + 101, + 116, + 97 + ], + "reserved_meta_valid": true, + "metadata_version": 13, + "core_version": "polkadot-9051 (parity-polkadot-0.tx7.au0)", + "proposal_hash": "0x73470f4dcc83d491eac816248ac0c91557087f82d123db2c1a5ee098977b0d41", + "ipfs_hash": "QmSX3Kho3TWrZUmywLiyWDwDgbuK9uvXixjuFjFREiaUGg", + "blake2_256": "0xb8708536646e506c95716bbbaa0a51edbaaf19070c8ba401ef4c0002b308224b" + } + } + } + } From c27fddbd5c4cb5c569823ae4b30cd9130e2272e5 Mon Sep 17 00:00:00 2001 From: Chevdor Date: Wed, 21 Jul 2021 15:00:44 +0200 Subject: [PATCH 07/10] feat: move code from main to runner and use crate detection --- cli/src/main.rs | 104 ++++++++-------------------- cli/src/opts.rs | 50 +++++++++---- lib/src/digest/digest_v2.rs | 23 ++++-- lib/src/digest/versionned_digest.rs | 5 +- lib/src/lib.rs | 2 +- lib/src/run_specs.rs | 36 +++++----- lib/src/runner.rs | 40 +++++++---- lib/src/runtime_crate.rs | 34 +++++++-- 8 files changed, 156 insertions(+), 138 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index b82cebb..e731540 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,12 +1,11 @@ mod opts; -use clap::{crate_version, Clap}; +use clap::{crate_name, crate_version, Clap}; use log::{debug, info}; use opts::*; use serde_json::json; use srtool_lib::*; use std::fs::File; use std::io::BufReader; -use std::path::PathBuf; use std::process::Command; use std::{env, fs}; @@ -20,7 +19,7 @@ fn handle_exit() { fn main() { env_logger::init(); - info!("Running srtool-cli v{}", crate_version!()); + info!("Running {} v{}", crate_name!(), crate_version!()); let opts: Opts = Opts::parse(); let image = opts.image; @@ -41,86 +40,39 @@ fn main() { info!("Using {}:{}", image, tag); match opts.subcmd { - SubCommand::Pull(_) => { - Runner::pull(&image, &tag); - } - - SubCommand::Build(build_opts) => { - println!("Found {tag}, we will be using {image}:{tag} for the build", tag = tag, image = image); - - let app = if build_opts.app { " --app" } else { "" }; - let json = if opts.json || build_opts.json { " --json" } else { "" }; - let chain = build_opts.package.replace("-runtime", ""); - let default_runtime_dir = format!("runtime/{}", chain); - let runtime_dir = build_opts.runtime_dir.unwrap_or_else(|| PathBuf::from(&default_runtime_dir)); - let tmpdir = env::temp_dir().join("cargo"); - let digest = get_image_digest(&image, &tag).unwrap_or_default(); - let package = build_opts.package; - let profile = build_opts.profile; - let workdir = fs::canonicalize(&build_opts.path).unwrap(); - let _cache_mount = if !build_opts.no_cache { - format!("-v {tmpdir}:/cargo-home", tmpdir = tmpdir.display()) - } else { - String::new() - }; - - let search_info = RuntimeCrateSearchInfo { workdir: workdir.to_owned(), options: None }; - let _rtm_crate = RuntimeCrate::search(&search_info); - - debug!("app: '{}'", &app); - debug!("json: '{}'", &json); - debug!("chain: '{}'", &chain); - debug!("default_runtime_dir: '{}'", &default_runtime_dir); - debug!("runtime_dir: '{}'", &runtime_dir.display()); - debug!("tmpdir: '{}'", &tmpdir.display()); - debug!("digest: '{}'", &digest); - debug!("no-cache: '{}'", build_opts.no_cache); - - let specs = RunSpecs::new(&package, &runtime_dir, &profile, &image, &tag); - let opts = srtool_lib::BuildOpts { json: json == "json", app: app == "app", workdir }; - Runner::build(&specs, &opts); - - // format!( - // "docker run --name srtool --rm \ - // -e PACKAGE={package} \ - // -e RUNTIME_DIR={runtime_dir} \ - // -e BUILD_OPTS={c_build_opts} \ - // -e DEFAULT_FEATURES={default_features} \ - // -e PROFILE={profile} \ - // -e IMAGE={digest} \ - // -v {dir}:/build \ - // {cache_mount} \ - // {image}:{tag} build{app}{json}", - // package = build_opts.package, - // dir = path.display(), - // cache_mount = cache_mount, - // image = image, - // tag = tag, - // runtime_dir = runtime_dir.display(), - // c_build_opts = build_opts.build_opts.unwrap_or_else(|| String::from("")), - // default_features = build_opts.default_features.unwrap_or_else(|| String::from("")), - // profile = build_opts.profile, - // json = json, - // app = app, - // digest = digest, - // ) - } + SubCommand::Pull(_) => Runner::pull(&image, &tag), + SubCommand::Version(_) => Runner::version(&image, &tag), SubCommand::Info(info_opts) => { - // let path = fs::canonicalize(&info_opts.path).unwrap(); - let chain = info_opts.package.replace("-runtime", ""); - let default_runtime_dir = format!("runtime/{}", chain); - let runtime_dir = info_opts.runtime_dir.unwrap_or_else(|| PathBuf::from(&default_runtime_dir)); + let workdir = fs::canonicalize(&info_opts.workdir).unwrap(); + let rtm_crate = + RuntimeCrate::search_flattened(&workdir, &info_opts.package, &info_opts.chain, &info_opts.runtime_dir) + .unwrap(); - let specs = RunSpecs::new("", &runtime_dir, "release", &image, &tag); - Runner::info(&specs, &runtime_dir); + let specs = RunSpecs::new(&rtm_crate.runtime_dir, "release", &image, &tag, None, false); + Runner::info(&specs, &rtm_crate.workdir); } - SubCommand::Version(_) => { - Runner::version(&image, &tag); + SubCommand::Build(build_opts) => { + let workdir = fs::canonicalize(&build_opts.workdir).unwrap(); + + let rtm_crate = RuntimeCrate::search_flattened( + &workdir, + &build_opts.package, + &build_opts.chain, + &build_opts.runtime_dir, + ) + .unwrap(); + + let specs = + RunSpecs::new(&rtm_crate.runtime_dir, &build_opts.profile, &image, &tag, None, !build_opts.no_cache); + let opts = srtool_lib::BuildOpts { json: build_opts.json, app: build_opts.app, workdir }; + Runner::build(&specs, &opts); } SubCommand::Verify(verify_opts) => { + // let workdir = fs::canonicalize(&verify_opts.workdir).unwrap(); + debug!("Digest from: {:?}", verify_opts.digest); let file = File::open(verify_opts.digest).unwrap(); let reader = BufReader::new(file); @@ -132,6 +84,8 @@ fn main() { debug!("specs = {:#?}", specs); let build_opts = srtool_lib::BuildOpts { json: true, app: true, workdir: "/projects/polkadot".into() }; + // let rtm_crate = RuntimeCrate::search_flattened(&workdir, &None, &None, &Some(specs.runtime_dir)).unwrap(); + Runner::build(&specs, &build_opts); todo!() } diff --git a/cli/src/opts.rs b/cli/src/opts.rs index 70baffe..03309c4 100644 --- a/cli/src/opts.rs +++ b/cli/src/opts.rs @@ -36,11 +36,11 @@ pub enum SubCommand { #[clap(version = crate_version!(), author = crate_authors!())] Pull(PullOpts), - /// Start a new srtool container to build your runtime + /// Start a new srtool container to build a Substrate based runtime #[clap(version = crate_version!(), author = crate_authors!())] Build(BuildOpts), - /// Provide information about the srtool container and your repo + /// Provide information about the srtool container and your repository #[clap(version = crate_version!(), author = crate_authors!())] Info(InfoOpts), @@ -64,9 +64,24 @@ pub struct PullOpts; /// Build opts #[derive(Clap)] pub struct BuildOpts { - /// Provide the runtime such as kusama-runtime, polkadot-runtime, etc... + /// If your runtime is not in the standard location runtime/ + /// and/or you use a name different than -runtime for your runtime crate, + /// you must pass this value to allow srtool find it. + #[clap(short, long, env = "RUNTIME_DIR", conflicts_with = "chain", conflicts_with = "package")] + pub runtime_dir: Option, + + /// If your runtime is in the standard location, you can simply pass its name here. + /// This is assuming your runtime crate is called -runtime and located under + /// `runtime/`. + #[clap(short, long, env = "CHAIN", conflicts_with = "package")] + pub chain: Option, + + /// Passing the `chain` argument is probably easier but this option is left + /// for compatibility reason. You may pass this value if and it formatted as + /// -runtime and your runtime crate is under `runtime/`. For other + /// case, please pass `runtime-dir` to disambiguate. #[clap(long, short, env = "PACKAGE")] - pub package: String, + pub package: Option, /// Enable json output, same than the global --json option #[clap(long, short)] @@ -81,12 +96,7 @@ pub struct BuildOpts { /// By default, srtool will work in the current folder. /// If your project is located in another location, you can pass it here. #[clap(index = 1, default_value = ".")] - pub path: PathBuf, - - /// If your runtime is not in the standard location runtime/ - /// you can pass this args to help srtool find it. - #[clap(short, long, env = "RUNTIME_DIR")] - pub runtime_dir: Option, + pub workdir: PathBuf, /// You may pass options to cargo directly here. WARNING, if you pass /// this value, the automatic build options for Kusama and Polkadot will @@ -102,7 +112,8 @@ pub struct BuildOpts { pub default_features: Option, /// The default profile to build runtimes is always `release`. - /// You may override the default with this flag. + /// You may override the default with this flag should you need it, + /// which is btw very unlikely. #[clap(long, env = "PROFILE", default_value = "release")] pub profile: String, @@ -119,11 +130,15 @@ pub struct InfoOpts { /// By default, srtool will work in the current folder. /// If your project is located in another location, you can pass it here. #[clap(index = 1, default_value = ".")] - pub path: PathBuf, + pub workdir: PathBuf, /// Provide the runtime such as kusama-runtime, polkadot-runtime, etc... - #[clap(long, short, env = "PACKAGE")] - pub package: String, + #[clap(long, short, env = "CHAIN", conflicts_with = "package", conflicts_with = "runtime-dir")] + pub chain: Option, + + /// Provide the runtime such as kusama-runtime, polkadot-runtime, etc... + #[clap(long, short, env = "PACKAGE", conflicts_with = "runtime-dir")] + pub package: Option, /// If your runtime is not in the standard location runtime/ /// you can pass this args to help srtool find it. @@ -140,6 +155,11 @@ pub struct VersionOpts; pub struct VerifyOpts { /// The path of the srtool digest (json) where most of the settings /// will be fetched to reproduce the exact same build. - #[clap(index = 1)] + #[clap(long, short, default_value = "digest.json", required = true)] pub digest: PathBuf, + + /// By default, srtool will work in the current folder. + /// If your project is located in another location, you can pass it here. + #[clap(index = 1, default_value = ".")] + pub workdir: PathBuf, } diff --git a/lib/src/digest/digest_v2.rs b/lib/src/digest/digest_v2.rs index 6ae9bd6..a389292 100644 --- a/lib/src/digest/digest_v2.rs +++ b/lib/src/digest/digest_v2.rs @@ -12,15 +12,15 @@ pub enum Source { #[derive(Debug, PartialEq, Serialize, Deserialize)] pub struct Generator { - name: String, - version: Version, + pub name: String, + pub version: Version, } #[derive(Debug, PartialEq, Serialize, Deserialize)] pub struct GitInfo { - commit: String, - tag: String, - branch: String, + pub commit: String, + pub tag: String, + pub branch: String, } //TODO: in V2, in order to NOT break compatibility, some fields are duplicated. That must be reworked. The profile for instance should be in the Context only. @@ -55,8 +55,8 @@ pub struct Info { #[derive(Debug, PartialEq, Serialize, Deserialize)] pub struct DockerContext { pub image: String, - #[serde(alias = "tag")] - pub full_tag: String, + pub tag: String, + pub digest: Option, } /// This struct describes all the information required @@ -78,3 +78,12 @@ pub struct V2 { pub(crate) info: Info, pub(crate) context: Context, } + +impl V2 { + /// The `full_tag` is made of -. + /// While using only will produce the same artifacts, we have no insurance + /// that another version of the srtool build provides the same output. + pub fn get_full_tag(&self) -> String { + format!("{}-{}", self.context.docker.tag.to_owned(), self.info.generator.version) + } +} diff --git a/lib/src/digest/versionned_digest.rs b/lib/src/digest/versionned_digest.rs index 836c6cb..2f1e38c 100644 --- a/lib/src/digest/versionned_digest.rs +++ b/lib/src/digest/versionned_digest.rs @@ -27,14 +27,13 @@ impl Digest { // TODO: what we could do for V1 if really needed if to let the user provide the missing information Digest::V1(_v1) => panic!("Older V1 digests do not contain enough information to generate runspecs"), Digest::V2(v2) => Ok(RunSpecs { - package: v2.context.package.to_owned(), runtime_dir: v2.context.runtime_dir.to_owned(), profile: v2.context.profile.to_owned(), image: v2.context.docker.image.to_owned(), - image_sha256: String::new(), // TODO + image_sha256: v2.context.docker.digest.to_owned(), cargo_build_opts: Vec::new(), // TODO default_features: Vec::new(), // TODO - tag: v2.context.docker.full_tag.to_owned(), + tag: v2.get_full_tag(), cache_mount: false, // TODO }), } diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 47d118c..7c0d2c9 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -99,7 +99,7 @@ pub fn get_image_digest(image: &str, tag: &str) -> Option { let output_str = String::from_utf8(output.unwrap().stdout).unwrap_or_else(|_| "".into()); let json: serde_json::Value = serde_json::from_str(&output_str).unwrap_or_default(); let digest_str = json[0]["RepoDigests"][0].as_str().unwrap_or_default(); - let digest = digest_str.split(':').nth(1); + let digest = digest_str.split('@').nth(1); digest.map(String::from) } diff --git a/lib/src/run_specs.rs b/lib/src/run_specs.rs index 622a208..2089ac1 100644 --- a/lib/src/run_specs.rs +++ b/lib/src/run_specs.rs @@ -6,39 +6,42 @@ use std::path::{Path, PathBuf}; #[derive(Debug)] pub struct RunSpecs { - /// Name of the crate of the runtime - pub(crate) package: String, - /// Path to the runtime crate relative to the root of the repository - pub(crate) runtime_dir: PathBuf, + pub runtime_dir: PathBuf, /// Usually `release` - pub(crate) profile: String, + pub profile: String, /// The docker image, ie: paritytech/srtool - pub(crate) image: String, + pub image: String, /// The digest of the docker image - pub(crate) image_sha256: String, - pub(crate) cargo_build_opts: Vec, - pub(crate) default_features: Vec, + pub image_sha256: Option, + pub cargo_build_opts: Vec, + pub default_features: Vec, - pub(crate) tag: String, - pub(crate) cache_mount: bool, + pub tag: String, + pub cache_mount: bool, } impl RunSpecs { - pub fn new(package: &str, runtime_dir: &Path, profile: &str, image: &str, tag: &str) -> Self { + pub fn new( + runtime_dir: &Path, + profile: &str, + image: &str, + tag: &str, + image_sha256: Option, + cache_mount: bool, + ) -> Self { Self { - package: package.to_string(), runtime_dir: runtime_dir.to_owned(), profile: profile.to_string(), image: image.to_string(), tag: tag.to_string(), - image_sha256: String::new(), // TODO + image_sha256, cargo_build_opts: Vec::new(), // TODO, default_features: Vec::new(), // TODO - cache_mount: true, // TODO + cache_mount, } } } @@ -48,11 +51,10 @@ impl RunSpecs { impl Default for RunSpecs { fn default() -> Self { Self { - package: "polkadot-runtime".to_string(), runtime_dir: PathBuf::from("runtime/polkadot"), profile: "release".to_string(), image: "paritytech/srtool".to_string(), - image_sha256: String::new(), + image_sha256: Some(String::from("sha256:31a302da3198ac5d9fe0beb5cb4b456552e8745544172dffa244c439750c0133")), cargo_build_opts: vec![], default_features: vec![], tag: "1.53.0-0.9.15".to_string(), diff --git a/lib/src/runner.rs b/lib/src/runner.rs index b6e64e5..f081da3 100644 --- a/lib/src/runner.rs +++ b/lib/src/runner.rs @@ -1,7 +1,7 @@ //! The runner is effectively a wrapper around docker -use crate::{get_image_digest, run_specs::RunSpecs}; -use log::debug; +use crate::{get_image_digest, run_specs::RunSpecs, RuntimeCrate}; +use log::{debug, info, trace, warn}; use std::{ env, fs, path::{Path, PathBuf}, @@ -19,18 +19,25 @@ pub struct Runner; impl Runner { /// Pulls the image pub fn pull(image: &str, tag: &str) { - debug!("Found {tag}, we will be using {image}:{tag} for the build", tag = tag, image = image); + trace!("pull()"); + + debug!("We will be pulling {image}:{tag}", tag = tag, image = image); let cmd = format!("docker pull {image}:{tag}", image = image, tag = tag); Runner::run(cmd); } /// Invoke the build pub fn build(specs: &RunSpecs, opts: &BuildOpts) { - println!("Found {tag}, we will be using {image}:{tag} for the build", tag = specs.tag, image = specs.image); + trace!("build()"); + let workdir = fs::canonicalize(&opts.workdir).unwrap(); + debug!("Workdir: {:?}", &workdir); + println!("We will be using {image}:{tag} for the build", tag = specs.tag, image = specs.image); + let rtm_crate = + RuntimeCrate::search_flattened(&workdir, &None, &None, &Some(specs.runtime_dir.to_owned())).unwrap(); let app = if opts.app { " --app" } else { "" }; let json = if opts.json { " --json" } else { "" }; - let chain = specs.package.replace("-runtime", ""); + let chain = rtm_crate.package.replace("-runtime", ""); let default_runtime_dir = format!("runtime/{}", chain); let runtime_dir = specs.runtime_dir.to_owned(); let tmpdir = env::temp_dir().join("cargo"); @@ -48,9 +55,18 @@ impl Runner { debug!("runtime_dir: '{}'", &runtime_dir.display()); debug!("tmpdir: '{}'", &tmpdir.display()); debug!("digest: '{}'", &digest); - debug!("cache-mount: '{}'", specs.cache_mount); - let path = fs::canonicalize(&opts.workdir).unwrap(); + if let Some(sha256) = &specs.image_sha256 { + if sha256 == &digest { + info!("Docker image digest matches: {}", &digest); + } else { + warn!("Docker image digests DO NOT match:"); + warn!(" - expected: {} ", &sha256); + warn!(" - found : {} ", &digest); + } + } + + debug!("cache-mount: '{}'", specs.cache_mount); let cmd = format!( "docker run --name srtool --rm \ @@ -63,8 +79,8 @@ impl Runner { -v {dir}:/build \ {cache_mount} \ {image}:{tag} build{app}{json}", - package = specs.package, - dir = path.display(), + package = rtm_crate.package, + dir = workdir.display(), cache_mount = cache_mount, image = specs.image, tag = specs.tag, @@ -81,12 +97,10 @@ impl Runner { /// Show infos pub fn info(specs: &RunSpecs, workdir: &Path) { - // let path = fs::canonicalize(&workdir).unwrap(); - let chain = specs.package.replace("-runtime", ""); - // let default_runtime_dir = format!("runtime/{}", chain); + // let rtm_crate = + // RuntimeCrate::search_flattened(&workdir, &None, &None, &Some(specs.runtime_dir.to_owned())).unwrap(); debug!("specs: '{:#?}'", &specs); - debug!("chain: '{}'", &chain); let cmd = format!( "docker run --name srtool --rm \ diff --git a/lib/src/runtime_crate.rs b/lib/src/runtime_crate.rs index 79114e6..e5e44fc 100644 --- a/lib/src/runtime_crate.rs +++ b/lib/src/runtime_crate.rs @@ -1,4 +1,7 @@ -use std::{fs, path::PathBuf}; +use std::{ + fs, + path::{Path, PathBuf}, +}; use toml::Value; use std::error::Error; @@ -7,15 +10,15 @@ use std::error::Error; /// runtime to build. #[derive(Debug)] pub struct RuntimeCrate { - workdir: PathBuf, - runtime_dir: PathBuf, - package: String, - chain: String, + pub workdir: PathBuf, + pub runtime_dir: PathBuf, + pub package: String, + pub chain: String, } #[derive(Debug)] pub enum RuntimeCrateSearchOption { - RuntimeDir(String), + RuntimeDir(PathBuf), Package(String), ChainName(String), } @@ -30,6 +33,23 @@ pub struct RuntimeCrateSearchInfo { } impl RuntimeCrate { + pub fn search_flattened( + workdir: &Path, + package: &Option, + chain: &Option, + runtime_dir: &Option, + ) -> Result> { + let options = if let Some(package) = package { + Some(RuntimeCrateSearchOption::Package(package.into())) + } else if let Some(chain) = chain { + Some(RuntimeCrateSearchOption::ChainName(chain.into())) + } else { + runtime_dir.as_ref().map(|runtime_dir| RuntimeCrateSearchOption::RuntimeDir(runtime_dir.into())) + }; + + RuntimeCrate::search(&RuntimeCrateSearchInfo { workdir: fs::canonicalize(&workdir).unwrap(), options }) + } + /// This function helps find the runtime crate based on *some* information. /// The result will be Ok if and only if the search critera lead to one single /// result. In any other cases, it will return an error. @@ -76,7 +96,7 @@ impl RuntimeCrate { }) } }, - None => todo!("This feature is not implemented yet, please pass one other search criteria"), + None => todo!("This feature is not implemented yet, please pass one search criteria among `chain`, `package` or `runtime_dir`"), } } } From 13e430192edb8567887873fe661b17ece47abf77 Mon Sep 17 00:00:00 2001 From: Chevdor Date: Wed, 21 Jul 2021 18:25:07 +0200 Subject: [PATCH 08/10] WIP: Adding fetching and returning the data from the calls --- cli/src/main.rs | 11 +- cli/src/opts.rs | 1 + ...{test_digest_v2.json => digest_v2_01.json} | 0 data/info.json | 16 ++ data/pull.json | 0 data/version.json | 5 + lib/src/lib.rs | 1 + lib/src/runner.rs | 150 +++++++++++++----- lib/src/rustc_version.rs | 8 +- lib/src/version.rs | 35 ++++ 10 files changed, 182 insertions(+), 45 deletions(-) rename data/{test_digest_v2.json => digest_v2_01.json} (100%) create mode 100644 data/info.json create mode 100644 data/pull.json create mode 100644 data/version.json create mode 100644 lib/src/version.rs diff --git a/cli/src/main.rs b/cli/src/main.rs index e731540..68a2613 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -40,8 +40,15 @@ fn main() { info!("Using {}:{}", image, tag); match opts.subcmd { - SubCommand::Pull(_) => Runner::pull(&image, &tag), - SubCommand::Version(_) => Runner::version(&image, &tag), + SubCommand::Pull(_) => { + let res = Runner::pull(&image, &tag); + println!("{}:{} => {:?}", image, tag, res); + } + + SubCommand::Version(_) => { + let v = Runner::version(&image, &tag); + println!("v = {:?}", v); + } SubCommand::Info(info_opts) => { let workdir = fs::canonicalize(&info_opts.workdir).unwrap(); diff --git a/cli/src/opts.rs b/cli/src/opts.rs index 03309c4..f6799b0 100644 --- a/cli/src/opts.rs +++ b/cli/src/opts.rs @@ -60,6 +60,7 @@ pub enum SubCommand { /// Build opts #[derive(Clap)] pub struct PullOpts; +// TODO: we may want to let the user specify a repo /// Build opts #[derive(Clap)] diff --git a/data/test_digest_v2.json b/data/digest_v2_01.json similarity index 100% rename from data/test_digest_v2.json rename to data/digest_v2_01.json diff --git a/data/info.json b/data/info.json new file mode 100644 index 0000000..9fa6367 --- /dev/null +++ b/data/info.json @@ -0,0 +1,16 @@ +{ + "generator": { + "name": "srtool", + "version": "0.9.15" + }, + "src": "git", + "version": "0.9.8", + "git": { + "commit": "3a10ee63c0b5703a1c802db3438ab7e01344a8ce", + "tag": "v0.9.8", + "branch": "heads/v0.9.8" + }, + "rustc": "rustc 1.53.0 (53cb7b09b 2021-06-17)", + "pkg": "polkadot-runtime", + "profile": "release" +} diff --git a/data/pull.json b/data/pull.json new file mode 100644 index 0000000..e69de29 diff --git a/data/version.json b/data/version.json new file mode 100644 index 0000000..04a0264 --- /dev/null +++ b/data/version.json @@ -0,0 +1,5 @@ +{ + "name": "srtool", + "version": "0.9.15", + "rustc": "1.53.0" +} diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 7c0d2c9..382a35b 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -6,6 +6,7 @@ mod runtime_crate; mod rustc_version; mod samples; mod srtool_tag; +mod version; pub use digest::*; pub use run_specs::*; diff --git a/lib/src/runner.rs b/lib/src/runner.rs index f081da3..1c7d931 100644 --- a/lib/src/runner.rs +++ b/lib/src/runner.rs @@ -1,11 +1,12 @@ //! The runner is effectively a wrapper around docker -use crate::{get_image_digest, run_specs::RunSpecs, RuntimeCrate}; +use crate::{get_image_digest, run_specs::RunSpecs, version::SrtoolVersion, RuntimeCrate}; use log::{debug, info, trace, warn}; +use serde_json::Value; use std::{ env, fs, path::{Path, PathBuf}, - process::Command, + process::{Command, Stdio}, }; pub struct BuildOpts { @@ -18,12 +19,36 @@ pub struct Runner; impl Runner { /// Pulls the image - pub fn pull(image: &str, tag: &str) { + pub fn pull(image: &str, tag: &str) -> Result<(), String> { trace!("pull()"); debug!("We will be pulling {image}:{tag}", tag = tag, image = image); let cmd = format!("docker pull {image}:{tag}", image = image, tag = tag); - Runner::run(cmd); + Runner::run(cmd).map(|_| ()) + } + + /// Get version + pub fn version(image: &str, tag: &str) -> Result { + let cmd = format!("docker run --name srtool --rm {image}:{tag} version", image = image, tag = tag); + let version = Runner::run(cmd).unwrap(); + serde_json::from_value::(version).map_err(|e| e.to_string()) + } + + /// Show infos + pub fn info(specs: &RunSpecs, workdir: &Path) { + debug!("specs: '{:#?}'", &specs); + + let cmd = format!( + "docker run --name srtool --rm \ + -v {dir}:/build \ + -e RUNTIME_DIR={runtime_dir} \ + {image}:{tag} info", + dir = workdir.display(), + runtime_dir = specs.runtime_dir.display(), + image = specs.image, + tag = specs.tag, + ); + let _info = Runner::run(cmd).unwrap(); } /// Invoke the build @@ -92,43 +117,34 @@ impl Runner { app = app, digest = digest, ); - Runner::run(cmd); - } - - /// Show infos - pub fn info(specs: &RunSpecs, workdir: &Path) { - // let rtm_crate = - // RuntimeCrate::search_flattened(&workdir, &None, &None, &Some(specs.runtime_dir.to_owned())).unwrap(); - - debug!("specs: '{:#?}'", &specs); - - let cmd = format!( - "docker run --name srtool --rm \ - -v {dir}:/build \ - -e RUNTIME_DIR={runtime_dir} \ - {image}:{tag} info", - dir = workdir.display(), - runtime_dir = specs.runtime_dir.display(), - image = specs.image, - tag = specs.tag, - ); - Runner::run(cmd); - } - /// Get version - pub fn version(image: &str, tag: &str) { - let cmd = format!("docker run --name srtool --rm {image}:{tag} version", image = image, tag = tag); - Runner::run(cmd); + let _digest = Runner::run(cmd).unwrap(); } - /// Run the docker command that is passed - fn run(cmd: String) { - if cfg!(target_os = "windows") { - Command::new("cmd").args(&["/C", cmd.as_str()]).output().expect("failed to execute process"); + /// Run the docker command that is passed and retrive its json output + fn run(cmd: String) -> Result { + let output = if cfg!(target_os = "windows") { + Command::new("cmd") + .args(&["/C", cmd.as_str()]) + .stdout(Stdio::piped()) + .spawn() + .expect("failed to execute process") + .wait_with_output() } else { - let _ = - Command::new("sh").arg("-c").arg(cmd).spawn().expect("failed to execute process").wait_with_output(); - } + Command::new("sh") + .arg("-c") + .arg(cmd) + .stdout(Stdio::piped()) + .spawn() + .expect("failed to execute process") + .wait_with_output() + }; + + output + .map(|o| o.stdout) + .map(|v| String::from_utf8(v).unwrap_or("".into())) + .map(|s| serde_json::from_str(&s).unwrap_or(Value::Null)) + .map_err(|e| e.to_string()) } } @@ -139,13 +155,13 @@ mod test_runner { #[test] fn test_pull() { let specs = RunSpecs::default(); - Runner::pull(&specs.image, &specs.tag); + let _ = Runner::pull(&specs.image, &specs.tag); } #[test] fn test_version() { let specs = RunSpecs::default(); - Runner::version(&specs.image, &specs.tag); + let _ = Runner::version(&specs.image, &specs.tag); } #[test] @@ -164,4 +180,60 @@ mod test_runner { let opts = BuildOpts { json: true, app: true, workdir }; Runner::build(&specs, &opts); } + + #[test] + fn test_fake_build_json() { + let digest = include_str!("../../data/digest_v2_01.json"); + let cmd = format!("docker run --rm -it busybox echo '{}'", digest); + + let res = Runner::run(cmd); + println!("res = {:?}", res); + } +} + +#[cfg(test)] +mod test_runner_runs { + use crate::Runner; + use std::include_str; + + #[test] + fn test_fake_version_json() { + let data = include_str!("../../data/version.json"); + let cmd = format!("docker run --rm -it busybox echo '{}'", data); + + let res = Runner::run(cmd).unwrap(); + assert!(res["name"] == "srtool"); + println!("{}", serde_json::to_string_pretty(&res).unwrap()); + } + + #[test] + fn test_fake_info_json() { + let data = include_str!("../../data/info.json"); + let cmd = format!("docker run --rm -it busybox echo '{}'", data); + + let res = Runner::run(cmd).unwrap(); + assert!(res["generator"]["name"] == "srtool"); + + println!("{}", serde_json::to_string_pretty(&res).unwrap()); + } + + #[test] + fn test_fake_pull_json() { + let data = include_str!("../../data/pull.json"); + let cmd = format!("docker run --rm -it busybox echo '{}'", data); + + let res = Runner::run(cmd).unwrap(); + assert!(res.is_null()); + } + + #[test] + fn test_fake_build_json() { + let digest = include_str!("../../data/digest_v2_01.json"); + let cmd = format!("docker run --rm -it busybox echo '{}'", digest); + + let res = Runner::run(cmd).unwrap(); + assert!(res["info"]["generator"]["name"] == "srtool"); + + println!("{}", serde_json::to_string_pretty(&res).unwrap()); + } } diff --git a/lib/src/rustc_version.rs b/lib/src/rustc_version.rs index ad71e33..d4711a7 100644 --- a/lib/src/rustc_version.rs +++ b/lib/src/rustc_version.rs @@ -1,8 +1,8 @@ -use std::str::FromStr; - use semver::Version; +use serde::{Deserialize, Serialize}; +use std::str::FromStr; -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Serialize, Deserialize)] pub enum RustcVersion { Stable(Version), // Beta(String), @@ -11,7 +11,7 @@ pub enum RustcVersion { } impl FromStr for RustcVersion { - type Err = String; + type Err = String; XXX // TODO: The checks below are very light ... See https://docs.rs/rustc_version/0.2.3/rustc_version/ fn from_str(s: &str) -> Result { diff --git a/lib/src/version.rs b/lib/src/version.rs new file mode 100644 index 0000000..31bb73a --- /dev/null +++ b/lib/src/version.rs @@ -0,0 +1,35 @@ +use crate::rustc_version::RustcVersion; +use semver::Version; +use serde::{Deserialize, Serialize}; +use std::str::FromStr; + +/// A structure describing the output the info command +#[derive(Debug, Serialize, Deserialize)] +pub struct SrtoolVersion { + name: String, + version: Version, + + #[serde(deserialize_with= "RustcVersion::from_str")] + rustc: RustcVersion, +} + +#[cfg(test)] +mod test_version { + use super::*; + use std::include_str; + + #[test] + fn test_deserialize_from_str() { + let txt = include_str!("../../data/info.json"); + let v: SrtoolVersion = serde_json::from_str(txt).unwrap(); + println!("v = {:?}", v); + } + + #[test] + fn test_deserialize_from_json() { + let txt = include_str!("../../data/info.json"); + let json = serde_json::from_str(txt).unwrap(); + let v: SrtoolVersion = serde_json::from_value(json).unwrap(); + println!("v = {:?}", v); + } +} From e492b09cde7d8243e7fe9602907dea2935fee17b Mon Sep 17 00:00:00 2001 From: Chevdor Date: Thu, 19 Aug 2021 09:39:55 +0200 Subject: [PATCH 09/10] WIP --- cli/src/opts.rs | 4 ++-- lib/src/rustc_version.rs | 8 ++++++-- lib/src/version.rs | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/cli/src/opts.rs b/cli/src/opts.rs index f6799b0..27556c4 100644 --- a/cli/src/opts.rs +++ b/cli/src/opts.rs @@ -10,8 +10,8 @@ pub struct Opts { /// Choose an alternative image. Beware to choose an image that is /// compatible with the original srtool image. Using a random image, /// you take the risk to NOT produce exactly the same deterministic - /// result as srtool. - #[clap(short, long, default_value = "paritytech/srtool")] + /// result than srtool. + #[clap(short, long, default_value = "paritytech/srtool", env = "SRTOOL_IMAGE")] pub image: String, /// This option is DEPRECATED and has no effect diff --git a/lib/src/rustc_version.rs b/lib/src/rustc_version.rs index d4711a7..8afeaf0 100644 --- a/lib/src/rustc_version.rs +++ b/lib/src/rustc_version.rs @@ -10,11 +10,14 @@ pub enum RustcVersion { // Dev(String), } +pub struct Error; + impl FromStr for RustcVersion { - type Err = String; XXX + type Err = String; // TODO: The checks below are very light ... See https://docs.rs/rustc_version/0.2.3/rustc_version/ - fn from_str(s: &str) -> Result { + fn from_str(s: &str) -> Result { + println!("s = {:?}", s); match semver::Version::from_str(s) { Ok(version) => Ok(RustcVersion::Stable(version)), _ => Ok(RustcVersion::Nightly(s.to_string())), @@ -30,6 +33,7 @@ mod test_rustc_version { fn test_from_str() { println!("v = {:?}", RustcVersion::from_str("1.53.0")); println!("v = {:?}", RustcVersion::from_str("nightly-2021-03-15")); + println!("v = {:?}", RustcVersion::from_str("rustc 1.53.0 (53cb7b09b 2021-06-17)")); // TODO: the following should error // println!("v = {:?}", RustcVersion::from_str("junk")); diff --git a/lib/src/version.rs b/lib/src/version.rs index 31bb73a..5c643b1 100644 --- a/lib/src/version.rs +++ b/lib/src/version.rs @@ -9,7 +9,7 @@ pub struct SrtoolVersion { name: String, version: Version, - #[serde(deserialize_with= "RustcVersion::from_str")] + // #[serde(deserialize_with= "RustcVersion::from_str")] rustc: RustcVersion, } From 607a793233f8899fd924cd67a9cb3e5e94a880b1 Mon Sep 17 00:00:00 2001 From: Chevdor Date: Thu, 19 Aug 2021 10:29:13 +0200 Subject: [PATCH 10/10] WIP: linting fixes --- Cargo.lock | 6 +++--- lib/src/lib.rs | 2 +- lib/src/runner.rs | 2 +- lib/src/rustc_version.rs | 2 +- lib/src/version.rs | 1 - 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7cb21a3..d35aa2f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -469,16 +469,16 @@ dependencies = [ name = "serde" version = "1.0.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03" +checksum = "f03b9878abf6d14e6779d3f24f07b2cfa90352cfec4acc5aab8f1ac7f146fae8" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.126" +version = "1.0.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43" +checksum = "a024926d3432516606328597e0f224a51355a493b49fdd67e9209187cbe55ecc" dependencies = [ "proc-macro2", "quote", diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 382a35b..66e23e9 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -112,7 +112,7 @@ mod tests { fn it_fetches_the_version() { let tag = fetch_image_tag().unwrap(); println!("current tag = {:?}", tag); - assert!(tag.len() > 0); + assert!(!tag.is_empty()); } #[test] diff --git a/lib/src/runner.rs b/lib/src/runner.rs index 1c7d931..10cd1de 100644 --- a/lib/src/runner.rs +++ b/lib/src/runner.rs @@ -142,7 +142,7 @@ impl Runner { output .map(|o| o.stdout) - .map(|v| String::from_utf8(v).unwrap_or("".into())) + .map(|v| String::from_utf8(v).unwrap_or_else(|_| "".into())) .map(|s| serde_json::from_str(&s).unwrap_or(Value::Null)) .map_err(|e| e.to_string()) } diff --git a/lib/src/rustc_version.rs b/lib/src/rustc_version.rs index 8afeaf0..f9ac1bc 100644 --- a/lib/src/rustc_version.rs +++ b/lib/src/rustc_version.rs @@ -10,7 +10,7 @@ pub enum RustcVersion { // Dev(String), } -pub struct Error; +// pub struct Error; impl FromStr for RustcVersion { type Err = String; diff --git a/lib/src/version.rs b/lib/src/version.rs index 5c643b1..341c7ec 100644 --- a/lib/src/version.rs +++ b/lib/src/version.rs @@ -1,7 +1,6 @@ use crate::rustc_version::RustcVersion; use semver::Version; use serde::{Deserialize, Serialize}; -use std::str::FromStr; /// A structure describing the output the info command #[derive(Debug, Serialize, Deserialize)]