diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b5ee1e0..2dfc4902 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ bump. Currently experimental: project bundling # Unreleased +* feat: Projects can now depend on other `icp` projects vendored into them (e.g. as git submodules) via a top-level `dependencies:` block in `icp.yaml`. + * `icp deploy` deploys the dependency alongside your project and injects its canister IDs. + * Running `icp` from inside a vendored sub-project resolves up to the workspace root, so the whole workspace shares one network and one set of canister IDs. + * `:` is now reserved in canister names as the dependency namespace separator. + * See the [Project Dependencies](docs/concepts/project-dependencies.md) concept guide for details. * feat: `icp canister delete` will now send the canister's remaining cycles to the caller # v1.0.2 diff --git a/Cargo.lock b/Cargo.lock index 48d8001d..4fb9f6a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3695,6 +3695,7 @@ dependencies = [ "num-integer", "num-traits", "p256", + "pathdiff", "pem", "pkcs8", "rand 0.10.1", @@ -5269,6 +5270,9 @@ name = "pathdiff" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" +dependencies = [ + "camino", +] [[package]] name = "pbkdf2" diff --git a/crates/icp-cli/src/commands/deploy.rs b/crates/icp-cli/src/commands/deploy.rs index 207ab151..1b9ee693 100644 --- a/crates/icp-cli/src/commands/deploy.rs +++ b/crates/icp-cli/src/commands/deploy.rs @@ -14,7 +14,7 @@ use icp_canister_interfaces::candid_ui::MAINNET_CANDID_UI_CID; use serde::Serialize; use std::collections::BTreeMap; use std::time::Duration; -use tracing::info; +use tracing::{info, warn}; use crate::{ commands::{args::ArgsOpt, canister::create}, @@ -98,12 +98,29 @@ pub(crate) async fn exec(ctx: &Context, args: &DeployArgs) -> Result<(), anyhow: let env = ctx.get_environment(&environment_selection).await?; - let cnames = match args.names.is_empty() { - // No canisters specified - true => env.canisters.keys().cloned().collect(), - - // Individual canisters specified - false => args.names.clone(), + let cnames: Vec = if args.names.is_empty() { + // No canisters specified: default to the whole environment, unless the + // command is run inside a vendored member — then scope to that member's + // canisters and announce the resolved workspace root. + let project = ctx.project.load().await?; + let member_dir = ctx.project.member_dir(); + match icp::project::member_scoped_canisters(&project.dir, member_dir.as_deref(), &env) { + Some(scoped) => { + if let Some(member) = &member_dir { + warn!( + "Running inside sub-project '{member}'; resolved workspace root '{}'. \ + Deploying only this member's canisters into environment '{}'.", + project.dir, + environment_selection.name(), + ); + } + scoped + } + None => env.canisters.keys().cloned().collect(), + } + } else { + // Individual canisters specified. + args.names.clone() }; // Skip doing any work if no canisters are targeted diff --git a/crates/icp-cli/src/main.rs b/crates/icp-cli/src/main.rs index 5458b838..2875e62a 100644 --- a/crates/icp-cli/src/main.rs +++ b/crates/icp-cli/src/main.rs @@ -65,9 +65,11 @@ mod heading { )] struct Cli { /// Directory to use as your project root directory. - /// If not specified the directory structure is traversed up until an icp.yaml file is found + /// If not specified the directory structure is traversed up to the workspace + /// root (the top-most project that declares the one you are in as a dependency). #[arg( long, + env = "ICP_PROJECT_ROOT", global = true, help_heading = heading::GLOBAL_PARAMETERS )] diff --git a/crates/icp-cli/src/operations/binding_env_vars.rs b/crates/icp-cli/src/operations/binding_env_vars.rs index 88eff87f..a2118f19 100644 --- a/crates/icp-cli/src/operations/binding_env_vars.rs +++ b/crates/icp-cli/src/operations/binding_env_vars.rs @@ -116,11 +116,6 @@ pub(crate) async fn set_binding_env_vars_many( .fail(); } - let binding_vars = canister_list - .iter() - .map(|(n, p)| (format!("PUBLIC_CANISTER_ID:{n}"), p.to_text())) - .collect::>(); - let mut futs = FuturesOrdered::new(); let progress_manager = ProgressManager::new(ProgressManagerSettings { hidden: debug }); @@ -128,10 +123,27 @@ pub(crate) async fn set_binding_env_vars_many( let pb = progress_manager.create_progress_bar(&info.name); let canister_name = info.name.clone(); + // Each canister receives only the ids it is wired to (its own project's + // canisters by their local names, plus any declared dependencies under + // their aliases), resolved to the ids that exist in this environment. + // A project without dependencies wires every canister to every sibling, + // reproducing the previous flat behavior. + let binding_vars: Vec<(String, String)> = info + .bindings + .iter() + .filter_map(|(env_name, referenced_key)| { + canister_list.get(referenced_key).map(|principal| { + ( + format!("PUBLIC_CANISTER_ID:{env_name}"), + principal.to_text(), + ) + }) + }) + .collect(); + let settings_fn = { let agent = agent.clone(); let pb = pb.clone(); - let binding_vars = binding_vars.clone(); async move { pb.set_message("Updating environment variables..."); diff --git a/crates/icp-cli/src/operations/bundle.rs b/crates/icp-cli/src/operations/bundle.rs index c725ec8d..565949f3 100644 --- a/crates/icp-cli/src/operations/bundle.rs +++ b/crates/icp-cli/src/operations/bundle.rs @@ -271,6 +271,10 @@ pub(crate) async fn create_bundle( let bundle_manifest = ProjectManifest { canisters: canister_items, + // A bundle flattens every consolidated canister (including any pulled in + // from dependencies) into `canisters` as pre-built entries, so no external + // dependency references remain. + dependencies: vec![], networks, environments, }; diff --git a/crates/icp-cli/tests/dependency_tests.rs b/crates/icp-cli/tests/dependency_tests.rs new file mode 100644 index 00000000..84614db9 --- /dev/null +++ b/crates/icp-cli/tests/dependency_tests.rs @@ -0,0 +1,382 @@ +use indoc::formatdoc; +use predicates::{prelude::PredicateBooleanExt, str::contains}; + +use crate::common::{ENVIRONMENT_RANDOM_PORT, NETWORK_RANDOM_PORT, TestContext, clients}; +use icp::{fs::write_string, prelude::*}; + +mod common; + +/// Deploying a project that declares a dependency should deploy the whole +/// dependency and wire canister IDs per project scope: +/// - the app's canister sees the exposed dependency canister under its alias, +/// - the dependency's canisters keep their standalone view (own names only). +#[tokio::test] +async fn deploy_with_dependency_injects_namespaced_ids() { + let ctx = TestContext::new(); + let project_dir = ctx.create_project_dir("icp"); + + let wasm = ctx.make_asset("example_icp_mo.wasm"); + + // A self-contained vendored dependency project with two canisters. + let dep_dir = project_dir.join("vendor/openemail"); + std::fs::create_dir_all(&dep_dir).expect("failed to create dependency dir"); + let dep_manifest = formatdoc! {r#" + canisters: + - name: backend + build: + steps: + - type: script + command: cp '{wasm}' "$ICP_WASM_OUTPUT_PATH" + - name: frontend + build: + steps: + - type: script + command: cp '{wasm}' "$ICP_WASM_OUTPUT_PATH" + + # A member must declare every environment the workspace targets; + # the network binding is ignored (the root supplies it). + environments: + - name: random-environment + "#}; + write_string(&dep_dir.join("icp.yaml"), &dep_manifest) + .expect("failed to write dependency manifest"); + + // The app: one canister plus a dependency exposing only `openemail:backend`. + let pm = formatdoc! {r#" + canisters: + - name: backend + build: + steps: + - type: script + command: cp '{wasm}' "$ICP_WASM_OUTPUT_PATH" + + dependencies: + - name: openemail + path: ./vendor/openemail + canisters: [backend] + + {NETWORK_RANDOM_PORT} + {ENVIRONMENT_RANDOM_PORT} + "#}; + write_string(&project_dir.join("icp.yaml"), &pm).expect("failed to write project manifest"); + + let _g = ctx.start_network_in(&project_dir, "random-network").await; + ctx.ping_until_healthy(&project_dir, "random-network"); + + clients::icp(&ctx, &project_dir, Some("random-environment".to_string())) + .mint_cycles(200 * TRILLION); + + // Deploy the app and the whole dependency. The dependency canisters are keyed + // by their path relative to the project root. + ctx.icp() + .current_dir(&project_dir) + .args(["deploy", "--environment", "random-environment"]) + .assert() + .success() + .stdout(contains("vendor/openemail:backend").and(contains("vendor/openemail:frontend"))); + + // The app's `backend` sees the exposed dependency canister under its alias. + ctx.icp() + .current_dir(&project_dir) + .args([ + "canister", + "settings", + "show", + "backend", + "--environment", + "random-environment", + ]) + .assert() + .success() + .stdout(contains("PUBLIC_CANISTER_ID:openemail:backend")); + + // The dependency's own `backend` keeps its standalone view: it sees `backend` + // (itself) and `frontend`, but not the parent's `openemail:` alias. + ctx.icp() + .current_dir(&project_dir) + .args([ + "canister", + "settings", + "show", + "vendor/openemail:backend", + "--environment", + "random-environment", + ]) + .assert() + .success() + .stdout( + contains("PUBLIC_CANISTER_ID:backend") + .and(contains("PUBLIC_CANISTER_ID:frontend")) + .and(contains("PUBLIC_CANISTER_ID:openemail:backend").not()), + ); +} + +/// Running `icp deploy` from *inside* a vendored member resolves up to the +/// workspace root and deploys only that member's canisters, into the root's +/// environment and store (single source-of-truth ids). The app's own canister +/// is not touched. +#[tokio::test] +async fn deploy_from_member_scopes_to_member_and_uses_root_store() { + let ctx = TestContext::new(); + let project_dir = ctx.create_project_dir("icp"); + let wasm = ctx.make_asset("example_icp_mo.wasm"); + + // A self-contained vendored dependency with two canisters. + let dep_dir = project_dir.join("vendor/openemail"); + std::fs::create_dir_all(&dep_dir).expect("failed to create dependency dir"); + let dep_manifest = formatdoc! {r#" + canisters: + - name: backend + build: + steps: + - type: script + command: cp '{wasm}' "$ICP_WASM_OUTPUT_PATH" + - name: frontend + build: + steps: + - type: script + command: cp '{wasm}' "$ICP_WASM_OUTPUT_PATH" + + # A member must declare every environment the workspace targets; + # the network binding is ignored (the root supplies it). + environments: + - name: random-environment + "#}; + write_string(&dep_dir.join("icp.yaml"), &dep_manifest) + .expect("failed to write dependency manifest"); + + // The app: its own canister plus the dependency. The network/environment + // live only in the app (the workspace root). + let pm = formatdoc! {r#" + canisters: + - name: backend + build: + steps: + - type: script + command: cp '{wasm}' "$ICP_WASM_OUTPUT_PATH" + + dependencies: + - name: openemail + path: ./vendor/openemail + + {NETWORK_RANDOM_PORT} + {ENVIRONMENT_RANDOM_PORT} + "#}; + write_string(&project_dir.join("icp.yaml"), &pm).expect("failed to write project manifest"); + + let _g = ctx.start_network_in(&project_dir, "random-network").await; + ctx.ping_until_healthy(&project_dir, "random-network"); + + clients::icp(&ctx, &project_dir, Some("random-environment".to_string())) + .mint_cycles(200 * TRILLION); + + // Deploy from INSIDE the member. Resolution climbs to the app root; only the + // member's canisters are deployed, and the run announces the resolved root. + ctx.icp() + .current_dir(&dep_dir) + .args(["deploy", "--environment", "random-environment"]) + .assert() + .success() + .stdout(contains("vendor/openemail:backend").and(contains("vendor/openemail:frontend"))) + .stderr(contains("resolved workspace root")); + + // The member's ids were written to the *root* store: they are queryable from + // the app root. + ctx.icp() + .current_dir(&project_dir) + .args([ + "canister", + "status", + "--environment", + "random-environment", + "vendor/openemail:backend", + "--id-only", + ]) + .assert() + .success(); + + // The app's own canister was NOT deployed by the member-scoped run (no id in + // the store yet). + ctx.icp() + .current_dir(&project_dir) + .args([ + "canister", + "status", + "--environment", + "random-environment", + "backend", + "--id-only", + ]) + .assert() + .failure(); +} + +/// Deploying to an environment a vendored member does not declare fails fast +/// with a clear error (strict rule) — before any network is contacted. +#[tokio::test] +async fn deploy_to_env_missing_from_member_fails() { + let ctx = TestContext::new(); + let project_dir = ctx.create_project_dir("icp"); + let wasm = ctx.make_asset("example_icp_mo.wasm"); + + // The dependency does NOT declare `random-environment`. + let dep_dir = project_dir.join("vendor/openemail"); + std::fs::create_dir_all(&dep_dir).expect("failed to create dependency dir"); + let dep_manifest = formatdoc! {r#" + canisters: + - name: backend + build: + steps: + - type: script + command: cp '{wasm}' "$ICP_WASM_OUTPUT_PATH" + "#}; + write_string(&dep_dir.join("icp.yaml"), &dep_manifest) + .expect("failed to write dependency manifest"); + + let pm = formatdoc! {r#" + canisters: + - name: backend + build: + steps: + - type: script + command: cp '{wasm}' "$ICP_WASM_OUTPUT_PATH" + + dependencies: + - name: openemail + path: ./vendor/openemail + + {NETWORK_RANDOM_PORT} + {ENVIRONMENT_RANDOM_PORT} + "#}; + write_string(&project_dir.join("icp.yaml"), &pm).expect("failed to write project manifest"); + + // No network started: the strict check rejects the deploy before any network + // interaction. + ctx.icp() + .current_dir(&project_dir) + .args(["deploy", "--environment", "random-environment"]) + .assert() + .failure() + .stderr(contains("random-environment").and(contains("vendor/openemail"))); +} + +/// The "umbrella" layout: two independent sub-projects (`service-a`, `service-b`) +/// each depend on the same sibling `openemail` via `../openemail`, and the app +/// depends on both services. Because both edges resolve to the same directory, +/// openemail must be deployed exactly once and shared by both services. +#[tokio::test] +async fn deploy_with_shared_dependency_dedups_to_one_instance() { + let ctx = TestContext::new(); + let app = ctx.create_project_dir("icp"); + let wasm = ctx.make_asset("example_icp_mo.wasm"); + + let canister = |name: &str| { + formatdoc! {r#" + canisters: + - name: {name} + build: + steps: + - type: script + command: cp '{wasm}' "$ICP_WASM_OUTPUT_PATH" + "#} + }; + + // Every member must declare the environment the workspace targets. + let random_env = "\nenvironments:\n - name: random-environment\n"; + + // umbrella/openemail — the shared service. + let openemail = app.join("umbrella/openemail"); + std::fs::create_dir_all(&openemail).expect("failed to create openemail dir"); + write_string( + &openemail.join("icp.yaml"), + &format!("{}{random_env}", canister("backend")), + ) + .expect("failed to write openemail manifest"); + + // umbrella/service-a and umbrella/service-b each depend on ../openemail. + for svc in ["service-a", "service-b"] { + let dir = app.join(format!("umbrella/{svc}")); + std::fs::create_dir_all(&dir).expect("failed to create service dir"); + let manifest = formatdoc! {r#" + {service} + dependencies: + - name: openemail + path: ../openemail + canisters: [backend] + {random_env} + "#, service = canister("service")}; + write_string(&dir.join("icp.yaml"), &manifest).expect("failed to write service manifest"); + } + + // The app depends on both services. + let pm = formatdoc! {r#" + {frontend} + dependencies: + - name: service-a + path: ./umbrella/service-a + canisters: [service] + - name: service-b + path: ./umbrella/service-b + canisters: [service] + + {NETWORK_RANDOM_PORT} + {ENVIRONMENT_RANDOM_PORT} + "#, frontend = canister("frontend")}; + write_string(&app.join("icp.yaml"), &pm).expect("failed to write app manifest"); + + let _g = ctx.start_network_in(&app, "random-network").await; + ctx.ping_until_healthy(&app, "random-network"); + + clients::icp(&ctx, &app, Some("random-environment".to_string())).mint_cycles(500 * TRILLION); + + // Deploy succeeds: the two edges to `umbrella/openemail` collapse to one + // instance. (Without de-dup, importing the same store key twice would error.) + ctx.icp() + .current_dir(&app) + .args(["deploy", "--environment", "random-environment"]) + .assert() + .success(); + + // Capture the single shared openemail canister id. + let assert = ctx + .icp() + .current_dir(&app) + .args([ + "canister", + "status", + "--environment", + "random-environment", + "umbrella/openemail:backend", + "--id-only", + ]) + .assert() + .success(); + let openemail_id = String::from_utf8(assert.get_output().stdout.clone()) + .expect("canister id should be valid utf-8") + .trim() + .to_string(); + assert!( + !openemail_id.is_empty(), + "expected a shared openemail canister id" + ); + + // Both services' `openemail:backend` binding resolves to the SAME instance. + for svc in ["umbrella/service-a:service", "umbrella/service-b:service"] { + ctx.icp() + .current_dir(&app) + .args([ + "canister", + "settings", + "show", + svc, + "--environment", + "random-environment", + ]) + .assert() + .success() + .stdout( + contains("PUBLIC_CANISTER_ID:openemail:backend") + .and(contains(openemail_id.clone())), + ); + } +} diff --git a/crates/icp/Cargo.toml b/crates/icp/Cargo.toml index c0355bcb..004c53e6 100644 --- a/crates/icp/Cargo.toml +++ b/crates/icp/Cargo.toml @@ -48,6 +48,7 @@ num-bigint = { workspace = true } num-integer = { workspace = true } num-traits = { workspace = true } p256 = { workspace = true } +pathdiff = { workspace = true } pem = { workspace = true } pkcs8 = { workspace = true } rand = { workspace = true } diff --git a/crates/icp/src/context/mod.rs b/crates/icp/src/context/mod.rs index 32777188..6c8431b9 100644 --- a/crates/icp/src/context/mod.rs +++ b/crates/icp/src/context/mod.rs @@ -144,6 +144,19 @@ impl Context { name: environment.name().to_owned(), })?; + // Strict rule: every vendored member must declare the selected + // environment. Enforced here (not at load time) so a member missing some + // other environment never blocks deploys to the ones it does declare. + if let Some(missing) = p.member_missing_envs.get(environment.name()) + && let Some(member) = missing.first() + { + return MissingDependencyEnvironmentSnafu { + environment: environment.name().to_owned(), + member: member.clone(), + } + .fail(); + } + let network_type = match &env.network.configuration { NetworkConfiguration::Managed { .. } => NetworkType::Managed, NetworkConfiguration::Connected { .. } => NetworkType::Connected, @@ -605,6 +618,12 @@ pub enum GetEnvironmentError { #[snafu(display("project does not contain an environment named '{}'", name))] EnvironmentNotFound { name: String }, + + #[snafu(display( + "environment '{environment}' is not defined by dependency '{member}'; \ + a dependency must declare every environment the workspace targets" + ))] + MissingDependencyEnvironment { environment: String, member: String }, } #[derive(Debug, Snafu)] diff --git a/crates/icp/src/context/tests.rs b/crates/icp/src/context/tests.rs index 86b55f71..5ceca2d6 100644 --- a/crates/icp/src/context/tests.rs +++ b/crates/icp/src/context/tests.rs @@ -639,6 +639,7 @@ async fn test_get_agent_defaults_inside_project_with_default_local() { canisters: IndexMap::new(), // No canisters needed for get_agent test networks, environments, + member_missing_envs: std::collections::HashMap::new(), }; let ctx = Context { @@ -710,6 +711,7 @@ async fn test_get_agent_defaults_with_overridden_local_network() { canisters: IndexMap::new(), // No canisters needed for get_agent test networks, environments, + member_missing_envs: std::collections::HashMap::new(), }; let custom_root_key = vec![1, 2, 3, 4]; @@ -807,6 +809,7 @@ async fn test_get_agent_defaults_with_overridden_local_environment() { canisters: IndexMap::new(), // No canisters needed for get_agent test networks, environments, + member_missing_envs: std::collections::HashMap::new(), }; let local_root_key = vec![1, 2, 3, 4]; diff --git a/crates/icp/src/lib.rs b/crates/icp/src/lib.rs index d5ad70cf..1538db2c 100644 --- a/crates/icp/src/lib.rs +++ b/crates/icp/src/lib.rs @@ -1,4 +1,7 @@ -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::{BTreeMap, HashMap}, + sync::Arc, +}; use async_trait::async_trait; use indexmap::IndexMap; @@ -105,6 +108,17 @@ pub struct Canister { /// original recipe specifier string (e.g. `@dfinity/motoko@v4.0.0`). /// `None` when the canister uses explicit build/sync instructions. pub registry_recipe: Option, + + /// Canister-discovery wiring. Maps the name this canister reads in a + /// `PUBLIC_CANISTER_ID:` environment variable to the store key of the + /// referenced canister. Computed during consolidation so each canister sees + /// the view its owning project expects: its own project's canisters under + /// their local names, plus any declared dependencies under their aliases + /// (`:`). For a project with no dependencies this maps every + /// canister's local name to itself, reproducing the flat "every canister sees + /// every sibling" behavior. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub bindings: BTreeMap, } #[derive(Clone, Debug, PartialEq, Serialize)] @@ -149,6 +163,13 @@ pub struct Project { pub canisters: IndexMap, pub networks: HashMap, pub environments: HashMap, + + /// Environments the workspace defines that some vendored member does *not* + /// declare, keyed by environment name → the missing members' store-key + /// prefixes. Enforced when the environment is selected (strict rule). + /// Empty for standalone projects and workspaces whose members are complete. + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub member_missing_envs: HashMap>, } impl Project { @@ -175,6 +196,14 @@ pub enum ProjectLoadError { pub trait ProjectLoad: Sync + Send { async fn load(&self) -> Result; async fn exists(&self) -> Result; + + /// The directory of the member (sub-project) the command is standing in, + /// i.e. the nearest `icp.yaml` at or above cwd. Equals the workspace root + /// (`Project::dir`) at the root or in a standalone project. `None` when the + /// member directory cannot be determined; callers then skip member-scoping. + fn member_dir(&self) -> Option { + None + } } pub struct ProjectLoadImpl { @@ -214,6 +243,10 @@ impl ProjectLoad for ProjectLoadImpl { Err(ProjectRootLocateError::NotFound { .. }) => Ok(false), } } + + fn member_dir(&self) -> Option { + self.project_root_locate.locate_member().ok() + } } pub struct Lazy(T, Arc>>); @@ -249,6 +282,10 @@ impl ProjectLoad for Lazy { let v = self.0.exists().await?; Ok(v) } + + fn member_dir(&self) -> Option { + self.0.member_dir() + } } #[cfg(test)] @@ -292,6 +329,7 @@ impl MockProjectLoader { sync: SyncSteps::default(), init_args: None, registry_recipe: None, + bindings: BTreeMap::new(), }; let local_network = Network { @@ -327,6 +365,7 @@ impl MockProjectLoader { canisters, networks, environments, + member_missing_envs: HashMap::new(), }; Self::new(project) @@ -375,6 +414,7 @@ impl MockProjectLoader { sync: SyncSteps::default(), init_args: None, registry_recipe: None, + bindings: BTreeMap::new(), }; let frontend_canister = Canister { @@ -391,6 +431,7 @@ impl MockProjectLoader { sync: SyncSteps::default(), init_args: None, registry_recipe: None, + bindings: BTreeMap::new(), }; let database_canister = Canister { @@ -407,6 +448,7 @@ impl MockProjectLoader { sync: SyncSteps::default(), init_args: None, registry_recipe: None, + bindings: BTreeMap::new(), }; // Create networks @@ -552,6 +594,7 @@ impl MockProjectLoader { canisters, networks, environments, + member_missing_envs: HashMap::new(), }; Self::new(project) @@ -617,6 +660,10 @@ mod tests { fn locate(&self) -> Result { Ok(self.path.clone()) } + + fn locate_member(&self) -> Result { + Ok(self.path.clone()) + } } struct MockRecipeResolver; diff --git a/crates/icp/src/manifest/dependency.rs b/crates/icp/src/manifest/dependency.rs new file mode 100644 index 00000000..f0928925 --- /dev/null +++ b/crates/icp/src/manifest/dependency.rs @@ -0,0 +1,170 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Deserializer, Serialize}; + +use super::environment::CanisterSelection; + +/// Raw form of a dependency entry as written in `icp.yaml`. +/// +/// `canisters` follows the same convention as an environment's canister +/// selection: omitted means "all", an empty list means "none". +#[derive(Clone, Debug, PartialEq, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct DependencyInner { + pub name: String, + pub path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub canisters: Option>, +} + +/// Declares a dependency on another `icp` project vendored into this one +/// (typically as a git submodule). +/// +/// Running `icp deploy` deploys the dependency's canisters into the parent's +/// environment and injects the *selected* dependency canisters' IDs into the +/// parent's canisters as `PUBLIC_CANISTER_ID::` environment +/// variables. +#[derive(Clone, Debug, PartialEq, JsonSchema)] +pub struct DependencyManifest { + /// Local alias for the dependency project. Namespaces the dependency's + /// canister IDs when they are exposed to the parent's canisters as + /// `PUBLIC_CANISTER_ID::` environment variables. Must be + /// unique among dependencies, must not collide with a local canister name, + /// and must not contain `:`. + pub name: String, + + /// Path to the directory containing the dependency's `icp.yaml`, resolved + /// relative to this manifest's project directory. + pub path: String, + + /// Which of the dependency's canisters to **expose** to the parent as + /// `PUBLIC_CANISTER_ID` environment variables. Defaults to all. This is an + /// exposure filter only: the whole dependency is always deployed, because a + /// dependency's canisters may call each other. + #[schemars(with = "Option>")] + pub canisters: CanisterSelection, +} + +impl From for DependencyManifest { + fn from(v: DependencyInner) -> Self { + let DependencyInner { + name, + path, + canisters, + } = v; + + let canisters = match canisters { + Some(cs) => match cs.is_empty() { + true => CanisterSelection::None, + false => CanisterSelection::Named(cs), + }, + None => CanisterSelection::Everything, + }; + + Self { + name, + path, + canisters, + } + } +} + +impl<'de> Deserialize<'de> for DependencyManifest { + fn deserialize>(d: D) -> Result { + let inner: DependencyInner = Deserialize::deserialize(d)?; + Ok(inner.into()) + } +} + +impl From<&DependencyManifest> for DependencyInner { + fn from(dep: &DependencyManifest) -> Self { + let canisters = match &dep.canisters { + CanisterSelection::Everything => None, + CanisterSelection::Named(names) => Some(names.clone()), + CanisterSelection::None => Some(vec![]), + }; + + DependencyInner { + name: dep.name.clone(), + path: dep.path.clone(), + canisters, + } + } +} + +impl Serialize for DependencyManifest { + fn serialize(&self, serializer: S) -> Result { + DependencyInner::from(self).serialize(serializer) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn expose_all_when_canisters_omitted() { + assert_eq!( + serde_yaml::from_str::( + r#" + name: openemail + path: ./vendor/openemail + "# + ) + .expect("failed to deserialize DependencyManifest"), + DependencyManifest { + name: "openemail".to_string(), + path: "./vendor/openemail".to_string(), + canisters: CanisterSelection::Everything, + }, + ); + } + + #[test] + fn named_subset() { + assert_eq!( + serde_yaml::from_str::( + r#" + name: openemail + path: ./vendor/openemail + canisters: [backend] + "# + ) + .expect("failed to deserialize DependencyManifest"), + DependencyManifest { + name: "openemail".to_string(), + path: "./vendor/openemail".to_string(), + canisters: CanisterSelection::Named(vec!["backend".to_string()]), + }, + ); + } + + #[test] + fn empty_list_exposes_none() { + assert_eq!( + serde_yaml::from_str::( + r#" + name: openemail + path: ./vendor/openemail + canisters: [] + "# + ) + .expect("failed to deserialize DependencyManifest") + .canisters, + CanisterSelection::None, + ); + } + + #[test] + fn unknown_field_rejected() { + assert!( + serde_yaml::from_str::( + r#" + name: openemail + path: ./vendor/openemail + bogus: true + "# + ) + .is_err() + ); + } +} diff --git a/crates/icp/src/manifest/mod.rs b/crates/icp/src/manifest/mod.rs index fc2fa202..0a811e35 100644 --- a/crates/icp/src/manifest/mod.rs +++ b/crates/icp/src/manifest/mod.rs @@ -1,3 +1,4 @@ +use std::collections::HashSet; use std::marker::PhantomData; use schemars::JsonSchema; @@ -9,6 +10,7 @@ use crate::prelude::*; pub(crate) mod adapter; pub(crate) mod canister; +pub(crate) mod dependency; pub(crate) mod environment; pub(crate) mod network; pub(crate) mod project; @@ -22,6 +24,7 @@ pub use { ArgsFormat, BuildStep, BuildSteps, CanisterManifest, Instructions, ManifestInitArgs, SyncStep, SyncSteps, }, + dependency::DependencyManifest, environment::EnvironmentManifest, network::{ManagedMode, Mode, NetworkManifest}, project::ProjectManifest, @@ -113,8 +116,14 @@ pub enum ProjectRootLocateError { /// Trait for locating the project root directory containing the project manifest file (`icp.yaml`). pub trait ProjectRootLocate: Sync + Send { - /// Locate the project root directory. + /// Locate the workspace root directory: the top-most project that + /// transitively declares the project the command is standing in. fn locate(&self) -> Result; + + /// Locate the *member* directory the command is standing in: the nearest + /// project manifest at or above cwd, without climbing to the workspace root. + /// Equals [`locate`](Self::locate) at the root or in a standalone project. + fn locate_member(&self) -> Result; } /// Implementation of [`ProjectRootLocate`]. @@ -136,9 +145,118 @@ impl ProjectRootLocateImpl { } } +/// The nearest directory at or above `start` that contains a project manifest. +fn nearest_manifest_dir(start: &Path) -> Option { + let mut dir = start.to_owned(); + loop { + if dir.join(PROJECT_MANIFEST).exists() { + return Some(dir); + } + dir = dir.parent()?.to_owned(); + } +} + +/// The nearest directory *strictly above* `dir` that contains a project manifest. +fn next_manifest_dir_above(dir: &Path) -> Option { + let mut cur = dir.parent()?.to_owned(); + loop { + if cur.join(PROJECT_MANIFEST).exists() { + return Some(cur); + } + cur = cur.parent()?.to_owned(); + } +} + +/// Canonicalize a directory (resolving `..` and symlinks) into a UTF-8 path. +/// Returns `None` if the path does not exist or is not valid UTF-8; callers +/// treat that as "cannot establish identity", which is safe for resolution. +fn canonicalize_dir(dir: &Path) -> Option { + let canon = dunce::canonicalize(dir.as_std_path()).ok()?; + PathBuf::try_from(canon).ok() +} + +/// Read only the dependency `path:` entries from a manifest, ignoring every +/// other field. Deliberately lenient: any read/parse failure yields no +/// dependencies, so an unrelated or malformed ancestor manifest is treated as +/// declaring nothing (it will not be adopted as a workspace root). +fn read_dependency_paths(manifest_path: &Path) -> Vec { + #[derive(Deserialize)] + struct DepProbe { + path: String, + } + #[derive(Deserialize)] + struct ManifestProbe { + #[serde(default)] + dependencies: Vec, + } + + let Ok(content) = fs::read(manifest_path) else { + return Vec::new(); + }; + match serde_yaml::from_slice::(&content) { + Ok(p) => p.dependencies.into_iter().map(|d| d.path).collect(), + Err(_) => Vec::new(), + } +} + +/// The set of canonical directories a manifest declares as dependencies, +/// transitively. Each `path:` is resolved relative to the manifest that +/// declares it, then canonicalized so identity is independent of how the path +/// is spelled (matches [`crate::project`] dependency de-duplication). +fn transitive_dep_dirs(manifest_dir: &Path) -> HashSet { + let mut out = HashSet::new(); + let Some(start) = canonicalize_dir(manifest_dir) else { + return out; + }; + let mut visited: HashSet = HashSet::from([start.clone()]); + let mut stack = vec![start]; + while let Some(dir) = stack.pop() { + for rel in read_dependency_paths(&dir.join(PROJECT_MANIFEST)) { + let Some(dep) = canonicalize_dir(&dir.join(&rel)) else { + continue; + }; + out.insert(dep.clone()); + if visited.insert(dep.clone()) { + stack.push(dep); + } + } + } + out +} + impl ProjectRootLocate for ProjectRootLocateImpl { fn locate(&self) -> Result { - // Specified path + // Start from the project the command is standing in. An explicit + // override forces member == root (no climb) — see `locate_member`. + let start = self.locate_member()?; + if self.dir.is_some() { + return Ok(start); + } + + // Climb to the workspace root: adopt an ancestor only if its transitive + // dependency closure declares `start`. Early-stop at the first ancestor + // that does not — this never crosses a "gap" and never adopts an + // unrelated ancestor. With no declaring ancestor this + // degenerates to returning `start`, i.e. today's behavior. + let start_canonical = canonicalize_dir(&start).unwrap_or_else(|| start.clone()); + let mut root = start.clone(); + let mut cursor = start; + while let Some(ancestor) = next_manifest_dir_above(&cursor) { + if transitive_dep_dirs(&ancestor).contains(&start_canonical) { + root = ancestor.clone(); + cursor = ancestor; + } else { + break; + } + } + Ok(root) + } + + fn locate_member(&self) -> Result { + // An explicit override (`--project-root-override` / `ICP_PROJECT_ROOT`) + // forces the project directory and skips the upward climb — the escape + // hatch for "operate on exactly this project" (e.g. deploy a vendored + // member as a standalone project). Member and root are then identical. if let Some(dir) = &self.dir { if !dir.join(PROJECT_MANIFEST).exists() { return NotFoundSnafu { @@ -150,24 +268,13 @@ impl ProjectRootLocate for ProjectRootLocateImpl { return Ok(dir.to_owned()); } - // Unspecified path - let mut dir = self.cwd.to_owned(); - - loop { - if !dir.join(PROJECT_MANIFEST).exists() { - if let Some(p) = dir.parent() { - dir = p.to_path_buf(); - continue; - } - - return NotFoundSnafu { - path: self.cwd.to_owned(), - } - .fail(); + // The project the command is standing in: nearest manifest at/above cwd. + nearest_manifest_dir(&self.cwd).ok_or_else(|| { + NotFoundSnafu { + path: self.cwd.to_owned(), } - - return Ok(dir); - } + .build() + }) } } @@ -204,6 +311,152 @@ mod tests { std::fs::write(dir.join(PROJECT_MANIFEST), "").unwrap(); } + /// Create `dir` (and parents) and write an `icp.yaml` declaring the given + /// `(alias, path)` dependencies. + fn write_project(dir: &Path, deps: &[(&str, &str)]) { + std::fs::create_dir_all(dir).unwrap(); + let mut body = String::new(); + if !deps.is_empty() { + body.push_str("dependencies:\n"); + for (name, path) in deps { + body.push_str(&format!(" - name: {name}\n path: {path}\n")); + } + } + std::fs::write(dir.join(PROJECT_MANIFEST), body).unwrap(); + } + + // A lone project (no declaring ancestor) is its own root. + #[test] + fn locate_standalone_member_is_its_own_root() { + let tmp = Utf8TempDir::new().unwrap(); + let member = tmp.path().join("openemail"); + write_project(&member, &[]); + + let locator = ProjectRootLocateImpl::new(member.clone(), None); + assert_eq!(locator.locate().unwrap(), member); + } + + // Running inside a member climbs to the parent that declares it. + #[test] + fn locate_climbs_to_declaring_parent() { + let tmp = Utf8TempDir::new().unwrap(); + let openhr = tmp.path().join("openhr"); + write_project(&openhr, &[("openemail", "./openemail")]); + let openemail = openhr.join("openemail"); + write_project(&openemail, &[]); + + let locator = ProjectRootLocateImpl::new(openemail, None); + assert_eq!(locator.locate().unwrap(), openhr); + } + + // A transitive chain climbs all the way to the top-most declaring project, + // from any member in the chain. + #[test] + fn locate_climbs_transitive_chain_to_top() { + let tmp = Utf8TempDir::new().unwrap(); + let app = tmp.path().join("app"); + write_project(&app, &[("openhr", "./openhr")]); + let openhr = app.join("openhr"); + write_project(&openhr, &[("openemail", "./openemail")]); + let openemail = openhr.join("openemail"); + write_project(&openemail, &[]); + + assert_eq!( + ProjectRootLocateImpl::new(openemail, None) + .locate() + .unwrap(), + app + ); + assert_eq!( + ProjectRootLocateImpl::new(openhr, None).locate().unwrap(), + app + ); + } + + // Diamond: the shared member is declared via siblings, not directly by the + // top project, and sits at a hoisted location. Transitive containment still + // resolves the top project as root. + #[test] + fn locate_resolves_diamond_via_transitive_closure() { + let tmp = Utf8TempDir::new().unwrap(); + let app = tmp.path().join("app"); + write_project( + &app, + &[ + ("service_a", "./umbrella/service-a"), + ("service_b", "./umbrella/service-b"), + ], + ); + write_project( + &app.join("umbrella/service-a"), + &[("openemail", "../openemail")], + ); + write_project( + &app.join("umbrella/service-b"), + &[("openemail", "../openemail")], + ); + let openemail = app.join("umbrella/openemail"); + write_project(&openemail, &[]); + + // `umbrella/` has no manifest, so the nearest ancestor above openemail is + // `app`, which declares openemail only transitively (app -> service-a -> + // ../openemail). + assert_eq!( + ProjectRootLocateImpl::new(openemail, None) + .locate() + .unwrap(), + app + ); + } + + // An ancestor that does not declare the project is not adopted as root. + #[test] + fn locate_rejects_unrelated_ancestor() { + let tmp = Utf8TempDir::new().unwrap(); + let outer = tmp.path().join("outer"); + write_project(&outer, &[]); // declares nothing + let app = outer.join("app"); + write_project(&app, &[]); + + let locator = ProjectRootLocateImpl::new(app.clone(), None); + assert_eq!(locator.locate().unwrap(), app); + } + + // Gap: a declaring project sits above a non-declaring manifest. Early-stop + // stops at the contiguous declaring chain and does not cross the gap. + #[test] + fn locate_early_stops_at_gap() { + let tmp = Utf8TempDir::new().unwrap(); + let outer = tmp.path().join("outer"); + // outer declares openhr through the gap directory. + write_project(&outer, &[("openhr", "./legacy/openhr")]); + let legacy = outer.join("legacy"); + write_project(&legacy, &[]); // the gap: declares nothing + let openhr = legacy.join("openhr"); + write_project(&openhr, &[("openemail", "./openemail")]); + let openemail = openhr.join("openemail"); + write_project(&openemail, &[]); + + // Climb stops at openhr because `legacy` (the next ancestor) does not + // declare openemail, even though `outer` above it does. + let locator = ProjectRootLocateImpl::new(openemail, None); + assert_eq!(locator.locate().unwrap(), openhr); + } + + // An explicit override forces that directory as root, with no upward climb. + #[test] + fn locate_override_forces_root_without_climbing() { + let tmp = Utf8TempDir::new().unwrap(); + let openhr = tmp.path().join("openhr"); + write_project(&openhr, &[("openemail", "./openemail")]); + let openemail = openhr.join("openemail"); + write_project(&openemail, &[]); + + // cwd is openemail but override pins openemail itself as the root. + let locator = ProjectRootLocateImpl::new(openemail.clone(), Some(openemail.clone())); + assert_eq!(locator.locate().unwrap(), openemail); + } + #[test] fn locate_returns_cwd_when_manifest_present() { let tmp = Utf8TempDir::new().unwrap(); diff --git a/crates/icp/src/manifest/project.rs b/crates/icp/src/manifest/project.rs index 310b57f9..12c5cded 100644 --- a/crates/icp/src/manifest/project.rs +++ b/crates/icp/src/manifest/project.rs @@ -2,7 +2,8 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use crate::manifest::{ - Item, canister::CanisterManifest, environment::EnvironmentManifest, network::NetworkManifest, + Item, canister::CanisterManifest, dependency::DependencyManifest, + environment::EnvironmentManifest, network::NetworkManifest, }; #[derive(Debug, PartialEq, JsonSchema, Deserialize, Serialize)] @@ -11,6 +12,13 @@ pub struct ProjectManifest { #[serde(default)] pub canisters: Vec>, + /// Other `icp` projects this project depends on. Their canisters are + /// deployed into this project's environment and their IDs are injected into + /// this project's canisters as `PUBLIC_CANISTER_ID::` + /// environment variables. + #[serde(default)] + pub dependencies: Vec, + #[serde(default)] pub networks: Vec>, @@ -111,6 +119,7 @@ mod tests { assert_eq!( serde_yaml::from_str::(r#""#).unwrap(), ProjectManifest { + dependencies: vec![], canisters: vec![], networks: vec![], environments: vec![], @@ -130,6 +139,7 @@ mod tests { command: dosomething.sh "#}), ProjectManifest { + dependencies: vec![], canisters: vec![Item::Manifest(CanisterManifest { name: "my-canister".to_string(), settings: Settings::default(), @@ -211,6 +221,7 @@ mod tests { command: dosomething.sh "#}), ProjectManifest { + dependencies: vec![], canisters: vec![Item::Manifest(CanisterManifest { name: "my-canister".to_string(), settings: Settings::default(), @@ -245,6 +256,7 @@ mod tests { - canisters/* "#}), ProjectManifest { + dependencies: vec![], canisters: vec![ Item::Manifest(CanisterManifest { name: "my-canister".to_string(), @@ -278,6 +290,7 @@ mod tests { mode: managed "#}), ProjectManifest { + dependencies: vec![], canisters: vec![], networks: vec![Item::Manifest(NetworkManifest { name: "my-network".to_string(), @@ -309,6 +322,7 @@ mod tests { canisters: [my-canister] "#}), ProjectManifest { + dependencies: vec![], canisters: vec![], networks: vec![], environments: vec![Item::Manifest(EnvironmentManifest { @@ -332,6 +346,7 @@ mod tests { canisters: [my-canister] "#}), ProjectManifest { + dependencies: vec![], canisters: vec![], networks: vec![], environments: vec![Item::Manifest(EnvironmentManifest { @@ -357,6 +372,7 @@ mod tests { - name: environment-3 "#}), ProjectManifest { + dependencies: vec![], canisters: vec![], networks: vec![], environments: vec![ @@ -399,6 +415,7 @@ mod tests { compute_allocation: 2 "#}), ProjectManifest { + dependencies: vec![], canisters: vec![], networks: vec![], environments: vec![Item::Manifest(EnvironmentManifest { @@ -440,6 +457,7 @@ mod tests { format: hex "#}), ProjectManifest { + dependencies: vec![], canisters: vec![], networks: vec![], environments: vec![Item::Manifest(EnvironmentManifest { diff --git a/crates/icp/src/network/custom_domains.rs b/crates/icp/src/network/custom_domains.rs index 9938a070..d16c1cf5 100644 --- a/crates/icp/src/network/custom_domains.rs +++ b/crates/icp/src/network/custom_domains.rs @@ -81,11 +81,16 @@ pub fn canister_gateway_url( let domain = gateway_domain(http_gateway_url); let mut url = http_gateway_url.clone(); match (friendly, domain) { - (Some((canister_name, env_name)), Some(domain)) => { + // A namespaced dependency canister (e.g. `dep:backend` or `vendor/dep:backend`) + // is not a valid DNS label, so it can't get a friendly subdomain; those fall + // through to the principal host below. + (Some((canister_name, env_name)), Some(domain)) + if !canister_name.contains([':', '/', '\\']) => + { url.set_host(Some(&format!("{canister_name}.{env_name}.{domain}"))) .expect("friendly domain should be a valid host"); } - (None, Some(domain)) => { + (_, Some(domain)) => { url.set_host(Some(&format!("{canister_id}.{domain}"))) .expect("principal domain should be a valid host"); } @@ -181,6 +186,19 @@ mod tests { assert_eq!(url.as_str(), "http://backend.local.localhost:8000/"); } + #[test] + fn canister_gateway_url_namespaced_name_falls_back_to_principal() { + let base: Url = "http://localhost:8000".parse().unwrap(); + let cid = Principal::from_text("bkyz2-fmaaa-aaaaa-qaaaq-cai").unwrap(); + // A namespaced dependency canister (e.g. `dep:backend`) is not a valid DNS + // label, so the friendly host is rejected and we fall back to the principal. + let url = canister_gateway_url(&base, cid, Some(("dep:backend", "local"))); + assert_eq!( + url.as_str(), + "http://bkyz2-fmaaa-aaaaa-qaaaq-cai.localhost:8000/" + ); + } + #[test] fn canister_gateway_url_without_friendly_domain() { let base: Url = "http://localhost:8000".parse().unwrap(); diff --git a/crates/icp/src/project.rs b/crates/icp/src/project.rs index 3b8bcf6b..08c172fc 100644 --- a/crates/icp/src/project.rs +++ b/crates/icp/src/project.rs @@ -1,4 +1,4 @@ -use std::collections::{HashMap, HashSet, hash_map::Entry}; +use std::collections::{BTreeMap, HashMap, HashSet, hash_map::Entry}; use indexmap::{IndexMap, map::Entry as IndexEntry}; @@ -6,13 +6,13 @@ use snafu::prelude::*; use crate::{ Canister, Environment, InitArgs, Network, Project, - canister::recipe, + canister::{ControllerRef, Settings, recipe}, context::IC_ROOT_KEY, fs, manifest::{ - ArgsFormat, CANISTER_MANIFEST, CanisterManifest, EnvironmentManifest, Item, - LoadManifestFromPathError, ManifestInitArgs, NetworkManifest, ProjectManifest, - ProjectRootLocateError, + ArgsFormat, CANISTER_MANIFEST, CanisterManifest, DependencyManifest, EnvironmentManifest, + Item, LoadManifestFromPathError, ManifestInitArgs, NetworkManifest, PROJECT_MANIFEST, + ProjectManifest, ProjectRootLocateError, canister::{Instructions, SyncSteps}, environment::CanisterSelection, load_manifest_from_path, @@ -105,6 +105,44 @@ pub enum ConsolidateManifestError { controller: String, }, + #[snafu(display( + "canister name '{name}' is invalid: ':' is reserved as the dependency namespace separator" + ))] + InvalidCanisterName { name: String }, + + #[snafu(display( + "dependency alias '{alias}' is invalid: ':' is reserved as the dependency namespace separator" + ))] + InvalidDependencyAlias { alias: String }, + + #[snafu(display("project declares two dependencies with the same alias '{alias}'"))] + DuplicateDependencyAlias { alias: String }, + + #[snafu(display( + "dependency alias '{alias}' collides with a canister of the same name in the same project" + ))] + DependencyAliasCollision { alias: String }, + + #[snafu(display("could not find a project manifest for dependency '{alias}' at: '{path}'"))] + DependencyNotFound { alias: String, path: String }, + + #[snafu(display("failed to canonicalize path for dependency '{alias}' at: '{path}'"))] + DependencyCanonicalize { alias: String, path: String }, + + #[snafu(display("failed to load project manifest for dependency '{alias}'"))] + LoadDependencyManifest { + source: LoadManifestFromPathError, + alias: String, + }, + + #[snafu(display( + "dependency '{alias}' selects canister '{canister}', which the dependency does not declare" + ))] + UnknownDependencyCanister { alias: String, canister: String }, + + #[snafu(display("dependency cycle detected: {chain}"))] + CircularDependency { chain: String }, + #[snafu(transparent)] Environment { source: EnvironmentError }, } @@ -152,24 +190,19 @@ fn is_glob(s: &str) -> bool { s.contains('*') || s.contains('?') || s.contains('[') || s.contains('{') } -/// Turns the ProjectManifest into a Project struct -/// - Adds the default Networks -/// - Adds the default Environment -/// - Validates the manifest to make sure that: -/// - There are no duplicates -/// - All the environments have networks -/// - All the referenced canisters exist -/// - All the recipes have been resolved -pub async fn consolidate_manifest( +/// Builds the canonical canisters declared directly in one project manifest, +/// resolving glob/path/inline entries, recipes, and init-args relative to +/// `pdir`. Returns `(local name, canister dir, canister)` with empty bindings; +/// callers assign store keys and bindings. Does not check for duplicate names +/// across projects — that is the caller's responsibility (via the global map). +async fn build_manifest_canisters( pdir: &Path, + manifest_canisters: &[Item], recipe_resolver: &dyn recipe::Resolve, - m: &ProjectManifest, -) -> Result { - // Canisters. IndexMap (not HashMap) so the order from the project manifest is preserved - // through to consumers like `icp project bundle`, which needs reproducible output. - let mut canisters: IndexMap = IndexMap::new(); +) -> Result, ConsolidateManifestError> { + let mut result: Vec<(String, PathBuf, Canister)> = Vec::new(); - for i in &m.canisters { + for i in manifest_canisters { let ms = match i { Item::Path(pattern) => { let is_glob_pattern = is_glob(pattern); @@ -179,7 +212,6 @@ pub async fn consolidate_manifest( // Glob pattern true => { - // Resolve glob let paths = glob::glob(pdir.join(pattern).as_str()).context(GlobParseSnafu)?; @@ -217,40 +249,34 @@ pub async fn consolidate_manifest( }; let mut ms = vec![]; - for p in paths { ms.push(( - // - // Canister root p.to_owned(), - // - // Canister manifest load_manifest_from_path::(&p.join(CANISTER_MANIFEST)) .await .context(LoadCanisterSnafu)?, )); } - ms } - Item::Manifest(m) => vec![( - // - // Canister root - pdir.to_owned(), - // - // Canister manifest - m.to_owned(), - )], + Item::Manifest(m) => vec![(pdir.to_owned(), m.to_owned())], }; for (cdir, m) in ms { + // `:` is reserved as the dependency namespace separator in store keys + // and `PUBLIC_CANISTER_ID:` env vars. + if m.name.contains(':') { + return InvalidCanisterNameSnafu { + name: m.name.clone(), + } + .fail(); + } + let registry_recipe = match &m.instructions { Instructions::BuildSync { .. } => None, Instructions::Recipe { recipe } => match &recipe.recipe_type { - crate::manifest::recipe::RecipeType::Registry { .. } => { - Some(recipe.recipe_type.to_string()) - } + RecipeType::Registry { .. } => Some(recipe.recipe_type.to_string()), _ => None, }, }; @@ -279,45 +305,536 @@ pub async fn consolidate_manifest( } }; - // Check for duplicates - match canisters.entry(m.name.to_owned()) { - // Duplicate - IndexEntry::Occupied(e) => { - return DuplicateSnafu { - kind: "canister".to_string(), - name: e.key().to_owned(), + let init_args = m + .init_args + .as_ref() + .map(|mia| resolve_manifest_init_args(mia, &cdir, &m.name)) + .transpose()?; + + result.push(( + m.name.clone(), + cdir, + Canister { + name: m.name.clone(), + settings: m.settings.clone(), + build, + sync, + init_args, + registry_recipe, + bindings: BTreeMap::new(), + }, + )); + } + } + + Ok(result) +} + +/// A dependency instance's own canisters, as `(local name, full store key)`. +/// Returned by [`import_dependency`] and cached per canonical path so diamond +/// dependencies reuse the same instance. +type ImportedInstance = Vec<(String, String)>; + +/// A member environment's per-canister config, to be folded into the root's +/// same-named environment beneath any root overrides. +#[derive(Default, Clone)] +struct MemberCanisterOverride { + settings: Option, + init_args: Option, +} + +/// Per-environment member overrides: env name → store key → override. +type MemberEnvOverrides = HashMap>; + +/// A member's identity (store-key prefix) and the environment names it defines, +/// used to enforce that a member declares every environment the root targets +/// (strict rule). +struct MemberEnvInfo { + prefix: String, + defined: HashSet, +} + +/// Canonicalize a dependency root (resolving symlinks and `..`) for use as a +/// de-dup / cycle-detection identity. +fn canonicalize_dep(alias: &str, dep_root: &Path) -> Result { + let build_err = || { + DependencyCanonicalizeSnafu { + alias: alias.to_owned(), + path: dep_root.to_string(), + } + .build() + }; + let canon = dunce::canonicalize(dep_root.as_std_path()).map_err(|_| build_err())?; + PathBuf::try_from(canon).map_err(|_| build_err()) +} + +/// Store-key prefix for a dependency instance: its canonical directory relative +/// to the canonical app root, forward-slash separated so keys are stable across +/// platforms and independent of how each edge spells the path. +fn relative_prefix(app_root_canonical: &Path, dep_canonical: &Path) -> String { + let rel = pathdiff::diff_utf8_paths(dep_canonical, app_root_canonical) + .unwrap_or_else(|| dep_canonical.to_owned()); + rel.as_str().replace('\\', "/") +} + +/// Rewrite `CanisterName` controller references from a dependency's local +/// canister names to their store keys, so global controller validation and +/// deploy-time id lookup operate uniformly on store keys. +fn translate_controllers(canister: &mut Canister, local_to_key: &BTreeMap) { + if let Some(controllers) = &mut canister.settings.controllers { + for cref in controllers.iter_mut() { + if let ControllerRef::CanisterName(name) = cref + && let Some(key) = local_to_key.get(name) + { + *name = key.clone(); + } + } + } +} + +/// Compute the `PUBLIC_CANISTER_ID` env-var wiring for canisters in one project +/// scope: its own canisters by local name, plus each dependency's exposed +/// canisters under `:`. +fn compute_bindings( + own: &[(String, String)], + edges: &[(String, Vec<(String, String)>)], +) -> BTreeMap { + let mut bindings = BTreeMap::new(); + for (local, key) in own { + bindings.insert(local.clone(), key.clone()); + } + for (alias, exposed) in edges { + for (dep_local, key) in exposed { + bindings.insert(format!("{alias}:{dep_local}"), key.clone()); + } + } + bindings +} + +/// Select which of a dependency instance's own canisters are exposed to the +/// parent, per the dependency's `canisters` selection. +fn select_exposed( + inst: &ImportedInstance, + selection: &CanisterSelection, + alias: &str, +) -> Result, ConsolidateManifestError> { + match selection { + CanisterSelection::Everything => Ok(inst.clone()), + CanisterSelection::None => Ok(vec![]), + CanisterSelection::Named(names) => { + let mut out = Vec::new(); + for name in names { + match inst.iter().find(|(local, _)| local == name) { + Some(pair) => out.push(pair.clone()), + None => { + return UnknownDependencyCanisterSnafu { + alias: alias.to_owned(), + canister: name.clone(), + } + .fail(); + } + } + } + Ok(out) + } + } +} + +/// Validate the dependency aliases declared in one project scope: no `:`, no +/// collision with a local canister name, and no duplicate alias. +fn validate_dependency_aliases( + deps: &[DependencyManifest], + own_canister_names: &HashSet, +) -> Result<(), ConsolidateManifestError> { + let mut seen: HashSet<&str> = HashSet::new(); + for d in deps { + if d.name.contains(':') { + return InvalidDependencyAliasSnafu { + alias: d.name.clone(), + } + .fail(); + } + if own_canister_names.contains(&d.name) { + return DependencyAliasCollisionSnafu { + alias: d.name.clone(), + } + .fail(); + } + if !seen.insert(&d.name) { + return DuplicateDependencyAliasSnafu { + alias: d.name.clone(), + } + .fail(); + } + } + Ok(()) +} + +/// Recursively import a dependency's canisters into `canisters`, keyed by their +/// app-root-relative store keys. De-duplicates instances by canonical path +/// (diamond dependencies deploy once) and detects cycles. Returns the imported +/// instance's prefix and its own canisters. +#[allow(clippy::too_many_arguments)] +async fn import_dependency( + app_root_canonical: &Path, + parent_dir: &Path, + dep: &DependencyManifest, + recipe_resolver: &dyn recipe::Resolve, + canisters: &mut IndexMap, + registry: &mut HashMap, + stack: &mut Vec, + member_env_overrides: &mut MemberEnvOverrides, + members: &mut Vec, +) -> Result { + let dep_root = parent_dir.join(&dep.path); + let manifest_path = dep_root.join(PROJECT_MANIFEST); + if !manifest_path.is_file() { + return DependencyNotFoundSnafu { + alias: dep.name.clone(), + path: dep_root.to_string(), + } + .fail(); + } + + let canonical = canonicalize_dep(&dep.name, &dep_root)?; + + // Cycle detection. + if stack.contains(&canonical) { + let mut chain: Vec = stack.iter().map(|p| p.to_string()).collect(); + chain.push(canonical.to_string()); + return CircularDependencySnafu { + chain: chain.join(" -> "), + } + .fail(); + } + + // Diamond de-dup: same resolved directory means the same instance. + if let Some(inst) = registry.get(&canonical) { + return Ok(inst.clone()); + } + + stack.push(canonical.clone()); + + let prefix = relative_prefix(app_root_canonical, &canonical); + + let dep_manifest: ProjectManifest = + load_manifest_from_path(&manifest_path) + .await + .context(LoadDependencyManifestSnafu { + alias: dep.name.clone(), + })?; + + // Build the dependency's own canisters and key them under the prefix. All of + // them are imported (deploy-all); the `canisters` exposure subset is applied + // by the caller when wiring env vars. + let built = + build_manifest_canisters(&dep_root, &dep_manifest.canisters, recipe_resolver).await?; + + let mut own: Vec<(String, String)> = Vec::new(); + let mut local_to_key: BTreeMap = BTreeMap::new(); + for (local, cdir, mut canister) in built { + let store_key = format!("{prefix}:{local}"); + canister.name = store_key.clone(); + own.push((local.clone(), store_key.clone())); + local_to_key.insert(local.clone(), store_key.clone()); + match canisters.entry(store_key.clone()) { + IndexEntry::Occupied(_) => { + return DuplicateSnafu { + kind: "canister".to_string(), + name: store_key, + } + .fail(); + } + IndexEntry::Vacant(e) => { + e.insert((cdir, canister)); + } + } + } + + // Now that every sibling's store key is known, translate the dependency's + // controller references (local sibling name -> store key). + for (_, key) in &own { + if let Some((_, canister)) = canisters.get_mut(key) { + translate_controllers(canister, &local_to_key); + } + } + + // Capture the member's own environments so the parent can honor its + // per-canister settings/init_args for the same-named environment + // (standalone-equivalence). The network binding and canister selection are + // ignored; only overrides on the member's *own* canisters are + // folded in — keys naming its dependencies are left to those dependencies. + let mut defined_envs: HashSet = HashSet::new(); + for env_item in &dep_manifest.environments { + let em: EnvironmentManifest = match env_item { + Item::Manifest(m) => m.clone(), + Item::Path(path) => { + let p = dep_root.join(path); + if !p.is_file() { + return NotFoundSnafu { + kind: "environment".to_string(), + path: p.to_string(), } .fail(); } + load_manifest_from_path::(&p) + .await + .context(LoadEnvironmentSnafu)? + } + }; + defined_envs.insert(em.name.clone()); + if let Some(settings) = &em.settings { + for (local, s) in settings { + if let Some(key) = local_to_key.get(local) { + member_env_overrides + .entry(em.name.clone()) + .or_default() + .entry(key.clone()) + .or_default() + .settings = Some(s.clone()); + } + } + } + if let Some(init_args) = &em.init_args { + for (local, ia) in init_args { + if let Some(key) = local_to_key.get(local) { + member_env_overrides + .entry(em.name.clone()) + .or_default() + .entry(key.clone()) + .or_default() + .init_args = Some(ia.clone()); + } + } + } + } + members.push(MemberEnvInfo { + prefix: prefix.clone(), + defined: defined_envs, + }); + + // Recurse into the dependency's own dependencies. + let own_names: HashSet = own.iter().map(|(l, _)| l.clone()).collect(); + validate_dependency_aliases(&dep_manifest.dependencies, &own_names)?; + + let mut edges: Vec<(String, Vec<(String, String)>)> = Vec::new(); + for nested in &dep_manifest.dependencies { + let inst = Box::pin(import_dependency( + app_root_canonical, + &dep_root, + nested, + recipe_resolver, + canisters, + registry, + stack, + member_env_overrides, + members, + )) + .await?; + let exposed = select_exposed(&inst, &nested.canisters, &nested.name)?; + edges.push((nested.name.clone(), exposed)); + } - // Ok - IndexEntry::Vacant(e) => { - let init_args = m - .init_args - .as_ref() - .map(|mia| resolve_manifest_init_args(mia, &cdir, &m.name)) - .transpose()?; - - e.insert(( - // - // Canister root - cdir, - // - // Canister - Canister { - name: m.name.to_owned(), - settings: m.settings.to_owned(), - build, - sync, - init_args, - registry_recipe, - }, - )); + // Assign env-var bindings for this instance's own canisters. + let bindings = compute_bindings(&own, &edges); + for (_, key) in &own { + if let Some((_, canister)) = canisters.get_mut(key) { + canister.bindings = bindings.clone(); + } + } + + stack.pop(); + registry.insert(canonical, own.clone()); + Ok(own) +} + +/// Canonicalize into a UTF-8 path, or `None` if it does not exist / is not UTF-8. +fn canonicalize_or(dir: &Path) -> Option { + let canon = dunce::canonicalize(dir.as_std_path()).ok()?; + PathBuf::try_from(canon).ok() +} + +/// The default set of target canisters when the user names none, honoring +/// member-scoping. +/// +/// When the command is run inside a vendored member — `member_dir` is a distinct +/// directory below the workspace `root_dir` — only that member's canisters are +/// targeted: those whose directory lies within `member_dir` (the member's own +/// canisters plus any dependencies nested under it). Dependencies hoisted +/// outside the member are assumed already deployed and keep their ids in the +/// shared root store, so cross-member wiring stays valid. +/// +/// Returns `None` meaning "no scoping — target the whole environment": at the +/// workspace root or a standalone project (`member_dir` resolves to `root_dir`), +/// when `member_dir` is unknown, or when paths cannot be resolved. +pub fn member_scoped_canisters( + root_dir: &Path, + member_dir: Option<&Path>, + env: &Environment, +) -> Option> { + let member = member_dir?; + let root_c = canonicalize_or(root_dir)?; + let member_c = canonicalize_or(member)?; + if root_c == member_c { + return None; + } + + let names = env + .canisters + .iter() + .filter(|(_, (dir, _))| { + canonicalize_or(dir).is_some_and(|c| c == member_c || c.starts_with(&member_c)) + }) + .map(|(name, _)| name.clone()) + .collect(); + Some(names) +} + +/// Build one environment's canister map: select from `canisters`, then apply the +/// member overrides for this environment (standalone-equivalence), then +/// the root's own overrides (highest precedence). Precedence is therefore +/// root-explicit > member-env > canister-base. +fn build_environment_canisters( + canisters: &IndexMap, + env_name: &str, + selection: &CanisterSelection, + member_overrides: Option<&HashMap>, + root_settings: Option<&HashMap>, + root_init_args: Option<&HashMap>, +) -> Result, ConsolidateManifestError> { + let mut cs = match selection { + CanisterSelection::None => IndexMap::new(), + CanisterSelection::Everything => canisters.clone(), + CanisterSelection::Named(names) => { + let mut cs: IndexMap = IndexMap::new(); + for name in names { + let v = canisters.get(name).ok_or( + InvalidCanisterSnafu { + environment: env_name.to_owned(), + canister: name.to_owned(), + } + .build(), + )?; + cs.insert(name.to_owned(), v.to_owned()); + } + cs + } + }; + + // Member overrides first (lower precedence than the root's own overrides). + if let Some(overrides) = member_overrides { + for (key, ov) in overrides { + if let Some((cpath, canister)) = cs.get_mut(key) { + if let Some(s) = &ov.settings { + canister.settings = s.clone(); + } + if let Some(ia) = &ov.init_args { + canister.init_args = Some(resolve_manifest_init_args(ia, cpath, key)?); + } + } + } + } + + // Root overrides last (highest precedence). + if let Some(settings) = root_settings { + for (name, s) in settings { + if let Some((_p, canister)) = cs.get_mut(name) { + canister.settings = s.clone(); + } + } + } + if let Some(init_args) = root_init_args { + for (name, ia) in init_args { + if let Some((cpath, canister)) = cs.get_mut(name) { + canister.init_args = Some(resolve_manifest_init_args(ia, cpath, name)?); + } + } + } + + Ok(cs) +} + +/// Turns the ProjectManifest into a Project struct +/// - Adds the default Networks +/// - Adds the default Environment +/// - Imports any dependency projects' canisters +/// - Validates the manifest to make sure that: +/// - There are no duplicates +/// - All the environments have networks +/// - All the referenced canisters exist +/// - All the recipes have been resolved +pub async fn consolidate_manifest( + pdir: &Path, + recipe_resolver: &dyn recipe::Resolve, + m: &ProjectManifest, +) -> Result { + // Canisters. IndexMap (not HashMap) so the order from the project manifest is preserved + // through to consumers like `icp project bundle`, which needs reproducible output. + let mut canisters: IndexMap = IndexMap::new(); + + // Canonical app root, used to derive stable, order-independent store-key + // prefixes for imported dependency canisters. + let app_root_canonical = + canonicalize_dep("", pdir).unwrap_or_else(|_| pdir.to_owned()); + + // This project's own canisters, keyed by their bare local names. + let app_built = build_manifest_canisters(pdir, &m.canisters, recipe_resolver).await?; + let mut app_own: Vec<(String, String)> = Vec::new(); + for (local, cdir, canister) in app_built { + app_own.push((local.clone(), local.clone())); + match canisters.entry(local.clone()) { + IndexEntry::Occupied(e) => { + return DuplicateSnafu { + kind: "canister".to_string(), + name: e.key().to_owned(), } + .fail(); + } + IndexEntry::Vacant(e) => { + e.insert((cdir, canister)); } } } + // Import dependency projects. Each dependency is deployed in full and keyed + // under its app-root-relative path; diamonds (the same directory reached via + // multiple edges) resolve to a single instance. + let mut registry: HashMap = HashMap::new(); + let mut stack: Vec = Vec::new(); + // Member environment config folded into the root's same-named environments, + // and the per-member set of declared environment names for the strict rule. + let mut member_env_overrides: MemberEnvOverrides = HashMap::new(); + let mut members: Vec = Vec::new(); + let app_own_names: HashSet = app_own.iter().map(|(l, _)| l.clone()).collect(); + validate_dependency_aliases(&m.dependencies, &app_own_names)?; + + let mut app_edges: Vec<(String, Vec<(String, String)>)> = Vec::new(); + for dep in &m.dependencies { + let inst = import_dependency( + &app_root_canonical, + pdir, + dep, + recipe_resolver, + &mut canisters, + &mut registry, + &mut stack, + &mut member_env_overrides, + &mut members, + ) + .await?; + let exposed = select_exposed(&inst, &dep.canisters, &dep.name)?; + app_edges.push((dep.name.clone(), exposed)); + } + + // Assign env-var bindings for this project's own canisters (own canisters by + // local name plus each dependency's exposed canisters under `:`). + let app_bindings = compute_bindings(&app_own, &app_edges); + for (_, key) in &app_own { + if let Some((_, canister)) = canisters.get_mut(key) { + canister.bindings = app_bindings.clone(); + } + } + // Validate that every canister-name controller reference points to a declared canister. // Catching typos here turns "perpetual warning" into a clear load-time error. for (canister_name, (_, canister)) in &canisters { @@ -485,59 +1002,16 @@ pub async fn consolidate_manifest( v.to_owned() }, - // Embed canisters in environment - canisters: { - let mut cs = match &m.canisters { - // None - CanisterSelection::None => IndexMap::new(), - - // Everything - CanisterSelection::Everything => canisters.clone(), - - // Named - CanisterSelection::Named(names) => { - let mut cs: IndexMap = IndexMap::new(); - - for name in names { - let v = canisters.get(name).ok_or( - InvalidCanisterSnafu { - environment: m.name.to_owned(), - canister: name.to_owned(), - } - .build(), - )?; - - cs.insert(name.to_owned(), v.to_owned()); - } - - cs - } - }; - - // Apply settings overrides if specified - if let Some(ref settings_overrides) = m.settings { - for (canister_name, settings) in settings_overrides { - if let Some((_path, canister)) = cs.get_mut(canister_name) { - canister.settings = settings.clone(); - } - } - } - - // Apply init_args overrides if specified - if let Some(ref init_args_overrides) = m.init_args { - for (canister_name, manifest_init_args) in init_args_overrides { - if let Some((canister_path, canister)) = cs.get_mut(canister_name) { - canister.init_args = Some(resolve_manifest_init_args( - manifest_init_args, - canister_path, - canister_name, - )?); - } - } - } - - cs - }, + // Embed canisters in environment, folding member overrides + // beneath the root's own settings/init_args overrides. + canisters: build_environment_canisters( + &canisters, + &m.name, + &m.canisters, + member_env_overrides.get(&m.name), + m.settings.as_ref(), + m.init_args.as_ref(), + )?, }); } } @@ -546,42 +1020,591 @@ pub async fn consolidate_manifest( // We're done adding all the user environments // Now we add the implicit `local` and `ic` environment if the user hasn't overriden it if let Entry::Vacant(vacant_entry) = environments.entry(LOCAL.to_string()) { + let network = networks + .get(LOCAL) + .ok_or( + InvalidNetworkSnafu { + environment: LOCAL.to_owned(), + network: LOCAL.to_owned(), + } + .build(), + )? + .to_owned(); vacant_entry.insert(Environment { name: LOCAL.to_string(), - network: networks - .get(LOCAL) - .ok_or( - InvalidNetworkSnafu { - environment: LOCAL.to_owned(), - network: LOCAL.to_owned(), - } - .build(), - )? - .to_owned(), - canisters: canisters.clone(), + network, + canisters: build_environment_canisters( + &canisters, + LOCAL, + &CanisterSelection::Everything, + member_env_overrides.get(LOCAL), + None, + None, + )?, }); } if let Entry::Vacant(vacant_entry) = environments.entry(IC.to_string()) { + let network = networks + .get(IC) + .ok_or( + InvalidNetworkSnafu { + environment: IC.to_owned(), + network: IC.to_owned(), + } + .build(), + )? + .to_owned(); vacant_entry.insert(Environment { name: IC.to_string(), - network: networks - .get(IC) - .ok_or( - InvalidNetworkSnafu { - environment: IC.to_owned(), - network: IC.to_owned(), - } - .build(), - )? - .to_owned(), - canisters: canisters.clone(), + network, + canisters: build_environment_canisters( + &canisters, + IC, + &CanisterSelection::Everything, + member_env_overrides.get(IC), + None, + None, + )?, }); } + // Strict rule: every member must declare each environment the root targets. + // `local`/`ic` are implicit for every project, so they never count + // as missing; other environments must be declared explicitly by the member. + // Recorded per-environment and enforced lazily when that environment is + // selected (so a missing `staging` never blocks `deploy -e local`). + let mut member_missing_envs: HashMap> = HashMap::new(); + for env_name in environments.keys() { + if env_name == LOCAL || env_name == IC { + continue; + } + for member in &members { + if !member.defined.contains(env_name) { + member_missing_envs + .entry(env_name.clone()) + .or_default() + .push(member.prefix.clone()); + } + } + } + Ok(Project { dir: pdir.into(), canisters, networks, environments, + member_missing_envs, }) } + +#[cfg(test)] +mod dependency_tests { + use super::*; + use crate::canister::recipe::{RecipeContext, Resolve, ResolveError}; + use crate::manifest::canister::{BuildSteps, SyncSteps}; + use crate::manifest::recipe::Recipe; + use camino_tempfile::Utf8TempDir; + + /// Recipes are never used in these tests; every canister is pre-built. + struct PanicResolver; + + #[async_trait::async_trait] + impl Resolve for PanicResolver { + async fn resolve( + &self, + _recipe: &Recipe, + _context: &RecipeContext, + ) -> Result<(BuildSteps, SyncSteps), ResolveError> { + panic!("recipe resolver should not be called in dependency tests"); + } + } + + fn write(dir: &Path, rel: &str, contents: &str) { + let p = dir.join(rel); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(p, contents).unwrap(); + } + + /// A minimal `icp.yaml` body declaring the given pre-built canisters, + /// followed by a raw `dependencies:` block (may be empty). + fn manifest(canisters: &[&str], deps: &str) -> String { + let mut s = String::new(); + if canisters.is_empty() { + s.push_str("canisters: []\n"); + } else { + s.push_str("canisters:\n"); + for c in canisters { + s.push_str(&format!( + " - name: {c}\n build:\n steps:\n - type: pre-built\n path: {c}.wasm\n" + )); + } + } + s.push_str(deps); + s + } + + async fn consolidate(pdir: &Path) -> Result { + let m: ProjectManifest = load_manifest_from_path(&pdir.join(PROJECT_MANIFEST)) + .await + .expect("failed to parse project manifest"); + consolidate_manifest(pdir, &PanicResolver, &m).await + } + + fn bindings_of<'a>(p: &'a Project, key: &str) -> &'a BTreeMap { + &p.canisters + .get(key) + .unwrap_or_else(|| { + panic!( + "canister '{key}' not found; have {:?}", + p.canisters.keys().collect::>() + ) + }) + .1 + .bindings + } + + #[tokio::test] + async fn single_project_bindings_are_self_and_siblings() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "icp.yaml", + &manifest(&["backend", "frontend"], ""), + ); + + let p = consolidate(tmp.path()).await.unwrap(); + + // Flat behavior preserved: every canister maps every sibling (incl. self) + // to itself. + let expected = BTreeMap::from([ + ("backend".to_string(), "backend".to_string()), + ("frontend".to_string(), "frontend".to_string()), + ]); + assert_eq!(bindings_of(&p, "backend"), &expected); + assert_eq!(bindings_of(&p, "frontend"), &expected); + } + + #[tokio::test] + async fn dependency_import_and_exposure_subset() { + let tmp = Utf8TempDir::new().unwrap(); + // Dependency nested inside the app (mirrors a submodule under the app). + write( + tmp.path(), + "openemail/icp.yaml", + &manifest(&["backend", "frontend"], ""), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &["backend"], + "dependencies:\n - name: openemail\n path: ./openemail\n canisters: [backend]\n", + ), + ); + + let p = consolidate(tmp.path()).await.unwrap(); + + // The whole dependency is deployed (both canisters imported), keyed by path. + assert!(p.canisters.contains_key("backend")); + assert!(p.canisters.contains_key("openemail:backend")); + assert!(p.canisters.contains_key("openemail:frontend")); + + // App's own canister sees itself and only the *exposed* dependency canister. + assert_eq!( + bindings_of(&p, "backend"), + &BTreeMap::from([ + ("backend".to_string(), "backend".to_string()), + ( + "openemail:backend".to_string(), + "openemail:backend".to_string() + ), + ]) + ); + + // The dependency's own canisters keep their standalone view (bare names). + assert_eq!( + bindings_of(&p, "openemail:backend"), + &BTreeMap::from([ + ("backend".to_string(), "openemail:backend".to_string()), + ("frontend".to_string(), "openemail:frontend".to_string()), + ]) + ); + } + + #[tokio::test] + async fn member_scope_targets_only_the_members_canisters() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "openemail/icp.yaml", + &manifest(&["backend", "frontend"], ""), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &["backend"], + "dependencies:\n - name: openemail\n path: ./openemail\n", + ), + ); + + let p = consolidate(tmp.path()).await.unwrap(); + let env = p.environments.get(LOCAL).expect("local environment"); + + // At the workspace root (member == root): no scoping. + assert_eq!(member_scoped_canisters(&p.dir, Some(&p.dir), env), None); + + // Unknown member dir: no scoping. + assert_eq!(member_scoped_canisters(&p.dir, None, env), None); + + // Inside the member: only the member's own canisters, not the app's. + let member = tmp.path().join("openemail"); + let mut scoped = member_scoped_canisters(&p.dir, Some(&member), env) + .expect("should scope when inside a member"); + scoped.sort(); + assert_eq!( + scoped, + vec![ + "openemail:backend".to_string(), + "openemail:frontend".to_string(), + ] + ); + } + + #[tokio::test] + async fn member_env_config_folds_in_with_root_override_winning() { + let tmp = Utf8TempDir::new().unwrap(); + // openemail defines `staging` with per-canister settings for its own + // canisters. + write( + tmp.path(), + "openemail/icp.yaml", + r#" +canisters: + - name: backend + build: + steps: + - type: pre-built + path: backend.wasm + - name: frontend + build: + steps: + - type: pre-built + path: frontend.wasm +environments: + - name: staging + settings: + backend: + compute_allocation: 5 + frontend: + compute_allocation: 7 +"#, + ); + // The app declares openemail and also defines `staging`, overriding the + // imported backend's settings (the root override must win). + write( + tmp.path(), + "icp.yaml", + r#" +canisters: + - name: app + build: + steps: + - type: pre-built + path: app.wasm +dependencies: + - name: openemail + path: ./openemail +environments: + - name: staging + settings: + "openemail:backend": + compute_allocation: 99 +"#, + ); + + let p = consolidate(tmp.path()).await.unwrap(); + let staging = p.environments.get("staging").expect("staging environment"); + + // Root override wins over the member's config. + assert_eq!( + staging + .canisters + .get("openemail:backend") + .unwrap() + .1 + .settings + .compute_allocation, + Some(99), + ); + // No root override → the member's own config applies (standalone-equivalence). + assert_eq!( + staging + .canisters + .get("openemail:frontend") + .unwrap() + .1 + .settings + .compute_allocation, + Some(7), + ); + // Both projects declared staging, so nothing is recorded as missing. + assert!(p.member_missing_envs.is_empty()); + } + + #[tokio::test] + async fn missing_member_environment_is_recorded() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "openemail/icp.yaml", + &manifest(&["backend"], ""), + ); + write( + tmp.path(), + "icp.yaml", + r#" +canisters: + - name: app + build: + steps: + - type: pre-built + path: app.wasm +dependencies: + - name: openemail + path: ./openemail +environments: + - name: staging +"#, + ); + + let p = consolidate(tmp.path()).await.unwrap(); + // openemail does not declare `staging`, so it is recorded as missing. + assert_eq!( + p.member_missing_envs.get("staging"), + Some(&vec!["openemail".to_string()]), + ); + // Implicit environments are never recorded as missing. + assert!(!p.member_missing_envs.contains_key("local")); + assert!(!p.member_missing_envs.contains_key("ic")); + } + + #[tokio::test] + async fn diamond_dedups_to_single_instance() { + let tmp = Utf8TempDir::new().unwrap(); + // umbrella layout: service-a and service-b both depend on ../openemail. + write( + tmp.path(), + "umbrella/openemail/icp.yaml", + &manifest(&["backend"], ""), + ); + write( + tmp.path(), + "umbrella/service-a/icp.yaml", + &manifest( + &["backend"], + "dependencies:\n - name: openemail\n path: ../openemail\n", + ), + ); + write( + tmp.path(), + "umbrella/service-b/icp.yaml", + &manifest( + &["backend"], + "dependencies:\n - name: openemail\n path: ../openemail\n", + ), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &[], + "dependencies:\n - name: service-a\n path: ./umbrella/service-a\n - name: service-b\n path: ./umbrella/service-b\n", + ), + ); + + let p = consolidate(tmp.path()).await.unwrap(); + + // openemail is imported exactly once despite two edges reaching it. + let openemail_keys: Vec<_> = p + .canisters + .keys() + .filter(|k| k.contains("openemail")) + .collect(); + assert_eq!( + openemail_keys, + vec![&"umbrella/openemail:backend".to_string()], + "expected a single shared openemail instance" + ); + + // Both services' code reads `openemail:backend`, resolving to the one instance. + assert_eq!( + bindings_of(&p, "umbrella/service-a:backend").get("openemail:backend"), + Some(&"umbrella/openemail:backend".to_string()) + ); + assert_eq!( + bindings_of(&p, "umbrella/service-b:backend").get("openemail:backend"), + Some(&"umbrella/openemail:backend".to_string()) + ); + } + + #[tokio::test] + async fn cycle_is_detected() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "icp.yaml", + &manifest(&[], "dependencies:\n - name: a\n path: ./a\n"), + ); + write( + tmp.path(), + "a/icp.yaml", + &manifest(&["x"], "dependencies:\n - name: b\n path: ../b\n"), + ); + write( + tmp.path(), + "b/icp.yaml", + &manifest(&["y"], "dependencies:\n - name: a\n path: ../a\n"), + ); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!(err, ConsolidateManifestError::CircularDependency { .. }), + "expected CircularDependency, got {err:?}" + ); + } + + #[tokio::test] + async fn alias_colliding_with_canister_name_is_rejected() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "openemail/icp.yaml", + &manifest(&["backend"], ""), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &["openemail"], + "dependencies:\n - name: openemail\n path: ./openemail\n", + ), + ); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!( + err, + ConsolidateManifestError::DependencyAliasCollision { .. } + ), + "got {err:?}" + ); + } + + #[tokio::test] + async fn duplicate_alias_is_rejected() { + let tmp = Utf8TempDir::new().unwrap(); + write(tmp.path(), "one/icp.yaml", &manifest(&["backend"], "")); + write(tmp.path(), "two/icp.yaml", &manifest(&["backend"], "")); + write( + tmp.path(), + "icp.yaml", + &manifest( + &[], + "dependencies:\n - name: dup\n path: ./one\n - name: dup\n path: ./two\n", + ), + ); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!( + err, + ConsolidateManifestError::DuplicateDependencyAlias { .. } + ), + "got {err:?}" + ); + } + + #[tokio::test] + async fn colon_in_canister_name_is_rejected() { + let tmp = Utf8TempDir::new().unwrap(); + write(tmp.path(), "icp.yaml", &manifest(&["foo:bar"], "")); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!(err, ConsolidateManifestError::InvalidCanisterName { .. }), + "got {err:?}" + ); + } + + #[tokio::test] + async fn unknown_exposed_canister_is_rejected() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "openemail/icp.yaml", + &manifest(&["backend"], ""), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &[], + "dependencies:\n - name: openemail\n path: ./openemail\n canisters: [nope]\n", + ), + ); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!( + err, + ConsolidateManifestError::UnknownDependencyCanister { .. } + ), + "got {err:?}" + ); + } + + #[tokio::test] + async fn missing_dependency_path_is_rejected() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "icp.yaml", + &manifest( + &[], + "dependencies:\n - name: openemail\n path: ./does-not-exist\n", + ), + ); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!(err, ConsolidateManifestError::DependencyNotFound { .. }), + "got {err:?}" + ); + } + + #[tokio::test] + async fn imported_canisters_appear_in_implicit_environments() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "openemail/icp.yaml", + &manifest(&["backend"], ""), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &["backend"], + "dependencies:\n - name: openemail\n path: ./openemail\n", + ), + ); + + let p = consolidate(tmp.path()).await.unwrap(); + + // Deploy-all: the implicit `local` environment includes the dependency. + let local = p.environments.get("local").unwrap(); + assert!(local.canisters.contains_key("backend")); + assert!(local.canisters.contains_key("openemail:backend")); + } +} diff --git a/crates/icp/src/store_artifact.rs b/crates/icp/src/store_artifact.rs index 602b84ad..0f2a715d 100644 --- a/crates/icp/src/store_artifact.rs +++ b/crates/icp/src/store_artifact.rs @@ -53,9 +53,30 @@ struct ArtifactPaths { dir: PathBuf, } +/// Encode a canister name into a single filename-safe segment. +/// +/// Canister names may be namespaced store keys containing `/` and `:` (imported +/// dependency canisters, e.g. `vendor/openemail:backend`), which are not valid +/// filename characters on every platform. Percent-encoding the unsafe set keeps +/// the mapping reversible and collision-free; plain names (alphanumeric/`-`/`_`/`.`) +/// are left unchanged, so existing artifact filenames are unaffected. +fn sanitize_artifact_name(name: &str) -> String { + let mut out = String::with_capacity(name.len()); + for c in name.chars() { + match c { + '%' => out.push_str("%25"), + '/' => out.push_str("%2F"), + '\\' => out.push_str("%5C"), + ':' => out.push_str("%3A"), + _ => out.push(c), + } + } + out +} + impl ArtifactPaths { fn artifact_by_name(&self, name: &str) -> PathBuf { - self.dir.join(name) + self.dir.join(sanitize_artifact_name(name)) } } @@ -162,3 +183,34 @@ impl Access for MockInMemoryArtifactStore { } } } + +#[cfg(test)] +mod tests { + use super::sanitize_artifact_name; + + #[test] + fn plain_names_unchanged() { + assert_eq!(sanitize_artifact_name("backend"), "backend"); + assert_eq!( + sanitize_artifact_name("my-canister_1.wasm"), + "my-canister_1.wasm" + ); + } + + #[test] + fn namespaced_names_are_filename_safe() { + let s = sanitize_artifact_name("vendor/openemail:backend"); + assert_eq!(s, "vendor%2Fopenemail%3Abackend"); + assert!(!s.contains('/')); + assert!(!s.contains(':')); + } + + #[test] + fn encoding_is_injective() { + // `%` is itself encoded, so a literal "%2F" never collides with "/". + assert_ne!( + sanitize_artifact_name("a%2Fb"), + sanitize_artifact_name("a/b") + ); + } +} diff --git a/docs-site/astro.config.mjs b/docs-site/astro.config.mjs index 556b5158..944feabe 100644 --- a/docs-site/astro.config.mjs +++ b/docs-site/astro.config.mjs @@ -165,6 +165,7 @@ export default defineConfig({ { label: 'Build, Deploy, Sync', slug: 'concepts/build-deploy-sync' }, { label: 'Environments and Networks', slug: 'concepts/environments' }, { label: 'Canister Discovery', slug: 'concepts/canister-discovery' }, + { label: 'Project Dependencies', slug: 'concepts/project-dependencies' }, { label: 'Binding Generation', slug: 'concepts/binding-generation' }, { label: 'Recipes', slug: 'concepts/recipes' }, { label: 'Sync Plugins', slug: 'concepts/sync-plugins' }, diff --git a/docs/concepts/canister-discovery.md b/docs/concepts/canister-discovery.md index 04bcdb38..eba433ed 100644 --- a/docs/concepts/canister-discovery.md +++ b/docs/concepts/canister-discovery.md @@ -133,6 +133,12 @@ If you prefer not to use canister environment variables: 1. **Init arguments** — Pass canister IDs as initialization parameters 2. **Configuration** — Store IDs in canister state during setup +## Dependency Projects + +A project can vendor another `icp` project as a dependency. IDs are injected per project scope: your canisters see the exposed dependency canisters under an alias (`PUBLIC_CANISTER_ID:openemail:backend`), while the dependency's own canisters keep their standalone view (`PUBLIC_CANISTER_ID:backend`, …). This per-project scoping means vendored code behaves the same whether deployed on its own or as a dependency. + +See [Project Dependencies](project-dependencies.md) for how to declare a dependency, deploy it as part of a workspace, and share a single set of canister IDs. + ## Custom Canister Environment Variables Beyond automatic `PUBLIC_CANISTER_ID:*` variables, you can define custom canister environment variables in `icp.yaml`. See the [Environment Variables Reference](../reference/environment-variables.md#custom-variables) for configuration syntax. diff --git a/docs/concepts/index.md b/docs/concepts/index.md index b067cfc1..06c8bede 100644 --- a/docs/concepts/index.md +++ b/docs/concepts/index.md @@ -13,6 +13,7 @@ Understanding how icp-cli organizes and manages your project. - [Recipes](recipes.md) — Templated, reusable build configurations - [Sync Plugins](sync-plugins.md) — Sandboxed WebAssembly components that run during the sync phase - [Canister Discovery](canister-discovery.md) — How canisters discover each other +- [Project Dependencies](project-dependencies.md) — Depending on other vendored icp projects and deploying them as a workspace - [Binding Generation](binding-generation.md) — Type-safe canister interfaces ## Quick Reference diff --git a/docs/concepts/project-dependencies.md b/docs/concepts/project-dependencies.md new file mode 100644 index 00000000..0ce6145e --- /dev/null +++ b/docs/concepts/project-dependencies.md @@ -0,0 +1,120 @@ +--- +title: Project Dependencies +description: How one icp project can depend on another vendored icp project, deploy it as part of a workspace, and share a single set of canister IDs. +--- + +An `icp` project can build on top of another `icp` project — a **dependency** — whose source is vendored into it, typically as a git submodule. The dependency stays a complete, standalone project: it can be developed and deployed on its own, and it does not need to know it is being consumed. + +This supports two related workflows: + +- **Reuse** — build on another project's canisters (your canisters call theirs). +- **Monorepo / workspace** — develop several projects together and deploy them onto one network with a single shared set of canister IDs. + +## Declaring a dependency + +Add a top-level `dependencies:` block to your `icp.yaml`: + +```yaml +dependencies: + - name: openemail # local alias — namespaces the dependency's canister IDs + path: ./vendor/openemail # directory containing the dependency's icp.yaml + canisters: [backend] # which of its canisters to expose (omit to expose all) +``` + +## What gets deployed + +`icp deploy` deploys **all** of the dependency's canisters into the same environment, not just the exposed ones. A dependency's canisters may call each other, and icp-cli does not track an internal "requires" graph, so the whole dependency is always deployed — exactly as it would deploy on its own. `canisters:` is an **exposure** filter (which IDs your canisters see), not a deployment filter. + +## Canister ID injection + +Each canister receives canister IDs from the perspective of the project that owns it. Your canisters see: + +- their own canisters by name — `PUBLIC_CANISTER_ID:backend` +- each **exposed** dependency canister under the alias — `PUBLIC_CANISTER_ID:openemail:backend` + +The dependency's own canisters keep their standalone view (`PUBLIC_CANISTER_ID:backend`, …), so vendored code behaves identically whether deployed on its own or through your project. See [Canister Discovery](canister-discovery.md) for the injection mechanism. + +## Addressing dependency canisters + +Because two projects may each define a `backend`, imported canisters are keyed by their path relative to the workspace root, for example `vendor/openemail:backend`. Use that name anywhere a canister name is accepted: + +```bash +icp canister status "vendor/openemail:backend" +icp deploy "vendor/openemail:backend" +``` + +`:` is reserved in canister names as the namespace separator. + +## Running commands inside a dependency (the workspace) + +Vendored dependencies form a **workspace**. When you run an `icp` command from inside a vendored project, icp-cli walks **up** the directory tree to the outermost project that declares the one you are in as a dependency and treats it as the **workspace root**. The network, environments, and the canister-ID store all come from that root, so there is a single source of truth for canister IDs no matter where you run from. + +``` +app/ + icp.yaml # depends on ./vendor/openemail + vendor/ + openemail/ + icp.yaml # a standalone project +``` + +- `cd app && icp deploy` — deploys `app` and `openemail` into app's environment. +- `cd app/vendor/openemail && icp deploy` — resolves up to `app` and deploys **only openemail's** canisters into app's environment and ID store. The IDs are the same ones app's canisters were wired to, so iterating on a vendored dependency in place does not fork a separate deployment. + +When a command resolves to a workspace root above the project you are standing in, icp-cli announces the resolved root so the behavior is visible. + +Resolution is **bounded**: an ancestor is adopted only if it (transitively) declares your project, so an unrelated `icp.yaml` higher up never captures your project. A dependency cloned on its own has no declaring ancestor and behaves as its own root. + +### Deploying part of a workspace + +From inside a member, `icp deploy` with no canister names defaults to **that member's own canisters**. Deploy the whole workspace by running from the root, or target canisters explicitly by their namespaced names from anywhere. + +### Setting the root explicitly + +Force the workspace root with the `--project-root-override` flag or the `ICP_PROJECT_ROOT` environment variable. This uses the given directory as the root with no upward walk — for example, to deploy a vendored project truly on its own. + +## Environments across a workspace + +The workspace root owns the **network** and the **canister-ID store** for every environment; a dependency's own network definitions are ignored when it is deployed as part of a workspace. + +A dependency's own same-named environment still contributes its **per-canister settings and init args**, so a vendored project's canisters get the configuration their author intended. Precedence, highest first: + +1. the workspace root's explicit override for that canister (e.g. `settings: { "openemail:backend": … }`) +2. the dependency's own environment configuration +3. the canister's base settings + +Because the root decides which environments exist, **every member must declare each environment the workspace targets.** Deploying to an environment a dependency does not declare fails with a clear error. If a dependency has no environment-specific configuration, declaring the environment with no overrides is enough: + +```yaml +# in the dependency's icp.yaml +environments: + - name: staging +``` + +`local` and `ic` are implicit for every project, so they never need to be declared. + +## Shared dependencies + +If two projects in a workspace depend on the same directory — for example two services that both vendor `../openemail` — it resolves to **one** instance, built and deployed once and shared by both. Identity is the resolved directory on disk, so two independent copies at different paths stay separate. + +## Keeping a dependency self-contained + +A vendored project must remain a complete `icp` project: it never references its parent, and you can copy or clone it elsewhere and it still works on its own. Vendoring may require [aligning environment names](#environments-across-a-workspace), but never changes to how the dependency finds its own canisters. + +## Limitations + +- icp-cli deploys a parent-owned copy of each dependency; binding directly to an already-deployed on-chain canister is not yet supported. +- Candid and binding generation for dependencies are out of scope — each canister generates the bindings it needs itself. See [Binding Generation](binding-generation.md). + +## Examples + +- [project-dependency](https://github.com/dfinity/icp-cli/tree/main/examples/icp-project-dependency) — a single vendored dependency. +- [project-dependency-shared](https://github.com/dfinity/icp-cli/tree/main/examples/icp-project-dependency-shared) — a shared dependency across sibling services. + +## See Also + +- [Canister Discovery](canister-discovery.md) — How canister IDs are injected +- [Environments and Networks](environments.md) — Deployment targets and how they relate +- [Project Model](project-model.md) — How icp-cli discovers and consolidates configuration +- [Configuration Reference](../reference/configuration.md) — `icp.yaml` fields + +[Browse all documentation →](../index.md) diff --git a/docs/reference/cli.md b/docs/reference/cli.md index ce3dfab6..3d799d81 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -99,7 +99,7 @@ This document contains the help content for the `icp` command-line program. ###### **Options:** -* `--project-root-override ` — Directory to use as your project root directory. If not specified the directory structure is traversed up until an icp.yaml file is found +* `--project-root-override ` — Directory to use as your project root directory. If not specified the directory structure is traversed up to the workspace root (the top-most project that declares the one you are in as a dependency) * `--debug` — Enable debug logging Default value: `false` diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index 34697af4..d1945c8d 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -300,6 +300,34 @@ ], "description": "An amount of cycles.\n\nDeserializes from a number or a string with suffixes (k, m, b, t) and optional underscore separators." }, + "DependencyManifest": { + "description": "Declares a dependency on another `icp` project vendored into this one\n(typically as a git submodule).\n\nRunning `icp deploy` deploys the dependency's canisters into the parent's\nenvironment and injects the *selected* dependency canisters' IDs into the\nparent's canisters as `PUBLIC_CANISTER_ID::` environment\nvariables.", + "properties": { + "canisters": { + "description": "Which of the dependency's canisters to **expose** to the parent as\n`PUBLIC_CANISTER_ID` environment variables. Defaults to all. This is an\nexposure filter only: the whole dependency is always deployed, because a\ndependency's canisters may call each other.", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "name": { + "description": "Local alias for the dependency project. Namespaces the dependency's\ncanister IDs when they are exposed to the parent's canisters as\n`PUBLIC_CANISTER_ID::` environment variables. Must be\nunique among dependencies, must not collide with a local canister name,\nand must not contain `:`.", + "type": "string" + }, + "path": { + "description": "Path to the directory containing the dependency's `icp.yaml`, resolved\nrelative to this manifest's project directory.", + "type": "string" + } + }, + "required": [ + "name", + "path" + ], + "type": "object" + }, "DurationAmount": { "anyOf": [ { @@ -984,6 +1012,14 @@ }, "type": "array" }, + "dependencies": { + "default": [], + "description": "Other `icp` projects this project depends on. Their canisters are\ndeployed into this project's environment and their IDs are injected into\nthis project's canisters as `PUBLIC_CANISTER_ID::`\nenvironment variables.", + "items": { + "$ref": "#/$defs/DependencyManifest" + }, + "type": "array" + }, "environments": { "default": [], "items": { diff --git a/examples/icp-project-dependency-shared/README.md b/examples/icp-project-dependency-shared/README.md new file mode 100644 index 00000000..dcef554e --- /dev/null +++ b/examples/icp-project-dependency-shared/README.md @@ -0,0 +1,65 @@ +# Shared Dependency ("Umbrella") Example + +This example shows a **shared** dependency: two independent sub-projects each +depend on the same third project, and it is deployed **once** and shared. + +## Layout + +``` +icp-project-dependency-shared/ + icp.yaml # the "app" — depends on both services + umbrella/ # bundle of sibling projects (a git submodule in practice) + service-a/ + icp.yaml # depends on ../openemail + service-b/ + icp.yaml # depends on ../openemail + openemail/ + icp.yaml # the shared service +``` + +`service-a` and `service-b` are self-contained `icp` projects — each can be +deployed on its own — and each declares: + +```yaml +dependencies: + - name: openemail + path: ../openemail + canisters: [backend] +``` + +The app depends on both services: + +```yaml +dependencies: + - name: service-a + path: ./umbrella/service-a + canisters: [service] + - name: service-b + path: ./umbrella/service-b + canisters: [service] +``` + +## Shared instance + +Both `service-a` and `service-b` reach `openemail` through `../openemail`, which +resolves to the **same** directory (`umbrella/openemail`). A dependency's identity +is its resolved source directory, so openemail is imported **once**: + +Running `icp deploy` produces these canisters: + +- `frontend` (the app) +- `umbrella/service-a:service` +- `umbrella/service-b:service` +- `umbrella/openemail:backend` — a **single** shared instance + +Both services' canisters read `PUBLIC_CANISTER_ID:openemail:backend`, and both +resolve to that one shared canister. If the two services instead vendored +separate copies of openemail (different directories), they would get isolated +instances. + +Inspect the result with: + +```bash +icp project show +icp canister status "umbrella/openemail:backend" +``` diff --git a/examples/icp-project-dependency-shared/icp.yaml b/examples/icp-project-dependency-shared/icp.yaml new file mode 100644 index 00000000..5d687747 --- /dev/null +++ b/examples/icp-project-dependency-shared/icp.yaml @@ -0,0 +1,28 @@ +# yaml-language-server: $schema=https://github.com/dfinity/icp-cli/raw/refs/tags/v0.1.0/docs/schemas/icp-yaml-schema.json +# +# Shared-dependency ("umbrella") example. +# +# `umbrella/` bundles three sibling projects — `service-a`, `service-b`, and +# `openemail` — as it would appear when vendored as a git submodule. Both +# `service-a` and `service-b` depend on `openemail` via `../openemail`. +# +# This app depends on both services. Because both service edges resolve to the +# SAME `umbrella/openemail` directory, openemail is deployed exactly once and +# shared: `service-a`'s and `service-b`'s canisters both discover the one +# openemail instance through `PUBLIC_CANISTER_ID:openemail:backend`. + +canisters: + - name: frontend + build: + steps: + - type: pre-built + path: ../icp-pre-built/dist/hello_world.wasm + sha256: 17a05e36278cd04c7ae6d3d3226c136267b9df7525a0657521405e22ec96be7a + +dependencies: + - name: service-a + path: ./umbrella/service-a + canisters: [service] + - name: service-b + path: ./umbrella/service-b + canisters: [service] diff --git a/examples/icp-project-dependency-shared/umbrella/openemail/icp.yaml b/examples/icp-project-dependency-shared/umbrella/openemail/icp.yaml new file mode 100644 index 00000000..efaace4e --- /dev/null +++ b/examples/icp-project-dependency-shared/umbrella/openemail/icp.yaml @@ -0,0 +1,12 @@ +# yaml-language-server: $schema=https://github.com/dfinity/icp-cli/raw/refs/tags/v0.1.0/docs/schemas/icp-yaml-schema.json +# +# A self-contained shared service, consumed by both sibling projects +# `../service-a` and `../service-b`. + +canisters: + - name: backend + build: + steps: + - type: pre-built + path: ../../../icp-pre-built/dist/hello_world.wasm + sha256: 17a05e36278cd04c7ae6d3d3226c136267b9df7525a0657521405e22ec96be7a diff --git a/examples/icp-project-dependency-shared/umbrella/service-a/icp.yaml b/examples/icp-project-dependency-shared/umbrella/service-a/icp.yaml new file mode 100644 index 00000000..ed7a67b7 --- /dev/null +++ b/examples/icp-project-dependency-shared/umbrella/service-a/icp.yaml @@ -0,0 +1,17 @@ +# yaml-language-server: $schema=https://github.com/dfinity/icp-cli/raw/refs/tags/v0.1.0/docs/schemas/icp-yaml-schema.json +# +# A self-contained service that depends on the sibling `openemail` project. +# Deployable on its own (`icp deploy` here) and reused by the parent app. + +canisters: + - name: service + build: + steps: + - type: pre-built + path: ../../../icp-pre-built/dist/hello_world.wasm + sha256: 17a05e36278cd04c7ae6d3d3226c136267b9df7525a0657521405e22ec96be7a + +dependencies: + - name: openemail + path: ../openemail + canisters: [backend] diff --git a/examples/icp-project-dependency-shared/umbrella/service-b/icp.yaml b/examples/icp-project-dependency-shared/umbrella/service-b/icp.yaml new file mode 100644 index 00000000..5a5b8a63 --- /dev/null +++ b/examples/icp-project-dependency-shared/umbrella/service-b/icp.yaml @@ -0,0 +1,18 @@ +# yaml-language-server: $schema=https://github.com/dfinity/icp-cli/raw/refs/tags/v0.1.0/docs/schemas/icp-yaml-schema.json +# +# A second self-contained service that also depends on the sibling `openemail` +# project. When both services are deployed together (via the parent app), they +# share a single openemail instance. + +canisters: + - name: service + build: + steps: + - type: pre-built + path: ../../../icp-pre-built/dist/hello_world.wasm + sha256: 17a05e36278cd04c7ae6d3d3226c136267b9df7525a0657521405e22ec96be7a + +dependencies: + - name: openemail + path: ../openemail + canisters: [backend] diff --git a/examples/icp-project-dependency/README.md b/examples/icp-project-dependency/README.md new file mode 100644 index 00000000..bf21f381 --- /dev/null +++ b/examples/icp-project-dependency/README.md @@ -0,0 +1,56 @@ +# Project Dependency Example + +This example shows how an `icp` project can depend on **another** `icp` project +that is vendored into it (typically as a git submodule), so both are developed in +one source tree and deployed together. + +## Layout + +``` +icp-project-dependency/ + icp.yaml # the "app" — has a `backend` canister and a dependency + vendor/ + openemail/ # a self-contained icp project (would be a git submodule) + icp.yaml # has `backend` and `frontend` canisters +``` + +The app declares: + +```yaml +dependencies: + - name: openemail + path: ./vendor/openemail + canisters: [backend] +``` + +## What `icp deploy` does + +Running `icp deploy` in this directory: + +1. Deploys the app's canisters **and all** of `openemail`'s canisters + (`backend` and `frontend`) into the same environment. The whole dependency is + always deployed because a dependency's canisters may call each other. +2. Injects the **selected** dependency canister IDs into the app's canisters as + environment variables. Here the app's `backend` receives + `PUBLIC_CANISTER_ID:openemail:backend`. + +Canister-ID environment variables are set per project scope: + +- The app's canisters see their own canisters by name + (`PUBLIC_CANISTER_ID:backend`) plus the exposed dependency canisters under the + alias (`PUBLIC_CANISTER_ID:openemail:backend`). +- `openemail`'s canisters see exactly what they would see when deployed + standalone: `PUBLIC_CANISTER_ID:backend` and `PUBLIC_CANISTER_ID:frontend`, + each resolving to openemail's own canisters — so vendored code behaves + identically whether deployed on its own or as a dependency. + +## Store keys + +Because the app and openemail both define a `backend` canister, imported +dependency canisters are keyed by their path relative to the project root, e.g. +`vendor/openemail:backend`. Use these names to address dependency canisters +directly: + +```bash +icp canister status "vendor/openemail:backend" +``` diff --git a/examples/icp-project-dependency/icp.yaml b/examples/icp-project-dependency/icp.yaml new file mode 100644 index 00000000..94457f82 --- /dev/null +++ b/examples/icp-project-dependency/icp.yaml @@ -0,0 +1,28 @@ +# yaml-language-server: $schema=https://github.com/dfinity/icp-cli/raw/refs/tags/v0.1.0/docs/schemas/icp-yaml-schema.json +# +# This example shows how one icp project can depend on another icp project that +# is vendored into it (typically as a git submodule). See ./vendor/openemail for +# the self-contained dependency project. +# +# `icp deploy` here deploys this project's canisters AND all of openemail's +# canisters into the same environment, and injects the selected openemail +# canister IDs into this project's canisters as environment variables named +# `PUBLIC_CANISTER_ID:openemail:`. + +canisters: + - name: backend + build: + steps: + - type: pre-built + path: ../icp-pre-built/dist/hello_world.wasm + sha256: 17a05e36278cd04c7ae6d3d3226c136267b9df7525a0657521405e22ec96be7a + +dependencies: + # Declare a dependency on the vendored `openemail` project. + # name: local alias — namespaces its canister IDs (PUBLIC_CANISTER_ID:openemail:) + # path: directory containing openemail's icp.yaml + # canisters: which of openemail's canisters to expose to this project's canisters + # (omit to expose all). The whole dependency is always deployed regardless. + - name: openemail + path: ./vendor/openemail + canisters: [backend] diff --git a/examples/icp-project-dependency/vendor/openemail/icp.yaml b/examples/icp-project-dependency/vendor/openemail/icp.yaml new file mode 100644 index 00000000..bd564827 --- /dev/null +++ b/examples/icp-project-dependency/vendor/openemail/icp.yaml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://github.com/dfinity/icp-cli/raw/refs/tags/v0.1.0/docs/schemas/icp-yaml-schema.json +# +# `openemail` is a self-contained icp project. It can be developed and deployed +# on its own (`icp deploy` from this directory), and it is also consumed as a +# dependency by the parent project one level up (../../icp.yaml). +# +# In the real workflow this directory would be a git submodule so its source can +# be iterated in place while the parent app is developed. + +canisters: + - name: backend + build: + steps: + - type: pre-built + path: ../../../icp-pre-built/dist/hello_world.wasm + sha256: 17a05e36278cd04c7ae6d3d3226c136267b9df7525a0657521405e22ec96be7a + + - name: frontend + build: + steps: + - type: pre-built + path: ../../../icp-pre-built/dist/hello_world.wasm + sha256: 17a05e36278cd04c7ae6d3d3226c136267b9df7525a0657521405e22ec96be7a