merge dev into main

.github/workflows/ci.yml

@@ -0,0 +1,41 @@
+name: Python 🐍 CI tests
+
+on:
+  pull_request:
+    types: [closed]
+    branches: [main]
+    paths-ignore:
+      - "**.md"
+      - "**.bib"
+      - "**.ya?ml"
+      - "LICENSE"
+      - ".gitignore"
+  workflow_dispatch:
+
+jobs:
+  test:
+    if: github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true
+    env:
+      SHIFT_SOLVER: highs
+      SHIFT_SOLVER_OPTIONS: highs-default
+      SHIFT_COMPUTE_IIS: "false"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Pixi
+        run: python -m pip install --upgrade pip pixi
+
+      - name: Install dependencies via Pixi (test env)
+        run: pixi install --environment test
+
+      - name: Validate environment
+        run: pixi info --environment test
+
+      - name: Run tests
+        run: pixi run -e test unit-tests

.pre-commit-config.yaml

@@ -0,0 +1,43 @@
+exclude: "^LICENSES|^config/config\\.yaml$"
+
+ci:
+  autoupdate_schedule: quarterly
+
+repos:
+  # Run ruff to lint and format
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.15.0
+    hooks:
+      # Run the linter
+      - id: ruff
+        args: [--fix]
+      # Run the formatter
+      - id: ruff-format
+
+  # Find common spelling mistakes
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.3.0
+    hooks:
+      - id: codespell
+        args: ['--ignore-regex="(\b[A-Z]+\b)"']
+        types_or: [python, rst, markdown]
+        files: ^(scripts|rules)/
+
+  # Format Snakemake rule files
+  - repo: https://github.com/snakemake/snakefmt
+    rev: v1.0.0
+    hooks:
+      - id: snakefmt
+
+  # Run some default pre-commit hooks
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: check-added-large-files
+        args: ["--maxkb=2000"]
+      - id: check-merge-conflict
+      - id: check-yaml
+        exclude: "pixi.lock"
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+

README.md

@@ -1,79 +1,111 @@
 # SHIFT – Steel & Hydrogen Integrated Freight Trade
 
-This repository contains the **SHIFT model**, a spatially resolved techno-economic optimization of global iron and steel supply chains under decarbonization. It explores how hydrogen-based direct reduced iron (DRI) production and hot-briquetted iron (HBI) trade can shift value creation to regions with renewable energy, infrastructure, and capital availability.
+This repository contains the **SHIFT model**, a spatially resolved techno-economic optimization of global iron and steel supply chains under decarbonization.
+It explores how hydrogen-based direct reduced iron (DRI) production and hot-briquetted iron (HBI) trade can shift value creation to regions with renewable energy and capital availability.
 
-The model identifies cost-optimal configurations for mining, hydrogen production, DRI processing, and steel trade, using the PyPSA framework.
+The model identifies cost-optimal configurations for mining, hydrogen production, DRI processing, and HBI trade, using a two-stage optimization pipeline and open energy system libraries.
 
----
+## Introduction SHIFT
 
+SHIFT evaluates global supply and trade of low-carbon iron and steel at high spatial resolution.
+The model quantifies where to produce, where to ship, and how to meet demand cost-effectively under decarbonization constraints.
 
-## Prerequesite: TRACE model
+Key features:
+- 🚀 two-stage process: greenfield supply curves + cross-region LP trade
+- 🌍 spatial renewable potentials: PyPSA-Earth wind/solar CF distributions
+- 💰 integrated LCOX: region-level cost curves for H2, DRI, HBI
+- 🛳️ global trade dispatch: route costs, flows, nodal prices, utilization
+- ⚙️ configurable automation: regions, technologies, scenarios via YAML + Snakemake rules
 
-The SHIFT model integrates the energy supply chain `shipping-steel` from a [fork](https://github.com/fneum/trace/tree/pypsa-eur-sec-imports-atlite) of the [TRACE model](https://github.com/euronion/trace).
+## Quick installation (PIXI)
 
-Therefore, clone the TRACE fork with `git`:
+Clone the repository:
 
 ```sh
-git clone https://github.com/fneum/trace.git
+git clone https://github.com/energyLS/shift.git
+cd shift
 ```
 
-and switch to the branch `pypsa-eur-sec-imports-atlite` (commit [8bf0571](https://github.com/fneum/trace/commit/8bf057142d4e035926ffb084493462eff64fe188)) and follow these steps:
-- delete `escs/shipping-steel/loads.csv`,
-- delete `escs/shipping-steel/ships.csv`,
-- in `data/efficiencies.csv` and `escs/shipping-steel/links.csv`, and `escs/shipping-hbi/links.csv`, add replace `direct iron reduction furnace` with `hydrogen direct iron reduction furnace` to match the latest technology-data version
-- set `technology_data: "v0.12.0"` in the `config/config.default.yaml` (same as in SHIFT: `config/config.yaml`),
-- run `snakemake -c1 resources/networks/default/2030/shipping-steel/DE-DE/network.nc`,
-- run `snakemake -c1 resources/networks/default/2050/shipping-steel/DE-DE/network.nc`.
-
-This creates a steel supply chain for 2030 and 2050 without loads and shipping, those parameters will be added later in the SHIFT workflow. The resulting steel model will be stored in `resources/networks/default/2050/shipping-steel/DE-DE/network.nc` and automatically fetched by the SHIFT model.
-
+Install dependencies with pixi:
 
-## Download, Install, and Run the SHIFT model
+```sh
+pixi install
+```
 
-Clone the repository with `git`:
+Activate the environment:
 
 ```sh
-git clone https://github.com/energyLS/shift.git
+pixi shell
 ```
 
-Create the environment with `conda`:
+(Optional) Install pre-commit hooks:
 
 ```sh
-conda env create -f environment.yaml
+pre-commit install
 ```
 
-Navigate to the `workflow/notebooks` and run the notebooks `global-iron-ore.ipynb`, `global-steel-production.iypnb`, `prepare-iron-ore.ipynb`, and `prepare-steel.ipynb` for preparation. Those steps will be included in the main workflow in a future version.
 
-Run the trade model by navigating to the `workflow/` folder via `cd workflow` and then run
+For details, see https://pixi.prefix.dev/latest/ (or your local PIXI docs).
+
+## Run (core workflow)
+
+In the workspace root:
 
 ```sh
-snakemake -call model_trade_all
+pixi run snakemake model_trade_all
 ```
 
-To plot the supply curves subtracted with demand, run
+To collect all figures (under development):
 
 ```sh
-snakemake -c1 create_all_supply_curves_with_demand
+pixi run snakemake collect_figures
 ```
 
 
-*Under development:*
+## Workflow overview
 
-Run the whole workflow using `snakemake`:
+### Configuration & Scenario Setup (implicit)
 
-```sh
-snakemake -call collect_figures
-```
+Before execution, Snakemake reads:
+- **Global settings**: `config/config.yaml` (regions, cost years, solver options, enable flags)
+- **Scenario matrix**: `config/trade_scenarios.csv` (rows = distinct trade scenarios)
 
+These expand into a deterministic wildcard space (cost_year, region, product, scenario) that drives all downstream rule creation. This bootstrap is handled automatically by Snakemake; no user action required.
 
-## Licence
+### Step 0: Renewable potentials (pre-computed inputs)
 
-This repository is licensed under the MIT License. See `LICENCE` for details.
+In this stage we generate the supply-side resource backbone.
+Renewable capacity-factor series and maximum deployable potentials are pre-computed externally using PyPSA-Earth's `build_renewable_profiles` rule and stored in `data/renewable_profiles/`.
+This stage is not part of the current Snakefile. SHIFT consumes pre-computed .nc datasets to avoid the long runtime of full GIS processing.
+
+> **For new users:** No action needed. Renewable data files are provided in the repository.
+
+### Step 1: Greenfield supply curve generation (PyPSA)
+
+With renewable profiles and [techno-economic assumptions](https://github.com/PyPSA/technology-data) in place, SHIFT builds regional PyPSA optimization models to size generation, storage, and process assets.
+It evaluates each candidate plant (H2 electrolyser, DRI furnace, HBI plant, steel mills) across resource quality and cost parameters to produce levelized cost curves (LCOX) as a function of capacity.
+The result is a fleet of supply curve elements (capacity buckets with marginal costs and metadata) for H2, DRI, HBI, and steel by region.
+
+Step 1 is a multi-part stage: load techno-economic data and build regional cost baselines, prepare renewable candidate sets per region, solve optimization problems at discrete demand levels, and consolidate results into piecewise supply curves. Each stage depends on the prior; files are persisted between steps to support reproducibility and debugging.
+The output is a harmonized set of supply curves used as input for the trade stage.
+
+### Step 2: Global trade optimization (LP)
+
+This stage takes regional supply curves and demand obligations, then runs a linear program over the regional network.
+It includes transport cost matrices, ore production constraints, and market compatibility.
+The solver decides how much each region should produce versus import/export, by product and route.
+
+The trade solution yields detailed outputs: regional production volume and shipped quantities.
+It can also be reconciled with scenarios for demand, policy constraints, and infrastructure availability.
 
 
 ## Acknowledgements
 
 Thanks to:
+- Oda Agdal and her Master's Thesis on the [Investigation of Future Global Trade of Hydrogen from Renewable Energy Sources](https://ntnuopen.ntnu.no/ntnu-xmlui/handle/11250/3031513)
+- Johannes Hampp and [TRACE](https://github.com/euronion/trace)
 
-* Oda Agdal and her Master's Thesis on the [Investigation of Future Global Trade of Hydrogen from Renewable Energy Sources](https://ntnuopen.ntnu.no/ntnu-xmlui/handle/11250/3031513).
 
+## Licence
+
+This repository is licensed under the MIT License. See `LICENCE` for details.

Snakefile

@@ -0,0 +1,94 @@
+"""Root Snakemake entrypoint for the Shift workflow.
+
+Loads shared configuration and includes the modular rule files for supply curves,
+trade optimization, and reporting.
+"""
+
+from pathlib import Path
+from shutil import copyfile
+import sys
+
+import pandas as pd
+from snakemake.utils import Paramspace
+
+
+WORKFLOW_DIR = Path(workflow.basedir) / "workflow"
+SCRIPT_DIR = WORKFLOW_DIR / "scripts"
+NOTEBOOKS_DIR = WORKFLOW_DIR / "notebooks"
+# DATA_
+
+if str(SCRIPT_DIR) not in sys.path:
+    sys.path.insert(0, str(SCRIPT_DIR))
+if str(NOTEBOOKS_DIR) not in sys.path:
+    sys.path.insert(0, str(NOTEBOOKS_DIR))
+
+from trade_chain_utils import (  # noqa: E402
+    derive_supply_curve_products,
+    get_ordered_stages,
+    get_stage_groups,
+    get_trade_chain,
+)
+
+
+configfile: "config/config.yaml"
+
+
+def _load_trade_scenarios():
+    trade_chains = config.get("trade_chains")
+    if trade_chains:
+        rows = []
+        chain = get_trade_chain(config)
+        stages_sorted = get_ordered_stages(chain)
+        if len(stages_sorted) < 2:
+            raise ValueError(
+                f"Trade chain '{chain.get('id', '<unnamed>')}' needs at least 2 stages"
+            )
+        stage_groups = get_stage_groups(chain)
+        if not stage_groups:
+            raise ValueError(
+                f"Trade chain '{chain.get('id', '<unnamed>')}' produced no stage groups"
+            )
+        scenarios = chain.get("trade_scenarios", "default")
+        if isinstance(scenarios, str):
+            scenarios = [scenarios]
+        for scenario in scenarios:
+            rows.append(
+                {
+                    "chain_id": str(chain.get("id", "default")),
+                    "cost_year": str(chain.get("cost_year", 2050)),
+                    "interone": str(stage_groups[0]["label"]),
+                    "intertwo": str(
+                        stages_sorted[-1].get(
+                            "process_label", stages_sorted[-1]["output_commodity"]
+                        )
+                    ),
+                    "wacc": str(chain.get("wacc", "regional")),
+                    "final": str(chain.get("final_product", "steel")),
+                    "scenario": str(scenario),
+                }
+            )
+        return Paramspace(pd.DataFrame(rows, dtype=str))
+
+    return Paramspace(pd.read_csv("config/trade_scenarios.csv", dtype=str))
+
+
+trade_scenarios = _load_trade_scenarios()
+
+
+def _derive_supply_curve_products():
+    return derive_supply_curve_products(config)
+
+
+SUPPLY_CURVE_PRODUCTS = _derive_supply_curve_products()
+
+
+wildcard_constraints:
+    country="[a-zA-Z]+",
+    sweep="[a-zA-Z]+",
+    rule="(0|[1-9][0-9]?|100)",
+
+
+include: "rules/supply_curves.smk"
+include: "rules/preparation.smk"
+include: "rules/trade_model.smk"
+include: "rules/reporting.smk"