diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bc708ce0..d4833a78 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,7 +16,9 @@ variables: python3: 'python3' # Config to build and test CI_CONFIG: 'cachepool_fpu_4g' - SW_PREFIX: 'test-cachepool-' + CI_CONFIG_DUAL: 'cachepool_dual_fpu_4g' + CI_CONFIG_16G: 'cachepool_fpu_16g' + SW_PREFIX: 'test-' default: tags: [shared] @@ -60,6 +62,63 @@ build: - hardware/deps/dram_rtl_sim/dramsys_lib/DRAMSys/build/lib/ - hardware/deps/dram_rtl_sim/dramsys_lib/DRAMSys/configs/ +# --------------------------------------------------------------------------- +# Build stage: same as `build`, for the dual-scalar-per-CC config. Only the +# kernels adapted for dual-scalar (see `test-dual`) are exercised against it. +# --------------------------------------------------------------------------- +build-dual: + stage: build + script: + - echo "Using CC=$CC" + - echo "Using CXX=$CXX" + - test -x "$CC" + - test -x "$CXX" + - source iis-env.sh + - make init + - make dram-build + - make clean generate update-floonoc bootrom vsim sw config=$CI_CONFIG_DUAL DEBUG=0 noc_profiling=0 + artifacts: + when: always + expire_in: 2h + paths: + - sim/work/ + - sim/bin/cachepool_cluster.vsim + - sim/bin/cachepool_cluster.vsim.gui + - sim/work-dpi/ + - software/build/CachePoolTests/ + - hardware/deps/dram_rtl_sim/dramsys_lib/DRAMSys/build/lib/ + - hardware/deps/dram_rtl_sim/dramsys_lib/DRAMSys/configs/ + +# --------------------------------------------------------------------------- +# Build stage: same as `build`, for the largest available config (16 groups). +# Manual (see test-16g): slow, so not run automatically on every push. +# --------------------------------------------------------------------------- +build-16g: + stage: build + when: manual + allow_failure: true + timeout: 4h + script: + - echo "Using CC=$CC" + - echo "Using CXX=$CXX" + - test -x "$CC" + - test -x "$CXX" + - source iis-env.sh + - make init + - make dram-build + - make clean all config=$CI_CONFIG_16G DEBUG=0 noc_profiling=0 + artifacts: + when: always + expire_in: 2h + paths: + - sim/work/ + - sim/bin/cachepool_cluster.vsim + - sim/bin/cachepool_cluster.vsim.gui + - sim/work-dpi/ + - software/build/CachePoolTests/ + - hardware/deps/dram_rtl_sim/dramsys_lib/DRAMSys/build/lib/ + - hardware/deps/dram_rtl_sim/dramsys_lib/DRAMSys/configs/ + # --------------------------------------------------------------------------- # Test stage: run each kernel in parallel on a separate runner. # Each job downloads the build artifacts, runs one simulation, and checks @@ -90,9 +149,69 @@ test: - python3 util/auto-benchmark/check-ci.py test_${KERNEL}.log artifacts: when: always - expire_in: 1 week + expire_in: 3 days paths: # Full simulation log - test_*.log # Performance-monitor trace files written by the simulator - sim/bin/logs/ + +# --------------------------------------------------------------------------- +# Test stage: dual-scalar-per-CC config. Limited to the kernels adapted and +# confirmed working under this config (host0-only Spatz access via +# snrt_cluster_vpu_*/snrt_cluster_is_primary); other tests are not yet +# verified for dual-scalar and are left to the default-config matrix above. +# --------------------------------------------------------------------------- +test-dual: + stage: test + needs: [build-dual] + parallel: + matrix: + - KERNEL: + - fdotp-32b_M32768 + - gemv_M512_N128_K32 + - fmatmul-32b_M64_N64_K64 + - fft-32b_M1024_N16 + script: + - mkdir -p sim/bin/logs sim/bin/logs/core sim/bin/logs/noc sim/bin/logs/others + - chmod +x sim/bin/cachepool_cluster.vsim + - BIN="${SW_PREFIX}${KERNEL}" + - sim/bin/cachepool_cluster.vsim software/build/CachePoolTests/$BIN 2>&1 | tee test_dual_${KERNEL}.log + - python3 util/auto-benchmark/check-ci.py test_dual_${KERNEL}.log + artifacts: + when: always + expire_in: 3 days + paths: + - test_dual_*.log + - sim/bin/logs/ + +# --------------------------------------------------------------------------- +# Test stage: largest available config (16 groups), largest available size +# per kernel. Runs once build-16g (manual) is triggered and succeeds -- see +# build-16g. +# --------------------------------------------------------------------------- +test-16g: + stage: test + needs: [build-16g] + timeout: 4h + parallel: + matrix: + - KERNEL: + - fdotp-32b_M65536 + - gemv_M1024_N128_K32 + - fmatmul-32b_M128_N128_K128 + - fft-32b_M1024_N16 + - byte-enable + - multi_producer_single_consumer_double_linked_list_M1_N1350_K10 + script: + - mkdir -p sim/bin/logs sim/bin/logs/core sim/bin/logs/noc sim/bin/logs/others + - chmod +x sim/bin/cachepool_cluster.vsim + - BIN="${SW_PREFIX}${KERNEL}" + - sim/bin/cachepool_cluster.vsim software/build/CachePoolTests/$BIN 2>&1 | tee test_16g_${KERNEL}.log + - python3 util/auto-benchmark/check-ci.py test_16g_${KERNEL}.log + artifacts: + when: always + expire_in: 3 days + paths: + - test_16g_*.log + - sim/bin/logs/ diff --git a/Bender.lock b/Bender.lock index 85d89802..b79fbf04 100644 --- a/Bender.lock +++ b/Bender.lock @@ -150,7 +150,7 @@ packages: dependencies: - tech_cells_generic spatz: - revision: f6ea19fb26c8bf992ae51a3e4acd3f2a36c6659f + revision: f4741b0a5bcc49103bd8a3db8d51fdf76d240df7 version: null source: Git: https://github.com/pulp-platform/spatz.git diff --git a/Bender.yml b/Bender.yml index e550590c..35f7ea4b 100644 --- a/Bender.yml +++ b/Bender.yml @@ -42,6 +42,9 @@ sources: - hardware/generated/floo_cachepool_noc_pkg.sv - hardware/src/cachepool_pkg.sv - hardware/src/cachepool_cc.sv + - hardware/src/cachepool_spatz_lock.sv + - hardware/src/acc_mux.sv + - hardware/src/cachepool_cc_dual.sv # Barrier - hardware/src/cachepool_tile_barrier.sv - hardware/src/cachepool_cluster_barrier.sv diff --git a/Makefile b/Makefile index 9edab93f..25bc3aec 100644 --- a/Makefile +++ b/Makefile @@ -190,12 +190,12 @@ $(info FLOO_DIR: $(FLOO_DIR)) # Generates the sources for FlooNoC .PHONY: update-floonoc install-floogen clean-floonoc install-floogen: - pip install -e $(FLOO_DIR) --quiet + $(PYTHON) -m pip install -e $(FLOO_DIR) --quiet update-floonoc: $(FLOO_NOC) $(FLOO_NOC): $(FLOO_CFG) mkdir -p $(FLOO_GEN_OUTDIR) - PATH="$(HOME)/.local/bin:$(PATH)" floogen pkg -c $(FLOO_CFG) -o $(FLOO_GEN_OUTDIR) --no-format + floogen pkg -c $(FLOO_CFG) -o $(FLOO_GEN_OUTDIR) --no-format clean-floonoc: rm -f $(FLOO_NOC) @@ -213,7 +213,7 @@ dram-build: $(DRAMSYS_PATH)/README.md dram-config if [ ! -d "build" ]; then \ mkdir build && cd build; \ $(CMAKE) -DCMAKE_CXX_FLAGS=-fPIC -DCMAKE_C_FLAGS=-fPIC -D DRAMSYS_WITH_DRAMPOWER=ON .. ; \ - make -j; \ + make -j4; \ fi $(DRAMSYS_PATH)/README.md: dram-init @@ -242,7 +242,7 @@ dram-clean: fi dram-init: - make -C ${DRAMSYS_DIR} -j8 dramsys CXX=$(CXX) CC=$(CC) + make -C ${DRAMSYS_DIR} -j4 dramsys CXX=$(CXX) CC=$(CC) ############ # Modelsim # @@ -298,6 +298,10 @@ VLOG_DEFS += -DSPATZ_NUM_FPU=$(spatz_num_fpu) VLOG_DEFS += -DSPATZ_NUM_IPU=$(spatz_num_ipu) VLOG_DEFS += -DSPATZ_MAX_TRANS=$(spatz_max_trans) VLOG_DEFS += -DSNITCH_MAX_TRANS=$(snitch_max_trans) +VLOG_DEFS += -DNUM_SCALAR_PER_CC=$(num_scalar_per_core) +ifeq ($(num_scalar_per_core),2) +VLOG_DEFS += -DCACHEPOOL_DUAL_CC +endif VLOG_DEFS += -DLG_PORT_PER_CORE=$(num_lg_ports_per_core) VLOG_DEFS += -DRG_PORT_PER_CORE=$(num_rg_ports_per_core) VLOG_DEFS += -DNOC_PORT_PER_TILE=$(num_noc_ports_per_tile) @@ -351,24 +355,28 @@ include sim/sim.mk TESTS_DIR := $(SOFTWARE_DIR)/tests +# Test-group roots (relative to SOFTWARE_DIR) scanned for data-generated tests. +TEST_ROOTS := cache sync kernels/fp kernels/int rlc tests + # Auto-discover every test that has a script/gen_data.py. # Convention: script/data_.json → data/data_.h # Adding a new test or a new data variant needs no Makefile changes: # - new test: drop gen_data.py + data_.json into its script/ dir # - new variant: add data_.json to an existing test's script/ dir -DATA_TESTS := $(patsubst $(TESTS_DIR)/%/script/gen_data.py,%, \ - $(wildcard $(TESTS_DIR)/*/script/gen_data.py)) +DATA_TESTS := $(patsubst $(SOFTWARE_DIR)/%/script/gen_data.py,%, \ + $(foreach root,$(TEST_ROOTS), \ + $(wildcard $(SOFTWARE_DIR)/$(root)/*/script/gen_data.py))) define gen_data_rules -$(1)_DATA := $$(patsubst $(TESTS_DIR)/$(1)/script/data_%.json, \ - $(TESTS_DIR)/$(1)/data/data_%.h, \ - $$(wildcard $(TESTS_DIR)/$(1)/script/data_*.json)) +$(1)_DATA := $$(patsubst $(SOFTWARE_DIR)/$(1)/script/data_%.json, \ + $(SOFTWARE_DIR)/$(1)/data/data_%.h, \ + $$(wildcard $(SOFTWARE_DIR)/$(1)/script/data_*.json)) ALL_GEN_DATA += $$($(1)_DATA) -$(TESTS_DIR)/$(1)/data/data_%.h: \ - $(TESTS_DIR)/$(1)/script/data_%.json \ - $(TESTS_DIR)/$(1)/script/gen_data.py - $$(PYTHON) $(TESTS_DIR)/$(1)/script/gen_data.py -c $$< +$(SOFTWARE_DIR)/$(1)/data/data_%.h: \ + $(SOFTWARE_DIR)/$(1)/script/data_%.json \ + $(SOFTWARE_DIR)/$(1)/script/gen_data.py + $$(PYTHON) $(SOFTWARE_DIR)/$(1)/script/gen_data.py -c $$< endef $(foreach test,$(DATA_TESTS),$(eval $(call gen_data_rules,$(test)))) @@ -384,23 +392,32 @@ $(BANDWIDTH_DATA): $(TESTS_DIR)/bandwidth/script/data.json \ .PHONY: gen-data gen-data: $(ALL_GEN_DATA) +# Clean targets, used to delete generated files .PHONY: clean.data clean.data: rm -f $(ALL_GEN_DATA) .PHONY: clean.sw -clean.sw: +clean.sw: clean.data rm -rf ${SOFTWARE_DIR}/build -.PHONY: clean -clean: clean.sw clean.vsim clean.data +.PHONY: clean.generate +clean.generate: rm -rf $(HJSON_OUT) $(BOOTROM_DIR)/bootdata.cc \ $(BOOTROM_DIR)/bootdata_bootrom.cc \ $(BOOTROM_DIR)/bootrom.sv \ $(BOOTROM_DIR)/bootrom.dump \ $(BOOTROM_DIR)/bootrom.elf \ + $(CACHEPOOL_DIR)/wlf* \ + $(CACHEPOOL_DIR)/noc_profiling/* \ $(SNRT_BOOTINFO_H) +.PHONY: clean.hw +clean.hw: clean.vsim clean.generate clean-floonoc + +.PHONY: clean +clean: clean.hw clean.sw + # Common CMake flags shared by sw and vsim targets. # vsim appends -DSNITCH_SIMULATOR to point tests at the compiled binary. SW_CMAKE_FLAGS = \ @@ -411,7 +428,8 @@ SW_CMAKE_FLAGS = \ -DLLVM_PATH=${LLVM_INSTALL_DIR} \ -DGCC_PATH=${GCC_INSTALL_DIR} \ -DPYTHON=${PYTHON} \ - -DBUILD_TESTS=ON + -DBUILD_TESTS=ON \ + -DNUM_SCALAR_PER_CORE=$(num_scalar_per_core) .PHONY: sw sw: generate bootrom gen-data @@ -428,8 +446,11 @@ vsim: generate bootrom dpi ${SIMBIN_DIR}/cachepool_cluster.vsim ############ # Just a shortcut to build everything +.PHONY: hw +hw: generate bootrom vsim + .PHONY: all -all: generate bootrom vsim sw +all: hw sw ######## # Lint # @@ -463,9 +484,7 @@ avg-log: # NoC traffic visualization frontend (fork of https://github.com/ueqri/vis4mesh # with CachePool-specific fixes/features). Consumes the Vis4Mesh dataset # directories produced by util/scripts/noc_profiling_to_vis4mesh.py from -# hardware/tb/cachepool_noc_profiling.sv's per-cycle NoC logs. Fetched as a -# plain pinned clone (same pattern as toolchain.mk's riscv-gnu-toolchain/ -# llvm-project targets), not a git submodule. +# hardware/tb/cachepool_noc_profiling.sv's per-cycle NoC logs. NPM ?= npm VIS4MESH_DIR ?= ${CACHEPOOL_DIR}/util/vis4mesh VIS4MESH_REPO ?= https://github.com/DiyouS/vis4mesh.git @@ -531,13 +550,18 @@ help: @echo "SW Build:" @echo "" @echo "*sw*: build software (generate + bootrom + cmake); overwrites previous build" - @echo "*clean.sw*: remove the software build directory" + @echo "*clean.sw*: remove the software build directory (also runs clean.data)" @echo "" @echo "Simulation:" @echo "" @echo "*vsim*: build hardware for QuestaSim simulation (use 'sw' to build software separately)" + @echo "*hw*: shortcut for generate + bootrom + vsim" + @echo "*all*: shortcut for hw + sw" @echo "*clean.vsim*: remove the hardware simulation build [from sim/sim.mk]" - @echo "*clean*: remove SW build, vsim build, and all generated HW files" + @echo "*clean.data*: remove generated data files (gen-data outputs)" + @echo "*clean.generate*: remove generated HJSON/bootrom/wlf/noc_profiling files [from 'generate'/'bootrom']" + @echo "*clean.hw*: clean.vsim + clean.generate" + @echo "*clean*: clean.hw + clean.sw (remove SW build, vsim build, and all generated HW files)" @echo "" @echo "Lint:" @echo "" diff --git a/README.md b/README.md index ed692ec8..dd4b82eb 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,8 @@ make vsim config=cachepool_fpu_4g Set `DEBUG=0` to disable `+acc` waveform visibility and speed up simulation (used by CI); default is `DEBUG=1`. +`make hw` is a shortcut for `generate bootrom vsim`; `make all` is a shortcut for `hw sw` (build everything). + #### Run the Simulation The wrapper script launches the simulation (GUI or CLI) and expects a software ELF path as argument: @@ -142,7 +144,7 @@ A lightweight benchmarking automation flow is provided under `util/auto-benchmar CONFIGS="cachepool_fpu_4g cachepool_fpu_16g" KERNELS="fdotp-32b_M32768 ffft-64b_M16384 fmatmul-64b_M2048" - PREFIX="test-cachepool-" + PREFIX="test-" ROOT_PATH=../.. 2. Run all builds and simulations: @@ -204,9 +206,13 @@ Configuration names encode the number of groups and whether the FPU is enabled: | `cachepool_fpu_4g` | 4 | 2×2 | Yes | 4 | 4 | 64 | | `cachepool_fpu_16g` | 16 | 4×4 | Yes | 4 | 4 | 256 | | `cachepool_fpu_16g_tiny` | 16 | 4×4 | Yes | 2 | 2 | 64 | +| `cachepool_dual_4g` | 4 | 2×2 | No (IPU only) | 4 | 4 | 64 CCs / 128 harts | +| `cachepool_dual_fpu_4g` | 4 | 2×2 | Yes | 4 | 4 | 64 CCs / 128 harts | `cachepool_fpu_16g_tiny` shrinks tiles/group and cores/tile for a faster-to-build, faster-to-simulate smoke test of the full 16-group mesh topology. +`cachepool_dual_4g`/`cachepool_dual_fpu_4g` set `num_scalar_per_core=2`: each Core Complex holds 2 Snitch scalar harts sharing 1 Spatz unit via a hardware ownership lock (`cachepool_spatz_lock.sv`), so "Cores" (Core Complex slots) and hart count diverge — 64 CCs, 128 harts total. `cachepool_dual_fpu_4g` is otherwise identical to `cachepool_fpu_4g` (same FPU/IPU counts), just with the shared-Spatz CC flavor — used to confirm the dual-scalar lock/mux design also works with the FPU enabled, not just IPU-only. The lock never blocks a core's pipeline: every acquire/release attempt (`software/snRuntime/include/spatz_lock.h`) completes immediately with an outcome (granted / denied / granted-but-still-draining), so a hart contending for a lock it doesn't get can always retry or do something else instead of hanging. See `note.md` for the full state-machine design. + The Spatz cluster consumes **`config/cachepool.hjson`**, which is **generated** from: - `config/cachepool.hjson.tmpl` (skeleton with comments) - `config/config.mk` (source of truth) @@ -341,7 +347,7 @@ For a spatial, time-scrubbable view of NoC traffic (as opposed to `cachepool_mon 1. **Enable the profiler and run a kernel** (default `noc_profiling ?= 1`, so this is on unless you passed `noc_profiling=0`): ```sh make vsim config= noc_profiling=1 - ./sim/bin/cachepool_cluster.vsim software/build/CachePoolTests/test-cachepool- + ./sim/bin/cachepool_cluster.vsim software/build/CachePoolTests/test- ``` L1 (inter-group cache-access mesh) logs (`router_g*.log`, `tile_g*.log`) only populate with `num_rg_ports_per_core > 0` (a multi-group config); L2 (DRAM-refill mesh) logs (`l2_router_g*.log`) and per-core PE logs (`pe_g*.log`) are always populated. @@ -370,6 +376,16 @@ For a spatial, time-scrubbable view of NoC traffic (as opposed to `cachepool_mon The default system uses a 32-bit Snitch core with a Spatz RVV accelerator. Double-precision is disabled by default for scalability; enable the FPU flavor (`cachepool_fpu.mk`) for single/half precision support. +### Dual-scalar flavor (`cachepool_cc_dual.sv`) + +Set `num_scalar_per_core=2` (e.g. `cachepool_dual_4g`, see [Configurations](#configurations)) to build a Core Complex with **2 Snitch scalar harts sharing 1 Spatz unit**. Motivating workload: tasks where not every hart needs the vector unit at the same time, and control hands Spatz off between the pair at coarse task boundaries. The two harts get sequential hart/core IDs (e.g. cid 0/1 within a pair); `snrt_cluster_is_primary()` (`snrt.h`) tells a hart whether it's the pair's default owner (even `cid`) or its partner (odd `cid`). + +Ownership is arbitrated by `cachepool_spatz_lock.sv`, a small hardware FSM (`Free`/`Locked`/`AcqWait`/`RelWait`) intercepting two dedicated peripheral addresses. It never blocks a core's pipeline: every acquire/release attempt is a plain **load** that always completes immediately, returning an outcome (`FAIL`/`SUCCESS`/`SUCCESS-WAIT`) instead of stalling the hart — so a hart that doesn't get the lock can retry, back off, or do other work instead of hanging. See `software/snRuntime/README.md` for the software API and `note.md` for the full state-machine design. + +**Free vs. Locked performance**: while unlocked (`Free` state), `acc_mux.sv` shares Spatz between both harts via real round-robin arbitration, but every loadstore op must fully drain (real memory completion, not just issue-accept) before the next grant is offered — this holds even for a single hart with no contention from its partner, since the mux has no way to know a sequence of ops belongs to an uncontended hart until it tries to arbitrate again. Loadstore-heavy code therefore runs serialized on real memory latency in `Free` mode. Acquiring the lock (`Locked` state) removes this: the owner's issue path bypasses round-robin arbitration entirely and pipelines back-to-back LSU ops at full throughput. A kernel that does sustained vector/FP loadstore work should hold the lock around that section even if its partner hart never contends for Spatz at all. + +Two per-role partial-barrier helpers, `snrt_cluster_host0_barrier()`/`snrt_cluster_host1_barrier()` (`snrt.h`), let a kernel synchronize only the pair's default owners (or only their partners) without hand-writing a participant mask; both degrade to an ordinary full barrier on a single-scalar-per-CC build, so kernels using them don't need a config-specific `#if`. + ## Stack Each core complex has a local **stack SPM**. Its depth is configured via parameters in `config/config.mk` (forwarded to RTL). If the stack exceeds the local SPM, it spills into the cache space (indexed with core ID bits). @@ -408,7 +424,7 @@ make lint config=cachepool_fpu_4g - If you change cacheline width, `AXI_USER_WIDTH` is derived (supported widths: 128→19, 256→18, 512→17). Unsupported widths error out at generation time. - `make generate` regenerates the FlooNoC package automatically; only run `make update-floonoc` standalone if you're iterating on a `config/floonoc_*.yml` topology file without a full generate. - `make sw` and `make vsim` are decoupled (hw/sw build independently); rebuild whichever side you changed. -- Use `make clean` when switching configs to prevent stale build artifacts. +- Use `make clean` when switching configs to prevent stale build artifacts; `clean.sw`/`clean.hw` (or their finer-grained parts `clean.data`/`clean.generate`/`clean.vsim`) clean only one side if you don't need a full rebuild. - Runtime functions `snrt_tile_id()` and `snrt_num_tiles()` are available to query tile topology from software. - Changing the partition mode or boundary address while the cache holds valid data requires a flush (`l1d_cluster_flush()` or the appropriate cluster-wide partition flush) before reconfiguring. - Set `DEBUG=0` to disable `+acc` and speed up simulation (used by CI); default is `DEBUG=1` for waveform visibility. diff --git a/config/cachepool_dual_4g.mk b/config/cachepool_dual_4g.mk new file mode 100644 index 00000000..e9c5cd63 --- /dev/null +++ b/config/cachepool_dual_4g.mk @@ -0,0 +1,117 @@ +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +# Author: Diyou Shen, ETH Zurich + +######################### +## CachePool Cluster ## +######################### + +# Number of groups +num_groups ?= 4 + +# 2×2 mesh +num_groups_x ?= 2 + +# Number of tiles +num_tiles_per_group ?= 4 + +# Number of Core Complex slots (each holding num_scalar_per_core harts) +num_cores_per_tile ?= 4 + +# Core datawidth +data_width ?= 32 + +# Core addrwidth +addr_width ?= 32 + +num_lg_ports_per_core ?= 2 + +num_rg_ports_per_core ?= 1 + +num_noc_ports_per_tile ?= 4 + + +###################### +## CachePool Tile ## +###################### + +##### L1 Data Cache ##### + +# L1 data cacheline width (in Bit) +l1d_cacheline_width ?= 512 + +# L1 data cache banking factor (how many banks per core?) +l1d_bank_factor ?= 1 + +# L1 coalecsing window +l1d_coal_window ?= 2 + +# L1 data cache number of ways per +l1d_num_way ?= 4 + +# L1 data cache size per tile (KiB) +l1d_tile_size ?= 256 + +# L1 data cache tag width (TODO: should be calcualted) +l1d_tag_data_width ?= 92 + +# Use folded (skewed) data banks +l1d_use_folded ?= 1 + +# Fold-way group size for skewed folded banks (0 = auto: min(4, ways)) +l1d_fold_way_group ?= 0 + +# Use hash-based way selection (required by l1d_use_folded / l1d_use_fwd_buf) +l1d_use_hash_way ?= 1 + +# Enable the SRAM forwarding buffer (requires l1d_use_hash_way) +l1d_use_fwd_buf ?= 1 + +#################### +## CachePool CC ## +#################### +# Spatz fpu support? +spatz_fpu_en ?= 0 + +# Spatz number of FPU +spatz_num_fpu ?= 0 + +# Spatz number of IPU +spatz_num_ipu ?= 4 + +# Spatz max outstanding transactions +spatz_max_trans ?= 32 + +# Snitch/FPU max outstanding transactions +snitch_max_trans ?= 16 + +# 2 Snitch scalar harts sharing one Spatz per Core Complex +num_scalar_per_core ?= 2 + + +##################### +## L2 Main Memory ## +##################### +# DRAM type (selects DRAMSys config and sets default refill_data_width) +dram_type ?= HBM2 + +# L2 number of channels +l2_channel ?= 4 + +# L2 bank width (DRAM width, change with care) +l2_bank_width ?= 512 + +# L2 interleaving factor (in order of bank_width) +l2_interleave ?= 16 + + +################## +## Peripherals ## +################## +# Hardware stack size (in Byte) +stack_hw_size ?= 1024 + +# Stack size (total, including share and private, 32'h800) +stack_tot_size ?= 2048 diff --git a/config/cachepool_dual_fpu_4g.mk b/config/cachepool_dual_fpu_4g.mk new file mode 100644 index 00000000..bb21a89b --- /dev/null +++ b/config/cachepool_dual_fpu_4g.mk @@ -0,0 +1,117 @@ +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +# Author: Diyou Shen, ETH Zurich + +######################### +## CachePool Cluster ## +######################### + +# Number of groups +num_groups ?= 4 + +# 2×2 mesh +num_groups_x ?= 2 + +# Number of tiles +num_tiles_per_group ?= 4 + +# Number of Core Complex slots (each holding num_scalar_per_core harts) +num_cores_per_tile ?= 4 + +# Core datawidth +data_width ?= 32 + +# Core addrwidth +addr_width ?= 32 + +num_lg_ports_per_core ?= 2 + +num_rg_ports_per_core ?= 1 + +num_noc_ports_per_tile ?= 4 + + +###################### +## CachePool Tile ## +###################### + +##### L1 Data Cache ##### + +# L1 data cacheline width (in Bit) +l1d_cacheline_width ?= 512 + +# L1 data cache banking factor (how many banks per core?) +l1d_bank_factor ?= 1 + +# L1 coalecsing window +l1d_coal_window ?= 2 + +# L1 data cache number of ways per +l1d_num_way ?= 4 + +# L1 data cache size per tile (KiB) +l1d_tile_size ?= 256 + +# L1 data cache tag width (TODO: should be calcualted) +l1d_tag_data_width ?= 92 + +# Use folded (skewed) data banks +l1d_use_folded ?= 1 + +# Fold-way group size for skewed folded banks (0 = auto: min(4, ways)) +l1d_fold_way_group ?= 0 + +# Use hash-based way selection (required by l1d_use_folded / l1d_use_fwd_buf) +l1d_use_hash_way ?= 1 + +# Enable the SRAM forwarding buffer (requires l1d_use_hash_way) +l1d_use_fwd_buf ?= 1 + +#################### +## CachePool CC ## +#################### +# Spatz fpu support? +spatz_fpu_en ?= 1 + +# Spatz number of FPU +spatz_num_fpu ?= 4 + +# Spatz number of IPU +spatz_num_ipu ?= 4 + +# Spatz max outstanding transactions +spatz_max_trans ?= 32 + +# Snitch/FPU max outstanding transactions +snitch_max_trans ?= 16 + +# 2 Snitch scalar harts sharing one Spatz per Core Complex +num_scalar_per_core ?= 2 + + +##################### +## L2 Main Memory ## +##################### +# DRAM type (selects DRAMSys config and sets default refill_data_width) +dram_type ?= HBM2 + +# L2 number of channels +l2_channel ?= 4 + +# L2 bank width (DRAM width, change with care) +l2_bank_width ?= 512 + +# L2 interleaving factor (in order of bank_width) +l2_interleave ?= 16 + + +################## +## Peripherals ## +################## +# Hardware stack size (in Byte) +stack_hw_size ?= 1024 + +# Stack size (total, including share and private, 32'h800) +stack_tot_size ?= 2048 diff --git a/config/config.mk b/config/config.mk index b0449bf8..ce97dde5 100644 --- a/config/config.mk +++ b/config/config.mk @@ -131,6 +131,9 @@ spatz_max_trans ?= 32 # Snitch/FPU max outstanding transactions snitch_max_trans ?= 16 +# Number of Snitch scalar harts sharing one Spatz per Core Complex (1 or 2) +num_scalar_per_core ?= 1 + ######################### ## AXI configuration ## diff --git a/hardware/cachepool_peripheral/Makefile b/hardware/cachepool_peripheral/Makefile index dc79b69d..fdb2da45 100644 --- a/hardware/cachepool_peripheral/Makefile +++ b/hardware/cachepool_peripheral/Makefile @@ -5,7 +5,9 @@ # Noah Huetter ROOTDIR=../.. -REGTOOL=`$(ROOTDIR)/install/bender/bender path register_interface`/vendor/lowrisc_opentitan/util/regtool.py +BENDER ?= bender + +REGTOOL=`$(BENDER) path register_interface`/vendor/lowrisc_opentitan/util/regtool.py PYTHON3=$(shell which python) CLANG_FORMAT=$(shell which clang-format-10.0.1) diff --git a/hardware/cachepool_peripheral/cachepool_peripheral_reg.hjson b/hardware/cachepool_peripheral/cachepool_peripheral_reg.hjson index 02aadb65..f9deba9c 100644 --- a/hardware/cachepool_peripheral/cachepool_peripheral_reg.hjson +++ b/hardware/cachepool_peripheral/cachepool_peripheral_reg.hjson @@ -31,6 +31,34 @@ desc: "Hardware barrier register." }] }, + { + name: "SPATZ_LOCK_ACQUIRE", + desc: '''Spatz ownership acquire for dual-Snitch core complexes. Load + attempts to acquire; the loaded value encodes the outcome (fail/ + success/success-wait) plus current owner/locked status.''' + swaccess: "ro", + hwaccess: "hrw", + hwext: "true", + fields: [{ + bits: "31:0", + name: "SPATZ_LOCK_ACQUIRE", + desc: "Spatz ownership acquire register." + }] + }, + { + name: "SPATZ_LOCK_RELEASE", + desc: '''Spatz ownership release for dual-Snitch core complexes. Load + attempts to release; the loaded value encodes the outcome (fail/ + success/success-wait) plus current owner/locked status.''' + swaccess: "ro", + hwaccess: "hrw", + hwext: "true", + fields: [{ + bits: "31:0", + name: "SPATZ_LOCK_RELEASE", + desc: "Spatz ownership release register." + }] + }, { name: "ICACHE_PREFETCH_ENABLE", desc: '''Controls prefetching of the instruction cache.''' diff --git a/hardware/cachepool_peripheral/cachepool_peripheral_reg_pkg.sv b/hardware/cachepool_peripheral/cachepool_peripheral_reg_pkg.sv index 52b2d3ba..6b038474 100644 --- a/hardware/cachepool_peripheral/cachepool_peripheral_reg_pkg.sv +++ b/hardware/cachepool_peripheral/cachepool_peripheral_reg_pkg.sv @@ -20,6 +20,14 @@ package cachepool_peripheral_reg_pkg; logic [31:0] q; } cachepool_peripheral_reg2hw_hw_barrier_reg_t; + typedef struct packed { + logic [31:0] q; + } cachepool_peripheral_reg2hw_spatz_lock_acquire_reg_t; + + typedef struct packed { + logic [31:0] q; + } cachepool_peripheral_reg2hw_spatz_lock_release_reg_t; + typedef struct packed { logic q; } cachepool_peripheral_reg2hw_icache_prefetch_enable_reg_t; @@ -84,6 +92,14 @@ package cachepool_peripheral_reg_pkg; logic [31:0] d; } cachepool_peripheral_hw2reg_hw_barrier_reg_t; + typedef struct packed { + logic [31:0] d; + } cachepool_peripheral_hw2reg_spatz_lock_acquire_reg_t; + + typedef struct packed { + logic [31:0] d; + } cachepool_peripheral_hw2reg_spatz_lock_release_reg_t; + typedef struct packed { logic d; logic de; @@ -105,7 +121,9 @@ package cachepool_peripheral_reg_pkg; // Register -> HW type typedef struct packed { - cachepool_peripheral_reg2hw_hw_barrier_reg_t hw_barrier; // [285:254] + cachepool_peripheral_reg2hw_hw_barrier_reg_t hw_barrier; // [349:318] + cachepool_peripheral_reg2hw_spatz_lock_acquire_reg_t spatz_lock_acquire; // [317:286] + cachepool_peripheral_reg2hw_spatz_lock_release_reg_t spatz_lock_release; // [285:254] cachepool_peripheral_reg2hw_icache_prefetch_enable_reg_t icache_prefetch_enable; // [253:253] cachepool_peripheral_reg2hw_spatz_status_reg_t spatz_status; // [252:252] cachepool_peripheral_reg2hw_spatz_cycle_reg_t spatz_cycle; // [251:220] @@ -125,7 +143,9 @@ package cachepool_peripheral_reg_pkg; // HW -> register type typedef struct packed { - cachepool_peripheral_hw2reg_hw_barrier_reg_t hw_barrier; // [38:7] + cachepool_peripheral_hw2reg_hw_barrier_reg_t hw_barrier; // [102:71] + cachepool_peripheral_hw2reg_spatz_lock_acquire_reg_t spatz_lock_acquire; // [70:39] + cachepool_peripheral_hw2reg_spatz_lock_release_reg_t spatz_lock_release; // [38:7] cachepool_peripheral_hw2reg_l1d_spm_commit_reg_t l1d_spm_commit; // [6:5] cachepool_peripheral_hw2reg_l1d_insn_commit_reg_t l1d_insn_commit; // [4:3] cachepool_peripheral_hw2reg_l1d_flush_status_reg_t l1d_flush_status; // [2:2] @@ -134,33 +154,39 @@ package cachepool_peripheral_reg_pkg; // Register offsets parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_HW_BARRIER_OFFSET = 7'h 0; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE_OFFSET = 7'h 4; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_SPATZ_STATUS_OFFSET = 7'h 8; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_SPATZ_CYCLE_OFFSET = 7'h c; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CLUSTER_BOOT_CONTROL_OFFSET = 7'h 10; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_OFFSET = 7'h 14; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_OFFSET = 7'h 18; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_OFFSET = 7'h 1c; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_0_OFFSET = 7'h 20; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_1_OFFSET = 7'h 24; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_0_OFFSET = 7'h 28; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_1_OFFSET = 7'h 2c; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT_OFFSET = 7'h 30; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT_OFFSET = 7'h 34; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_OFFSET = 7'h 38; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_PRIVATE_OFFSET = 7'h 3c; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_ADDR_OFFSET = 7'h 40; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET = 7'h 44; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT_OFFSET = 7'h 48; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_SPATZ_LOCK_ACQUIRE_OFFSET = 7'h 4; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_SPATZ_LOCK_RELEASE_OFFSET = 7'h 8; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE_OFFSET = 7'h c; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_SPATZ_STATUS_OFFSET = 7'h 10; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_SPATZ_CYCLE_OFFSET = 7'h 14; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CLUSTER_BOOT_CONTROL_OFFSET = 7'h 18; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_OFFSET = 7'h 1c; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_OFFSET = 7'h 20; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_OFFSET = 7'h 24; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_0_OFFSET = 7'h 28; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_1_OFFSET = 7'h 2c; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_0_OFFSET = 7'h 30; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_1_OFFSET = 7'h 34; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT_OFFSET = 7'h 38; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT_OFFSET = 7'h 3c; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_OFFSET = 7'h 40; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_PRIVATE_OFFSET = 7'h 44; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_ADDR_OFFSET = 7'h 48; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET = 7'h 4c; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT_OFFSET = 7'h 50; // Reset values for hwext registers and their fields parameter logic [31:0] CACHEPOOL_PERIPHERAL_HW_BARRIER_RESVAL = 32'h 0; + parameter logic [31:0] CACHEPOOL_PERIPHERAL_SPATZ_LOCK_ACQUIRE_RESVAL = 32'h 0; + parameter logic [31:0] CACHEPOOL_PERIPHERAL_SPATZ_LOCK_RELEASE_RESVAL = 32'h 0; parameter logic [0:0] CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_RESVAL = 1'h 0; parameter logic [0:0] CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_STATUS_RESVAL = 1'h 0; // Register index typedef enum int { CACHEPOOL_PERIPHERAL_HW_BARRIER, + CACHEPOOL_PERIPHERAL_SPATZ_LOCK_ACQUIRE, + CACHEPOOL_PERIPHERAL_SPATZ_LOCK_RELEASE, CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE, CACHEPOOL_PERIPHERAL_SPATZ_STATUS, CACHEPOOL_PERIPHERAL_SPATZ_CYCLE, @@ -182,26 +208,28 @@ package cachepool_peripheral_reg_pkg; } cachepool_peripheral_id_e; // Register width information to check illegal writes - parameter logic [3:0] CACHEPOOL_PERIPHERAL_PERMIT [19] = '{ + parameter logic [3:0] CACHEPOOL_PERIPHERAL_PERMIT [21] = '{ 4'b 1111, // index[ 0] CACHEPOOL_PERIPHERAL_HW_BARRIER - 4'b 0001, // index[ 1] CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE - 4'b 0001, // index[ 2] CACHEPOOL_PERIPHERAL_SPATZ_STATUS - 4'b 1111, // index[ 3] CACHEPOOL_PERIPHERAL_SPATZ_CYCLE - 4'b 1111, // index[ 4] CACHEPOOL_PERIPHERAL_CLUSTER_BOOT_CONTROL - 4'b 0001, // index[ 5] CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT - 4'b 0011, // index[ 6] CACHEPOOL_PERIPHERAL_CFG_L1D_SPM - 4'b 0001, // index[ 7] CACHEPOOL_PERIPHERAL_CFG_L1D_INSN - 4'b 1111, // index[ 8] CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_0 - 4'b 1111, // index[ 9] CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_1 - 4'b 1111, // index[10] CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_0 - 4'b 1111, // index[11] CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_1 - 4'b 0001, // index[12] CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT - 4'b 0001, // index[13] CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT - 4'b 0001, // index[14] CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS - 4'b 0001, // index[15] CACHEPOOL_PERIPHERAL_L1D_PRIVATE - 4'b 1111, // index[16] CACHEPOOL_PERIPHERAL_L1D_ADDR - 4'b 0001, // index[17] CACHEPOOL_PERIPHERAL_XBAR_OFFSET - 4'b 0001 // index[18] CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT + 4'b 1111, // index[ 1] CACHEPOOL_PERIPHERAL_SPATZ_LOCK_ACQUIRE + 4'b 1111, // index[ 2] CACHEPOOL_PERIPHERAL_SPATZ_LOCK_RELEASE + 4'b 0001, // index[ 3] CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE + 4'b 0001, // index[ 4] CACHEPOOL_PERIPHERAL_SPATZ_STATUS + 4'b 1111, // index[ 5] CACHEPOOL_PERIPHERAL_SPATZ_CYCLE + 4'b 1111, // index[ 6] CACHEPOOL_PERIPHERAL_CLUSTER_BOOT_CONTROL + 4'b 0001, // index[ 7] CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT + 4'b 0011, // index[ 8] CACHEPOOL_PERIPHERAL_CFG_L1D_SPM + 4'b 0001, // index[ 9] CACHEPOOL_PERIPHERAL_CFG_L1D_INSN + 4'b 1111, // index[10] CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_0 + 4'b 1111, // index[11] CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_1 + 4'b 1111, // index[12] CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_0 + 4'b 1111, // index[13] CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_1 + 4'b 0001, // index[14] CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT + 4'b 0001, // index[15] CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT + 4'b 0001, // index[16] CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS + 4'b 0001, // index[17] CACHEPOOL_PERIPHERAL_L1D_PRIVATE + 4'b 1111, // index[18] CACHEPOOL_PERIPHERAL_L1D_ADDR + 4'b 0001, // index[19] CACHEPOOL_PERIPHERAL_XBAR_OFFSET + 4'b 0001 // index[20] CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT }; endpackage diff --git a/hardware/cachepool_peripheral/cachepool_peripheral_reg_top.sv b/hardware/cachepool_peripheral/cachepool_peripheral_reg_top.sv index 8552d9e0..6932befe 100644 --- a/hardware/cachepool_peripheral/cachepool_peripheral_reg_top.sv +++ b/hardware/cachepool_peripheral/cachepool_peripheral_reg_top.sv @@ -70,6 +70,10 @@ module cachepool_peripheral_reg_top #( // or _{wd|we|qs} if field == 1 or 0 logic [31:0] hw_barrier_qs; logic hw_barrier_re; + logic [31:0] spatz_lock_acquire_qs; + logic spatz_lock_acquire_re; + logic [31:0] spatz_lock_release_qs; + logic spatz_lock_release_re; logic icache_prefetch_enable_wd; logic icache_prefetch_enable_we; logic spatz_status_wd; @@ -139,6 +143,38 @@ module cachepool_peripheral_reg_top #( ); + // R[spatz_lock_acquire]: V(True) + + prim_subreg_ext #( + .DW (32) + ) u_spatz_lock_acquire ( + .re (spatz_lock_acquire_re), + .we (1'b0), + .wd ('0), + .d (hw2reg.spatz_lock_acquire.d), + .qre (), + .qe (), + .q (reg2hw.spatz_lock_acquire.q ), + .qs (spatz_lock_acquire_qs) + ); + + + // R[spatz_lock_release]: V(True) + + prim_subreg_ext #( + .DW (32) + ) u_spatz_lock_release ( + .re (spatz_lock_release_re), + .we (1'b0), + .wd ('0), + .d (hw2reg.spatz_lock_release.d), + .qre (), + .qe (), + .q (reg2hw.spatz_lock_release.q ), + .qs (spatz_lock_release_qs) + ); + + // R[icache_prefetch_enable]: V(False) prim_subreg #( @@ -618,28 +654,30 @@ module cachepool_peripheral_reg_top #( - logic [18:0] addr_hit; + logic [20:0] addr_hit; always_comb begin addr_hit = '0; addr_hit[ 0] = (reg_addr == CACHEPOOL_PERIPHERAL_HW_BARRIER_OFFSET); - addr_hit[ 1] = (reg_addr == CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE_OFFSET); - addr_hit[ 2] = (reg_addr == CACHEPOOL_PERIPHERAL_SPATZ_STATUS_OFFSET); - addr_hit[ 3] = (reg_addr == CACHEPOOL_PERIPHERAL_SPATZ_CYCLE_OFFSET); - addr_hit[ 4] = (reg_addr == CACHEPOOL_PERIPHERAL_CLUSTER_BOOT_CONTROL_OFFSET); - addr_hit[ 5] = (reg_addr == CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_OFFSET); - addr_hit[ 6] = (reg_addr == CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_OFFSET); - addr_hit[ 7] = (reg_addr == CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_OFFSET); - addr_hit[ 8] = (reg_addr == CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_0_OFFSET); - addr_hit[ 9] = (reg_addr == CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_1_OFFSET); - addr_hit[10] = (reg_addr == CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_0_OFFSET); - addr_hit[11] = (reg_addr == CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_1_OFFSET); - addr_hit[12] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT_OFFSET); - addr_hit[13] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT_OFFSET); - addr_hit[14] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_OFFSET); - addr_hit[15] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_PRIVATE_OFFSET); - addr_hit[16] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_ADDR_OFFSET); - addr_hit[17] = (reg_addr == CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET); - addr_hit[18] = (reg_addr == CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT_OFFSET); + addr_hit[ 1] = (reg_addr == CACHEPOOL_PERIPHERAL_SPATZ_LOCK_ACQUIRE_OFFSET); + addr_hit[ 2] = (reg_addr == CACHEPOOL_PERIPHERAL_SPATZ_LOCK_RELEASE_OFFSET); + addr_hit[ 3] = (reg_addr == CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE_OFFSET); + addr_hit[ 4] = (reg_addr == CACHEPOOL_PERIPHERAL_SPATZ_STATUS_OFFSET); + addr_hit[ 5] = (reg_addr == CACHEPOOL_PERIPHERAL_SPATZ_CYCLE_OFFSET); + addr_hit[ 6] = (reg_addr == CACHEPOOL_PERIPHERAL_CLUSTER_BOOT_CONTROL_OFFSET); + addr_hit[ 7] = (reg_addr == CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_OFFSET); + addr_hit[ 8] = (reg_addr == CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_OFFSET); + addr_hit[ 9] = (reg_addr == CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_OFFSET); + addr_hit[10] = (reg_addr == CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_0_OFFSET); + addr_hit[11] = (reg_addr == CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_1_OFFSET); + addr_hit[12] = (reg_addr == CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_0_OFFSET); + addr_hit[13] = (reg_addr == CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_1_OFFSET); + addr_hit[14] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT_OFFSET); + addr_hit[15] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT_OFFSET); + addr_hit[16] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_OFFSET); + addr_hit[17] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_PRIVATE_OFFSET); + addr_hit[18] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_ADDR_OFFSET); + addr_hit[19] = (reg_addr == CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET); + addr_hit[20] = (reg_addr == CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT_OFFSET); end assign addrmiss = (reg_re || reg_we) ? ~|addr_hit : 1'b0 ; @@ -665,62 +703,68 @@ module cachepool_peripheral_reg_top #( (addr_hit[15] & (|(CACHEPOOL_PERIPHERAL_PERMIT[15] & ~reg_be))) | (addr_hit[16] & (|(CACHEPOOL_PERIPHERAL_PERMIT[16] & ~reg_be))) | (addr_hit[17] & (|(CACHEPOOL_PERIPHERAL_PERMIT[17] & ~reg_be))) | - (addr_hit[18] & (|(CACHEPOOL_PERIPHERAL_PERMIT[18] & ~reg_be))))); + (addr_hit[18] & (|(CACHEPOOL_PERIPHERAL_PERMIT[18] & ~reg_be))) | + (addr_hit[19] & (|(CACHEPOOL_PERIPHERAL_PERMIT[19] & ~reg_be))) | + (addr_hit[20] & (|(CACHEPOOL_PERIPHERAL_PERMIT[20] & ~reg_be))))); end assign hw_barrier_re = addr_hit[0] & reg_re & !reg_error; - assign icache_prefetch_enable_we = addr_hit[1] & reg_we & !reg_error; + assign spatz_lock_acquire_re = addr_hit[1] & reg_re & !reg_error; + + assign spatz_lock_release_re = addr_hit[2] & reg_re & !reg_error; + + assign icache_prefetch_enable_we = addr_hit[3] & reg_we & !reg_error; assign icache_prefetch_enable_wd = reg_wdata[0]; - assign spatz_status_we = addr_hit[2] & reg_we & !reg_error; + assign spatz_status_we = addr_hit[4] & reg_we & !reg_error; assign spatz_status_wd = reg_wdata[0]; - assign spatz_cycle_we = addr_hit[3] & reg_we & !reg_error; + assign spatz_cycle_we = addr_hit[5] & reg_we & !reg_error; assign spatz_cycle_wd = reg_wdata[31:0]; - assign cluster_boot_control_we = addr_hit[4] & reg_we & !reg_error; + assign cluster_boot_control_we = addr_hit[6] & reg_we & !reg_error; assign cluster_boot_control_wd = reg_wdata[31:0]; - assign cluster_eoc_exit_we = addr_hit[5] & reg_we & !reg_error; + assign cluster_eoc_exit_we = addr_hit[7] & reg_we & !reg_error; assign cluster_eoc_exit_wd = reg_wdata[3:0]; - assign cfg_l1d_spm_we = addr_hit[6] & reg_we & !reg_error; + assign cfg_l1d_spm_we = addr_hit[8] & reg_we & !reg_error; assign cfg_l1d_spm_wd = reg_wdata[9:0]; - assign cfg_l1d_insn_we = addr_hit[7] & reg_we & !reg_error; + assign cfg_l1d_insn_we = addr_hit[9] & reg_we & !reg_error; assign cfg_l1d_insn_wd = reg_wdata[1:0]; - assign cfg_l1d_tile_sel_0_we = addr_hit[8] & reg_we & !reg_error; + assign cfg_l1d_tile_sel_0_we = addr_hit[10] & reg_we & !reg_error; assign cfg_l1d_tile_sel_0_wd = reg_wdata[31:0]; - assign cfg_l1d_tile_sel_1_we = addr_hit[9] & reg_we & !reg_error; + assign cfg_l1d_tile_sel_1_we = addr_hit[11] & reg_we & !reg_error; assign cfg_l1d_tile_sel_1_wd = reg_wdata[31:0]; - assign hw_barrier_participation_mask_0_we = addr_hit[10] & reg_we & !reg_error; + assign hw_barrier_participation_mask_0_we = addr_hit[12] & reg_we & !reg_error; assign hw_barrier_participation_mask_0_wd = reg_wdata[31:0]; - assign hw_barrier_participation_mask_1_we = addr_hit[11] & reg_we & !reg_error; + assign hw_barrier_participation_mask_1_we = addr_hit[13] & reg_we & !reg_error; assign hw_barrier_participation_mask_1_wd = reg_wdata[31:0]; - assign l1d_spm_commit_we = addr_hit[12] & reg_we & !reg_error; + assign l1d_spm_commit_we = addr_hit[14] & reg_we & !reg_error; assign l1d_spm_commit_wd = reg_wdata[0]; - assign l1d_insn_commit_we = addr_hit[13] & reg_we & !reg_error; + assign l1d_insn_commit_we = addr_hit[15] & reg_we & !reg_error; assign l1d_insn_commit_wd = reg_wdata[0]; - assign l1d_flush_status_re = addr_hit[14] & reg_re & !reg_error; + assign l1d_flush_status_re = addr_hit[16] & reg_re & !reg_error; - assign l1d_private_we = addr_hit[15] & reg_we & !reg_error; + assign l1d_private_we = addr_hit[17] & reg_we & !reg_error; assign l1d_private_wd = reg_wdata[3:0]; - assign l1d_addr_we = addr_hit[16] & reg_we & !reg_error; + assign l1d_addr_we = addr_hit[18] & reg_we & !reg_error; assign l1d_addr_wd = reg_wdata[31:0]; - assign xbar_offset_we = addr_hit[17] & reg_we & !reg_error; + assign xbar_offset_we = addr_hit[19] & reg_we & !reg_error; assign xbar_offset_wd = reg_wdata[4:0]; - assign xbar_offset_commit_we = addr_hit[18] & reg_we & !reg_error; + assign xbar_offset_commit_we = addr_hit[20] & reg_we & !reg_error; assign xbar_offset_commit_wd = reg_wdata[0]; // Read data return @@ -732,74 +776,82 @@ module cachepool_peripheral_reg_top #( end addr_hit[1]: begin - reg_rdata_next[0] = '0; + reg_rdata_next[31:0] = spatz_lock_acquire_qs; end addr_hit[2]: begin - reg_rdata_next[0] = '0; + reg_rdata_next[31:0] = spatz_lock_release_qs; end addr_hit[3]: begin - reg_rdata_next[31:0] = spatz_cycle_qs; + reg_rdata_next[0] = '0; end addr_hit[4]: begin - reg_rdata_next[31:0] = cluster_boot_control_qs; + reg_rdata_next[0] = '0; end addr_hit[5]: begin - reg_rdata_next[3:0] = cluster_eoc_exit_qs; + reg_rdata_next[31:0] = spatz_cycle_qs; end addr_hit[6]: begin - reg_rdata_next[9:0] = cfg_l1d_spm_qs; + reg_rdata_next[31:0] = cluster_boot_control_qs; end addr_hit[7]: begin - reg_rdata_next[1:0] = cfg_l1d_insn_qs; + reg_rdata_next[3:0] = cluster_eoc_exit_qs; end addr_hit[8]: begin - reg_rdata_next[31:0] = cfg_l1d_tile_sel_0_qs; + reg_rdata_next[9:0] = cfg_l1d_spm_qs; end addr_hit[9]: begin - reg_rdata_next[31:0] = cfg_l1d_tile_sel_1_qs; + reg_rdata_next[1:0] = cfg_l1d_insn_qs; end addr_hit[10]: begin - reg_rdata_next[31:0] = hw_barrier_participation_mask_0_qs; + reg_rdata_next[31:0] = cfg_l1d_tile_sel_0_qs; end addr_hit[11]: begin - reg_rdata_next[31:0] = hw_barrier_participation_mask_1_qs; + reg_rdata_next[31:0] = cfg_l1d_tile_sel_1_qs; end addr_hit[12]: begin - reg_rdata_next[0] = l1d_spm_commit_qs; + reg_rdata_next[31:0] = hw_barrier_participation_mask_0_qs; end addr_hit[13]: begin - reg_rdata_next[0] = l1d_insn_commit_qs; + reg_rdata_next[31:0] = hw_barrier_participation_mask_1_qs; end addr_hit[14]: begin - reg_rdata_next[0] = l1d_flush_status_qs; + reg_rdata_next[0] = l1d_spm_commit_qs; end addr_hit[15]: begin - reg_rdata_next[3:0] = l1d_private_qs; + reg_rdata_next[0] = l1d_insn_commit_qs; end addr_hit[16]: begin - reg_rdata_next[31:0] = l1d_addr_qs; + reg_rdata_next[0] = l1d_flush_status_qs; end addr_hit[17]: begin - reg_rdata_next[4:0] = xbar_offset_qs; + reg_rdata_next[3:0] = l1d_private_qs; end addr_hit[18]: begin + reg_rdata_next[31:0] = l1d_addr_qs; + end + + addr_hit[19]: begin + reg_rdata_next[4:0] = xbar_offset_qs; + end + + addr_hit[20]: begin reg_rdata_next[0] = xbar_offset_commit_qs; end diff --git a/hardware/src/acc_mux.sv b/hardware/src/acc_mux.sv new file mode 100644 index 00000000..48e23ba1 --- /dev/null +++ b/hardware/src/acc_mux.sv @@ -0,0 +1,276 @@ +// Copyright 2025 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 + +// Author: Diyou Shen + +`include "common_cells/assertions.svh" +`include "common_cells/registers.svh" + +/// Arbitrates 2 Snitch acc interfaces onto 1 shared Spatz acc interface for +/// cachepool_cc_dual, gated by cachepool_spatz_lock's owner/locked/waiting +/// state. Locked: only the owner passes through, real access, no arbitration. +/// Free: both hosts get real round-robin access, one request outstanding at +/// a time -- a FIFO tracks which host is owed the in-flight response. +/// Response draining stays active through owner_active_i (Locked and +/// RelWait); new-issue forwarding is gated separately on locked_i alone, so +/// a response already in flight when release is requested still has a drain path. +/// Also demuxes Spatz's LSU-consistency signals (mem_finished/mem_str_finished/ +/// st_rsp_done) to the issuing host, attributed by current-cycle owner so a +/// same-cycle issue+finish (e.g. stores) is never misrouted to a stale owner. +module acc_mux #( + parameter type acc_issue_req_t = logic, + parameter type acc_issue_rsp_t = logic, + parameter type acc_rsp_t = logic +) ( + input logic clk_i, + input logic rst_ni, + + input logic owner_id_i, + input logic locked_i, + input logic waiting_i, + input logic owner_active_i, + + // Per-host Snitch acc interfaces + input acc_issue_req_t [1:0] acc_snitch_req_i, + output acc_issue_rsp_t [1:0] acc_snitch_rsp_o, + input logic [1:0] acc_snitch_qvalid_i, + output logic [1:0] acc_snitch_qready_o, + output acc_rsp_t [1:0] acc_snitch_prsp_o, + output logic [1:0] acc_snitch_pvalid_o, + input logic [1:0] acc_snitch_pready_i, + + // Shared Spatz acc interface + output acc_issue_req_t spatz_issue_req_o, + input acc_issue_rsp_t spatz_issue_rsp_i, + output logic spatz_issue_valid_o, + input logic spatz_issue_ready_i, + input acc_rsp_t spatz_rsp_i, + input logic spatz_rsp_valid_i, + output logic spatz_rsp_ready_o, + + // Spatz's raw LSU-consistency signals, demuxed to the per-host outputs below. + input logic [1:0] spatz_mem_finished_i, + input logic [1:0] spatz_mem_str_finished_i, + input logic spatz_st_rsp_done_i, + + // Drain feed for the lock's outstanding counter (owner and Free-mode paths both count). + output logic req_fire_o, + output logic rsp_fire_o, + + // Per-host demuxed copies of the signals above, attributed to the actual current-cycle + // owner (not a registered value that can go stale by one cycle on same-cycle completions). + output logic [1:0][1:0] acc_mem_finished_o, + output logic [1:0][1:0] acc_mem_str_finished_o, + output logic [1:0] acc_st_rsp_done_o +); + + logic req_fire_any; + assign req_fire_any = spatz_issue_valid_o & spatz_issue_ready_i; + + // Only a writeback-producing issue leaves something for the lock to drain + // before a switch; a non-writeback op completes at the same-cycle accept. + assign req_fire_o = req_fire_any & spatz_issue_rsp_i.writeback; + assign rsp_fire_o = spatz_rsp_valid_i & spatz_rsp_ready_o; + + // Free-mode round-robin arbitration, offered only while nothing is already outstanding. + logic free_gnt_i, free_req_o; + logic free_winner; + logic [1:0] free_req_i; + acc_issue_req_t free_data_o; + logic [1:0] free_gnt_o; + + logic route_fifo_empty, route_fifo_push, route_fifo_pop, route_fifo_route_q; + logic route_fifo_full; + + // Holds off the next Free-mode grant until the current LSU op has actually + // drained (not just accepted), so last_req_host_q can't go stale mid-drain. + logic lsu_busy_d, lsu_busy_q; + + always_comb begin + lsu_busy_d = lsu_busy_q; + if (req_fire_any && spatz_issue_rsp_i.loadstore) lsu_busy_d = 1'b1; + if (spatz_mem_finished_i[0] || spatz_mem_finished_i[1]) lsu_busy_d = 1'b0; + end + + `FF(lsu_busy_q, lsu_busy_d, 1'b0, clk_i, rst_ni) + + assign free_req_i = (!locked_i && !waiting_i && route_fifo_empty && !lsu_busy_q) ? acc_snitch_qvalid_i : 2'b00; + assign free_gnt_i = spatz_issue_ready_i; + + rr_arb_tree #( + .NumIn (2 ), + .DataType (acc_issue_req_t), + .AxiVldRdy (1'b1 ), + .LockIn (1'b1 ) + ) i_free_arb ( + .clk_i, + .rst_ni, + .flush_i (1'b0 ), + .rr_i ('0 ), + .req_i (free_req_i ), + .gnt_o (free_gnt_o ), + .data_i (acc_snitch_req_i), + .req_o (free_req_o ), + .gnt_i (free_gnt_i ), + .data_o (free_data_o ), + .idx_o (free_winner ) + ); + + // Only track requests with a real completion coming: non-writeback ops never produce a later acc_rsp_t. + assign route_fifo_push = free_req_o && free_gnt_i && spatz_issue_rsp_i.writeback; + assign route_fifo_pop = rsp_fire_o && !owner_active_i; + + // Depth 2 for safety margin; only 1 entry is ever pushed at a time. + fifo_v3 #( + .DEPTH (2 ), + .dtype (logic) + ) i_route_fifo ( + .clk_i, + .rst_ni, + .flush_i (1'b0 ), + .testmode_i (1'b0 ), + .full_o (route_fifo_full ), + .empty_o (route_fifo_empty ), + .usage_o ( ), + .data_i (free_winner ), + .push_i (route_fifo_push ), + .data_o (route_fifo_route_q), + .pop_i (route_fifo_pop ) + ); + + // Current owner for this cycle's LSU-consistency demux: the just-issued host when a + // grant is firing right now (covers same-cycle issue+finish, e.g. stores), else the + // latched host from whichever grant is still draining (covers delayed completions). + logic last_req_host_q; + logic cur_owner; + assign cur_owner = req_fire_any ? (locked_i ? owner_id_i : free_winner) : last_req_host_q; + + `FF(last_req_host_q, cur_owner, 1'b0, clk_i, rst_ni) + + for (genvar h = 0; h < 2; h++) begin : gen_lsu_consistency_demux + assign acc_mem_finished_o[h] = (cur_owner == h[0]) ? spatz_mem_finished_i : '0; + assign acc_mem_str_finished_o[h] = (cur_owner == h[0]) ? spatz_mem_str_finished_i : '0; + assign acc_st_rsp_done_o[h] = (cur_owner == h[0]) ? spatz_st_rsp_done_i : 1'b1; + end + + always_comb begin + spatz_issue_req_o = acc_issue_req_t'('0); + spatz_issue_valid_o = 1'b0; + spatz_rsp_ready_o = 1'b0; + + acc_snitch_qready_o = '0; + acc_snitch_rsp_o = '0; + acc_snitch_prsp_o = '0; + acc_snitch_pvalid_o = '0; + + if (owner_active_i) begin + // Draining runs through Locked/RelWait; new-issue forwarding is steady-state-Locked only. + spatz_rsp_ready_o = acc_snitch_pready_i[owner_id_i]; + acc_snitch_prsp_o[owner_id_i] = spatz_rsp_i; + acc_snitch_pvalid_o[owner_id_i] = spatz_rsp_valid_i; + + if (locked_i) begin + spatz_issue_req_o = acc_snitch_req_i[owner_id_i]; + spatz_issue_valid_o = acc_snitch_qvalid_i[owner_id_i]; + + acc_snitch_qready_o[owner_id_i] = spatz_issue_ready_i; + acc_snitch_rsp_o[owner_id_i] = spatz_issue_rsp_i; + end + end else begin + // Draining is unconditional on waiting_i; only new-request arbitration is not. + if (!route_fifo_empty) begin + spatz_rsp_ready_o = acc_snitch_pready_i[route_fifo_route_q]; + acc_snitch_prsp_o[route_fifo_route_q] = spatz_rsp_i; + acc_snitch_pvalid_o[route_fifo_route_q] = spatz_rsp_valid_i; + end else if (!waiting_i) begin + spatz_issue_req_o = free_data_o; + spatz_issue_valid_o = free_req_o; + acc_snitch_qready_o[free_winner] = free_req_o && spatz_issue_ready_i; + acc_snitch_rsp_o[free_winner] = spatz_issue_rsp_i; + end + end + end + +`ifndef TARGET_SYNTHESIS + // Debug-only: verbose per-event log (+acc_mux_verbose plusarg) and a + // stuck-transaction watchdog. Added to debug the FPU-enabled dual-CC + // boot hang; temporary instrumentation, safe to strip once resolved. + bit acc_mux_verbose = 1'b0; + initial begin + // verilog_lint: waive plusarg-assignment + acc_mux_verbose = $test$plusargs("acc_mux_verbose"); + end + + logic spatz_rsp_valid_q; + `FF(spatz_rsp_valid_q, spatz_rsp_valid_i, 1'b0, clk_i, rst_ni) + + always_ff @(posedge clk_i) begin + if (rst_ni && acc_mux_verbose) begin + if (free_req_o && free_gnt_i) begin + $display({"[ACC-MUX %0t %m] FREE-GRANT host=%0d accept=%0b writeback=%0b ", + "loadstore=%0b isfloat=%0b"}, + $time, free_winner, spatz_issue_rsp_i.accept, spatz_issue_rsp_i.writeback, + spatz_issue_rsp_i.loadstore, spatz_issue_rsp_i.isfloat); + end + if (route_fifo_push) begin + $display("[ACC-MUX %0t %m] ROUTE-FIFO PUSH host=%0d", $time, free_winner); + end + if (route_fifo_pop) begin + $display("[ACC-MUX %0t %m] ROUTE-FIFO POP host=%0d", $time, route_fifo_route_q); + end + if (req_fire_any) begin + $display({"[ACC-MUX %0t %m] REQ-FIRE owner_active=%0b locked=%0b owner=%0d ", + "writeback=%0b"}, + $time, owner_active_i, locked_i, owner_id_i, spatz_issue_rsp_i.writeback); + end + if (rsp_fire_o) begin + $display("[ACC-MUX %0t %m] RSP-FIRE owner_active=%0b route_host=%0d", + $time, owner_active_i, route_fifo_route_q); + end + if (spatz_mem_finished_i[0] || spatz_mem_finished_i[1]) begin + $display("[ACC-MUX %0t %m] LSU-FINISH cur_owner=%0d mem_finished=%0b", + $time, cur_owner, spatz_mem_finished_i); + end + // Edge-triggered: which side of the p-channel handshake (valid vs. + // ready) is actually stuck once route_fifo has something owed. + if (spatz_rsp_valid_i != spatz_rsp_valid_q) begin + $display({"[ACC-MUX %0t %m] SPATZ-RSP-VALID -> %0b route_fifo_empty=%0b ", + "route_host=%0d owner_active=%0b spatz_rsp_ready=%0b"}, + $time, spatz_rsp_valid_i, route_fifo_empty, route_fifo_route_q, + owner_active_i, spatz_rsp_ready_o); + end + end + end + + // Watchdog: warn if a round-robin (Free-mode) transaction has been + // accepted by Spatz but its response hasn't drained (route_fifo stuck + // non-empty) for more than AccMuxWdogPs, re-warning every AccMuxWdogPs + // while still stuck. + localparam longint unsigned AccMuxWdogPs = 5_000_000; + logic [63:0] acc_mux_last_progress_q, acc_mux_last_warn_q; + + always_ff @(posedge clk_i or negedge rst_ni) begin + if (!rst_ni) begin + acc_mux_last_progress_q <= '0; + acc_mux_last_warn_q <= '0; + end else begin + if (route_fifo_empty || route_fifo_push || route_fifo_pop) begin + acc_mux_last_progress_q <= 64'($time); + acc_mux_last_warn_q <= 64'($time); + end + if (!route_fifo_empty && + (64'($time) - acc_mux_last_progress_q) > AccMuxWdogPs && + (64'($time) - acc_mux_last_warn_q) > AccMuxWdogPs) begin + acc_mux_last_warn_q <= 64'($time); + $display({"[%0t] [ACC-MUX %m] STUCK: route_fifo non-empty, owed host=%0d ", + "locked=%0b waiting=%0b owner_active=%0b free_req_i=%0b ", + "spatz_rsp_valid=%0b spatz_rsp_ready=%0b"}, + $time, route_fifo_route_q, locked_i, waiting_i, owner_active_i, + free_req_i, spatz_rsp_valid_i, spatz_rsp_ready_o); + end + end + end +`endif // TARGET_SYNTHESIS + +endmodule diff --git a/hardware/src/cachepool_cc_dual.sv b/hardware/src/cachepool_cc_dual.sv new file mode 100644 index 00000000..c306d1d7 --- /dev/null +++ b/hardware/src/cachepool_cc_dual.sv @@ -0,0 +1,662 @@ +// Copyright 2025 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 + +// Author: Diyou Shen + +`include "common_cells/assertions.svh" +`include "common_cells/registers.svh" +`include "snitch_vm/typedef.svh" +`include "reqrsp_interface/typedef.svh" + +/// CachePool Core Complex (dual flavor): 2 Snitch Integer Cores sharing +/// 1 Spatz Vector Unit, gated by cachepool_spatz_lock. cachepool_cc.sv +/// (single-hart) is untouched; this is a sibling module for tiles built +/// with num_scalar_per_core=2. +module cachepool_cc_dual + import snitch_pkg::interrupts_t; + import fpnew_pkg::fpu_implementation_t; #( + /// Address width of the buses + parameter int unsigned AddrWidth = 0, + /// Data width of the buses. + parameter int unsigned DataWidth = 0, + /// User width of the buses. + parameter int unsigned UserWidth = 0, + + parameter int unsigned SpmStackDepth = 512, + /// Data port request type. + parameter type dreq_t = logic, + /// Data port response type. + parameter type drsp_t = logic, + parameter type dreq_chan_t = logic, + parameter type drsp_chan_t = logic, + // TCDM port types + parameter type tcdm_req_t = logic, + parameter type tcdm_user_t = logic, + parameter type tcdm_req_chan_t = logic, + parameter type tcdm_rsp_t = logic, + parameter type tcdm_rsp_chan_t = logic, + /// TCDM Address Width + parameter int unsigned TCDMAddrWidth = 0, + parameter type hive_req_t = logic, + parameter type hive_rsp_t = logic, + parameter type acc_issue_req_t = logic, + parameter type acc_issue_rsp_t = logic, + parameter type acc_rsp_t = logic, + /// FPU configuration. + parameter fpu_implementation_t FPUImplementation = fpu_implementation_t'(0), + /// Boot address of core. + parameter logic [31:0] BootAddr = 32'h0000_1000, + + /// Address to indicate start of L2 + parameter logic [AddrWidth-1:0] UartAddr = 32'h0C00_0000, + /// Reduced-register extension + parameter bit RVE = 0, + /// Enable F and D Extension + parameter bit RVF = 1, + parameter bit RVD = 0, + parameter bit XDivSqrt = 0, + parameter bit XF8 = 0, + parameter bit XF16 = 0, + parameter bit XF16ALT = 0, + parameter bit XF8ALT = 0, + parameter int unsigned NumIntOutstandingLoads = 0, + parameter int unsigned NumIntOutstandingMem = 0, + parameter int unsigned NumSpatzOutstandingLoads = 0, + // Enable V Extension + parameter bit RVV = 1, + // Spatz paramaters + parameter int unsigned NumSpatzFPUs = 4, + parameter int unsigned NumSpatzIPUs = 1, + /// Add isochronous clock-domain crossings e.g., make it possible to operate + /// the core in a slower clock domain. + parameter bit IsoCrossing = 0, + /// Timing Parameters + /// Insert Pipeline registers into off-loading path (response) + parameter bit RegisterOffloadRsp = 0, + /// Insert Pipeline registers into data memory path (request) + parameter bit RegisterCoreReq = 0, + /// Insert Pipeline registers into data memory path (response) + parameter bit RegisterCoreRsp = 0, + parameter snitch_pma_pkg::snitch_pma_t SnitchPMACfg = '{default: 0}, + /// Derived parameter *Do not override* + parameter int unsigned NumSpatzFUs = (NumSpatzFPUs > NumSpatzIPUs) ? NumSpatzFPUs : NumSpatzIPUs, + parameter int unsigned NumMemPortsPerSpatz = NumSpatzFUs, + /// 2 dedicated scalar ports (one per host) instead of 1 merged one. + parameter int unsigned TCDMPorts = RVV ? NumMemPortsPerSpatz + 2 : 2, + parameter type addr_t = logic [AddrWidth-1:0], + parameter type req_id_t = logic [$clog2(NumSpatzOutstandingLoads)-1:0] + ) ( + input logic clk_i, + input logic rst_ni, + input logic testmode_i, + input logic [1:0][31:0] hart_id_i, + input interrupts_t [1:0] irq_i, + output hive_req_t [1:0] hive_req_o, + input hive_rsp_t [1:0] hive_rsp_i, + // Core data ports (one per host, bypasses cache/TCDM) + output dreq_t [1:0] data_req_o, + input drsp_t [1:0] data_rsp_i, + // TCDM Streamer Ports + output tcdm_req_t [TCDMPorts-1:0] tcdm_req_o, + input tcdm_rsp_t [TCDMPorts-1:0] tcdm_rsp_i, + // Response-side ready, reflecting real downstream capacity (Spatz + scalar ports). + output logic [TCDMPorts-1:0] tcdm_rsp_ready_o, + input addr_t tcdm_addr_base_i, + input addr_t cluster_periph_start_address_i, + // Spatz lock/switch status (debug) + output logic owner_id_o + ); + + // FMA architecture is "merged" -> mulexp and macexp instructions are supported + localparam bit FPEn = RVF | RVD | XF16 | XF8; + localparam int unsigned FLEN = + RVD ? 64 : // D ext. + RVF ? 32 : // F ext. + XF16 ? 16 : // Xf16 ext. + XF8 ? 8 : // Xf8 ext. + 0; // Unused in case of no FP + + `SNITCH_VM_TYPEDEF(AddrWidth) + + typedef logic [DataWidth-1:0] data_t; + typedef logic [DataWidth/8-1:0] strb_t; + + `REQRSP_TYPEDEF_ALL(reqrsp, addr_t, data_t, strb_t, tcdm_user_t) + + logic owner_id, locked, waiting, owner_active; + logic req_fire, rsp_fire; + assign owner_id_o = owner_id; + + // Raw per-host Snitch data port, and its acc issue/response bundle. + dreq_t [1:0] snitch_dreq_d; + drsp_t [1:0] snitch_drsp_d; + + acc_issue_req_t [1:0] acc_snitch_req; + acc_issue_rsp_t [1:0] acc_snitch_rsp; + logic [1:0] acc_snitch_qvalid, acc_snitch_qready; + acc_rsp_t [1:0] acc_snitch_prsp; + logic [1:0] acc_snitch_pvalid, acc_snitch_pready; + + fpnew_pkg::roundmode_e [1:0] fpu_rnd_mode_h; + fpnew_pkg::fmt_mode_t [1:0] fpu_fmt_mode_h; + fpnew_pkg::status_t fpu_status; + + // Spatz's raw LSU-consistency signals; acc_mux demuxes these to the issuing host. + logic [1:0] spatz_mem_finished; + logic [1:0] spatz_mem_str_finished; + logic spatz_st_rsp_done; + + logic [1:0][1:0] acc_mem_finished_h; + logic [1:0][1:0] acc_mem_str_finished_h; + logic [1:0] acc_st_rsp_done_h; + + for (genvar h = 0; h < 2; h++) begin : gen_snitch + snitch #( + .AddrWidth (AddrWidth ), + .DataWidth (DataWidth ), + .acc_issue_req_t (acc_issue_req_t ), + .acc_issue_rsp_t (acc_issue_rsp_t ), + .acc_rsp_t (acc_rsp_t ), + .dreq_t (dreq_t ), + .drsp_t (drsp_t ), + .pa_t (pa_t ), + .l0_pte_t (l0_pte_t ), + .id_t (req_id_t ), + .BootAddr (BootAddr ), + .SnitchPMACfg (SnitchPMACfg ), + .NumIntOutstandingLoads (NumIntOutstandingLoads), + .NumIntOutstandingMem (NumIntOutstandingMem ), + .VMSupport (1'b0 ), + .RVE (RVE ), + .FP_EN (FPEn ), + .Xdma (1'b0 ), + .RVF (RVF ), + .RVD (RVD ), + .RVV (RVV ), + .XDivSqrt (XDivSqrt ), + .XF16 (XF16 ), + .XF16ALT (XF16ALT ), + .XF8 (XF8 ), + .XF8ALT (XF8ALT ), + .FLEN (FLEN ) + ) i_snitch ( + .clk_i (clk_i ), + .rst_i (!rst_ni ), + .hart_id_i (hart_id_i[h] ), + .irq_i (irq_i[h] ), + .flush_i_valid_o (hive_req_o[h].flush_i_valid ), + .flush_i_ready_i (hive_rsp_i[h].flush_i_ready ), + .inst_addr_o (hive_req_o[h].inst_addr ), + .inst_cacheable_o (hive_req_o[h].inst_cacheable), + .inst_data_i (hive_rsp_i[h].inst_data ), + .inst_valid_o (hive_req_o[h].inst_valid ), + .inst_ready_i (hive_rsp_i[h].inst_ready ), + .acc_qreq_o (acc_snitch_req[h] ), + .acc_qrsp_i (acc_snitch_rsp[h] ), + .acc_qvalid_o (acc_snitch_qvalid[h] ), + .acc_qready_i (acc_snitch_qready[h] ), + .acc_prsp_i (acc_snitch_prsp[h] ), + .acc_pvalid_i (acc_snitch_pvalid[h] ), + .acc_pready_o (acc_snitch_pready[h] ), + .acc_mem_finished_i (acc_mem_finished_h[h] ), + .acc_mem_str_finished_i(acc_mem_str_finished_h[h] ), + .acc_st_rsp_done_i (acc_st_rsp_done_h[h] ), + .data_req_o (snitch_dreq_d[h] ), + .data_rsp_i (snitch_drsp_d[h] ), + .ptw_valid_o (hive_req_o[h].ptw_valid ), + .ptw_ready_i (hive_rsp_i[h].ptw_ready ), + .ptw_va_o (hive_req_o[h].ptw_va ), + .ptw_ppn_o (hive_req_o[h].ptw_ppn ), + .ptw_pte_i (hive_rsp_i[h].ptw_pte ), + .ptw_is_4mega_i (hive_rsp_i[h].ptw_is_4mega ), + .fpu_rnd_mode_o (fpu_rnd_mode_h[h] ), + .fpu_fmt_mode_o (fpu_fmt_mode_h[h] ), + .fpu_status_i (fpu_status ), + .core_events_o (/* unused */ ) + ); + + // There is no shared muldiv in this configuration + assign hive_req_o[h].acc_qvalid = 1'b0; + assign hive_req_o[h].acc_pready = 1'b0; + assign hive_req_o[h].acc_req = '0; + end + + // Spatz issue/response bundle, owner-routed by the lock. + acc_issue_req_t spatz_issue_req; + acc_issue_rsp_t spatz_issue_rsp; + logic spatz_issue_valid, spatz_issue_ready; + acc_rsp_t spatz_rsp; + logic spatz_rsp_valid, spatz_rsp_ready; + + // LSU issue handshake fire, gates the lock's LSU drain counter. + logic spatz_lsu_issue_fire; + assign spatz_lsu_issue_fire = spatz_issue_valid && spatz_issue_ready && spatz_issue_rsp.loadstore; + + // Lock-module pass-through data interface (accept/deliver on the module's + // own registered output side; see cachepool_spatz_lock.sv). + dreq_t [1:0] lock_out_req; + drsp_t [1:0] lock_out_rsp; + + cachepool_spatz_lock #( + .AddrWidth (AddrWidth ), + .NumHosts (2 ), + .dreq_t (dreq_t ), + .drsp_t (drsp_t ), + .dreq_chan_t (dreq_chan_t ), + .drsp_chan_t (drsp_chan_t ), + .user_t (tcdm_user_t ), + .RegisterCoreReq (RegisterCoreReq ), + .RegisterCoreRsp (RegisterCoreRsp ), + .addr_t (addr_t ) + ) i_spatz_lock ( + .clk_i (clk_i ), + .rst_ni (rst_ni ), + .in_req_i (snitch_dreq_d ), + .in_rsp_o (snitch_drsp_d ), + .out_req_o (lock_out_req ), + .out_rsp_i (lock_out_rsp ), + .req_fire_i (req_fire ), + .rsp_fire_i (rsp_fire ), + .spatz_lsu_issue_fire_i (spatz_lsu_issue_fire), + .spatz_mem_finished_i (spatz_mem_finished ), + .spatz_st_rsp_done_i (spatz_st_rsp_done ), + .owner_id_o (owner_id ), + .locked_o (locked ), + .waiting_o (waiting ), + .owner_active_o (owner_active ), + .cluster_periph_start_address_i (cluster_periph_start_address_i) + ); + + // Acc mux: real round-robin access to Spatz for both hosts while + // unlocked, exclusive owner access while locked. See acc_mux.sv. + acc_mux #( + .acc_issue_req_t (acc_issue_req_t), + .acc_issue_rsp_t (acc_issue_rsp_t), + .acc_rsp_t (acc_rsp_t ) + ) i_acc_mux ( + .clk_i (clk_i ), + .rst_ni (rst_ni ), + .owner_id_i (owner_id ), + .locked_i (locked ), + .waiting_i (waiting ), + .owner_active_i (owner_active ), + .acc_snitch_req_i (acc_snitch_req ), + .acc_snitch_rsp_o (acc_snitch_rsp ), + .acc_snitch_qvalid_i (acc_snitch_qvalid ), + .acc_snitch_qready_o (acc_snitch_qready ), + .acc_snitch_prsp_o (acc_snitch_prsp ), + .acc_snitch_pvalid_o (acc_snitch_pvalid ), + .acc_snitch_pready_i (acc_snitch_pready ), + .spatz_issue_req_o (spatz_issue_req ), + .spatz_issue_rsp_i (spatz_issue_rsp ), + .spatz_issue_valid_o (spatz_issue_valid ), + .spatz_issue_ready_i (spatz_issue_ready ), + .spatz_rsp_i (spatz_rsp ), + .spatz_rsp_valid_i (spatz_rsp_valid ), + .spatz_rsp_ready_o (spatz_rsp_ready ), + .spatz_mem_finished_i (spatz_mem_finished ), + .spatz_mem_str_finished_i(spatz_mem_str_finished ), + .spatz_st_rsp_done_i (spatz_st_rsp_done ), + .req_fire_o (req_fire ), + .rsp_fire_o (rsp_fire ), + .acc_mem_finished_o (acc_mem_finished_h ), + .acc_mem_str_finished_o (acc_mem_str_finished_h ), + .acc_st_rsp_done_o (acc_st_rsp_done_h ) + ); + + // FPU rounding-mode/format CSRs: Spatz has one FPU, so it only ever sees the + // current owner's config; its status broadcasts back to both hosts. + fpnew_pkg::roundmode_e fpu_rnd_mode; + fpnew_pkg::fmt_mode_t fpu_fmt_mode; + assign fpu_rnd_mode = fpu_rnd_mode_h[owner_id]; + assign fpu_fmt_mode = fpu_fmt_mode_h[owner_id]; + + dreq_t fp_lsu_mem_req; + drsp_t fp_lsu_mem_rsp; + + tcdm_req_chan_t [NumMemPortsPerSpatz-1:0] spatz_mem_req; + logic [NumMemPortsPerSpatz-1:0] spatz_mem_req_valid; + logic [NumMemPortsPerSpatz-1:0] spatz_mem_req_ready; + tcdm_rsp_chan_t [NumMemPortsPerSpatz-1:0] spatz_mem_rsp, spatz_mem_fifo; + logic [NumMemPortsPerSpatz-1:0] spatz_mem_rsp_valid; + logic [NumMemPortsPerSpatz-1:0] spatz_mem_rsp_ready; + logic [NumMemPortsPerSpatz-1:0] spatz_mem_rsp_empty, spatz_mem_rsp_full; + logic [NumMemPortsPerSpatz-1:0] spatz_mem_rsp_pop, spatz_mem_rsp_push; + localparam int unsigned SpatzRspFifoDepth = + (NumSpatzOutstandingLoads > 0) ? NumSpatzOutstandingLoads : 1; + + spatz #( + .NrMemPorts (NumMemPortsPerSpatz ), + .NumOutstandingLoads(NumSpatzOutstandingLoads), + .FPUImplementation (FPUImplementation ), + .RegisterRsp (RegisterOffloadRsp ), + .dreq_t (dreq_t ), + .drsp_t (drsp_t ), + .spatz_mem_req_t (tcdm_req_chan_t ), + .spatz_mem_rsp_t (tcdm_rsp_chan_t ), + .spatz_issue_req_t (acc_issue_req_t ), + .spatz_issue_rsp_t (acc_issue_rsp_t ), + .spatz_rsp_t (acc_rsp_t ) + ) i_spatz ( + .clk_i (clk_i ), + .rst_ni (rst_ni ), + .testmode_i (testmode_i ), + .hart_id_i (hart_id_i[owner_id] ), + .issue_valid_i (spatz_issue_valid ), + .issue_ready_o (spatz_issue_ready ), + .issue_req_i (spatz_issue_req ), + .issue_rsp_o (spatz_issue_rsp ), + .rsp_valid_o (spatz_rsp_valid ), + .rsp_ready_i (spatz_rsp_ready ), + .rsp_o (spatz_rsp ), + .spatz_mem_req_o (spatz_mem_req ), + .spatz_mem_req_valid_o (spatz_mem_req_valid ), + .spatz_mem_req_ready_i (spatz_mem_req_ready ), + .spatz_mem_rsp_i (spatz_mem_rsp ), + .spatz_mem_rsp_valid_i (spatz_mem_rsp_valid ), + .spatz_mem_rsp_ready_o (spatz_mem_rsp_ready ), + .spatz_mem_finished_o (spatz_mem_finished ), + .spatz_mem_str_finished_o(spatz_mem_str_finished), + .spatz_st_rsp_done_o (spatz_st_rsp_done ), + .fp_lsu_mem_req_o (fp_lsu_mem_req ), + .fp_lsu_mem_rsp_i (fp_lsu_mem_rsp ), + .fpu_rnd_mode_i (fpu_rnd_mode ), + .fpu_fmt_mode_i (fpu_fmt_mode ), + .fpu_status_o (fpu_status ) + ); + + // Vector-FU TCDM ports pin to host 0's crossbar row (only one physical + // row per CC pair); acc_mux re-attributes responses to the true owner. + for (genvar p = 0; p < NumMemPortsPerSpatz; p++) begin : gen_spatz_mem_ports + assign tcdm_req_o[p] = '{ + q : spatz_mem_req[p], + q_valid: spatz_mem_req_valid[p] + }; + assign spatz_mem_req_ready[p] = tcdm_rsp_i[p].q_ready; + + fifo_v3 #( + .dtype (tcdm_rsp_chan_t ), + .DEPTH (SpatzRspFifoDepth ), + .FALL_THROUGH (1 ) + ) i_spatz_rsp_fifo ( + .clk_i (clk_i ), + .rst_ni (rst_ni ), + .flush_i (1'b0 ), + .testmode_i(1'b0 ), + .data_i (tcdm_rsp_i[p].p ), + .push_i (spatz_mem_rsp_push[p] ), + .data_o (spatz_mem_fifo[p] ), + .pop_i (spatz_mem_rsp_pop[p] ), + .full_o (spatz_mem_rsp_full[p] ), + .empty_o (spatz_mem_rsp_empty[p]), + .usage_o (/* Unused */ ) + ); + assign tcdm_rsp_ready_o[p] = !spatz_mem_rsp_full[p]; + + always_comb begin + spatz_mem_rsp_valid[p] = !spatz_mem_rsp_empty[p]; + spatz_mem_rsp[p] = spatz_mem_fifo[p]; + // Only push once the handshake completes (valid & ready), else the fifo overflows silently. + spatz_mem_rsp_push[p] = tcdm_rsp_i[p].p_valid & tcdm_rsp_ready_o[p]; + spatz_mem_rsp_pop[p] = spatz_mem_rsp_valid[p] & spatz_mem_rsp_ready[p]; + end + end + + // --------------------------------------------------------------------- + // Per-host scalar crossbar: {SnitchHMem, FPUMem-when-owner==h} -> + // {CacheMemH, SpmStackH, PeriphH}. Two small crossbars instead of one + // all-to-all one -- see note.md "Per-host crossbars". + // --------------------------------------------------------------------- + typedef enum integer { + SnitchMst = 0, + FpuMst = 1 + } scalar_mem_mst_e; + + typedef enum integer { + CacheMem = 0, + SpmStack = 1, + Periph = 2 + } scalar_mem_slv_e; + + localparam int unsigned NrScalarXbarMst = 2; + localparam int unsigned NrScalarXbarSlv = 3; + + localparam int unsigned SelectMstWidth = cf_math_pkg::idx_width(NrScalarXbarSlv); + localparam int unsigned SelectSlvWidth = cf_math_pkg::idx_width(NrScalarXbarMst); + typedef logic [SelectMstWidth-1:0] select_mst_t; + typedef logic [SelectSlvWidth-1:0] select_slv_t; + + localparam int unsigned StackAddrWidth = $clog2(SpmStackDepth); + typedef logic [StackAddrWidth-1:0] tcdm_stack_addr_t; + typedef struct packed { + tcdm_user_t user; + logic valid; + logic write; + } stack_meta_t; + + drsp_t [1:0] fpu_rsp_per_host; + + assign fp_lsu_mem_rsp = fpu_rsp_per_host[owner_id]; + + // Per-host signals, hoisted out of gen_host as 2D arrays (indexed [host][...]) + // for debug visibility, rather than living inside separate generate scopes. + dreq_t [1:0] fpu_req_gated; + + dreq_chan_t [1:0][NrScalarXbarMst-1:0] core_req_chan; + drsp_chan_t [1:0][NrScalarXbarMst-1:0] core_rsp_chan; + logic [1:0][NrScalarXbarMst-1:0] core_req_valid, core_req_ready; + logic [1:0][NrScalarXbarMst-1:0] core_rsp_valid, core_rsp_ready; + + dreq_chan_t [1:0][NrScalarXbarSlv-1:0] mem_req_chan; + drsp_chan_t [1:0][NrScalarXbarSlv-1:0] mem_rsp_chan; + logic [1:0][NrScalarXbarSlv-1:0] mem_req_valid, mem_req_ready; + logic [1:0][NrScalarXbarSlv-1:0] mem_rsp_valid, mem_rsp_ready; + + dreq_t [1:0][NrScalarXbarSlv-1:0] mem_req; + drsp_t [1:0][NrScalarXbarSlv-1:0] mem_rsp; + + select_mst_t [1:0][NrScalarXbarMst-1:0] core_req_sel; + select_slv_t [1:0][NrScalarXbarSlv-1:0] core_selected, mem_rsp_sel; + + logic [1:0] is_totstack; + + logic [1:0] stack_valid, stack_we; + tcdm_stack_addr_t [1:0] stack_add; + strb_t [1:0] stack_be; + data_t [1:0] stack_rdata, stack_wdata; + stack_meta_t [1:0] stack_req_meta, stack_rsp_meta; + tcdm_req_t [1:0] stack_req; + tcdm_rsp_t [1:0] stack_rsp; + + for (genvar h = 0; h < 2; h++) begin : gen_host + // FPU's request only ever contends on the crossbar matching the current owner. + assign fpu_req_gated[h] = '{ + q : fp_lsu_mem_req.q, + q_valid: fp_lsu_mem_req.q_valid && (owner_id == h[0]), + p_ready: fp_lsu_mem_req.p_ready + }; + + assign core_req_chan [h][SnitchMst] = lock_out_req[h].q; + assign core_req_valid [h][SnitchMst] = lock_out_req[h].q_valid; + assign core_rsp_ready [h][SnitchMst] = lock_out_req[h].p_ready; + assign lock_out_rsp[h] = '{ + p : core_rsp_chan [h][SnitchMst], + p_valid: core_rsp_valid [h][SnitchMst], + q_ready: core_req_ready [h][SnitchMst] + }; + + assign core_req_chan [h][FpuMst] = fpu_req_gated[h].q; + assign core_req_valid [h][FpuMst] = fpu_req_gated[h].q_valid; + assign core_rsp_ready [h][FpuMst] = fpu_req_gated[h].p_ready; + assign fpu_rsp_per_host[h] = '{ + p : core_rsp_chan [h][FpuMst], + p_valid: core_rsp_valid [h][FpuMst], + q_ready: core_req_ready [h][FpuMst] + }; + + // Give SpmStack higher priority for winning an overlap; TotStack-range + // addresses share the CacheMem slot with MainMem (patched below). + always_comb begin + for (int i = 0; i < NrScalarXbarMst; i++) begin + core_req_sel[h][i] = Periph; + if ((tcdm_addr_base_i + cachepool_pkg::TCDMSize - cachepool_pkg::SpmStackSize <= core_req_chan[h][i].addr) + && (core_req_chan[h][i].addr < tcdm_addr_base_i + cachepool_pkg::TCDMSize)) begin + core_req_sel[h][i] = SpmStack; + end else if ((tcdm_addr_base_i <= core_req_chan[h][i].addr) + && (core_req_chan[h][i].addr < tcdm_addr_base_i + cachepool_pkg::TotStackSize)) begin + core_req_sel[h][i] = CacheMem; + end else if ((cachepool_pkg::DramAddr <= core_req_chan[h][i].addr) + && (core_req_chan[h][i].addr < cachepool_pkg::DramAddr + cachepool_pkg::DramSize)) begin + core_req_sel[h][i] = CacheMem; + end + end + end + + reqrsp_xbar #( + .NumInp (NrScalarXbarMst ), + .NumOut (NrScalarXbarSlv ), + .PipeReg (1'b0 ), + .ExtReqPrio (1'b0 ), + .ExtRspPrio (1'b0 ), + .tcdm_req_chan_t (dreq_chan_t ), + .tcdm_rsp_chan_t (drsp_chan_t ) + ) i_scalar_xbar ( + .clk_i (clk_i ), + .rst_ni (rst_ni ), + .slv_req_i (core_req_chan [h] ), + .slv_req_valid_i (core_req_valid [h] ), + .slv_req_ready_o (core_req_ready [h] ), + .slv_rsp_o (core_rsp_chan [h] ), + .slv_rsp_valid_o (core_rsp_valid [h] ), + .slv_rsp_ready_i (core_rsp_ready [h] ), + .slv_sel_i (core_req_sel [h] ), + .slv_rr_i ('0 ), + .slv_selected_o (core_selected [h] ), + .mst_req_o (mem_req_chan [h] ), + .mst_req_valid_o (mem_req_valid [h] ), + .mst_req_ready_i (mem_req_ready [h] ), + .mst_rsp_i (mem_rsp_chan [h] ), + .mst_rr_i ('0 ), + .mst_rsp_valid_i (mem_rsp_valid [h] ), + .mst_rsp_ready_o (mem_rsp_ready [h] ), + .mst_sel_i (mem_rsp_sel [h] ) + ); + + // TotStack hits get patched with host h's own id regardless of which + // master won, since the FPU only ever wins here when already gated to owner_id==h. + assign is_totstack[h] = (tcdm_addr_base_i <= mem_req_chan[h][CacheMem].addr) + && (mem_req_chan[h][CacheMem].addr < tcdm_addr_base_i + cachepool_pkg::TotStackSize); + + always_comb begin + for (int i = 0; i < NrScalarXbarSlv; i++) begin + mem_req[h][i].q = mem_req_chan [h][i]; + mem_req[h][i].q_valid = mem_req_valid[h][i]; + mem_req[h][i].p_ready = mem_rsp_ready[h][i]; + mem_req[h][i].q.user.is_fpu = (core_selected[h][i] == FpuMst); + end + if (is_totstack[h]) begin + mem_req[h][CacheMem].q.addr[($clog2(cachepool_pkg::TotStackSize)-1)-:$clog2(cachepool_pkg::NumCores)] = + hart_id_i[h][$clog2(cachepool_pkg::NumCores)-1:0]; + end + for (int i = 0; i < NrScalarXbarSlv; i++) begin + mem_rsp_chan [h][i] = mem_rsp[h][i].p; + mem_rsp_valid[h][i] = mem_rsp[h][i].p_valid; + mem_req_ready[h][i] = mem_rsp[h][i].q_ready; + mem_rsp_sel [h][i] = (mem_rsp[h][i].p.user.is_fpu == FpuMst); + end + end + + assign data_req_o[h] = mem_req[h][Periph]; + assign mem_rsp[h][Periph] = data_rsp_i[h]; + + // Host's own dedicated Stack SPM + reqrsp_to_tcdm #( + .AddrWidth (AddrWidth ), + .DataWidth (DataWidth ), + .BufDepth (4 ), + .UserWidth ($bits(tcdm_user_t) ), + .reqrsp_req_t (dreq_t ), + .reqrsp_rsp_t (drsp_t ), + .tcdm_req_t (tcdm_req_t ), + .tcdm_rsp_t (tcdm_rsp_t ) + ) i_core_to_stack ( + .clk_i (clk_i ), + .rst_ni (rst_ni ), + .reqrsp_req_i (mem_req[h][SpmStack] ), + .reqrsp_rsp_o (mem_rsp[h][SpmStack] ), + .tcdm_req_o (stack_req[h] ), + .tcdm_rsp_i (stack_rsp[h] ) + ); + + assign stack_valid[h] = stack_req[h].q_valid; + assign stack_we[h] = stack_req[h].q.write; + assign stack_add[h] = stack_req[h].q.addr[StackAddrWidth+1:2]; + assign stack_wdata[h] = stack_req[h].q.data; + assign stack_be[h] = stack_req[h].q.strb; + + assign stack_req_meta[h] = '{ + user: stack_req[h].q.user, + valid: stack_req[h].q_valid, + write: stack_req[h].q.write + }; + + assign stack_rsp[h].p.data = stack_rdata[h]; + assign stack_rsp[h].p.user = stack_rsp_meta[h].user; + assign stack_rsp[h].p.write = stack_rsp_meta[h].write; + assign stack_rsp[h].p_valid = stack_rsp_meta[h].valid; + assign stack_rsp[h].q_ready = 1'b1; + + tc_sram_impl #( + .NumWords (SpmStackDepth ), + .DataWidth (DataWidth ), + .ByteWidth (8 ), + .NumPorts (1 ), + .Latency (1 ), + .SimInit ("zeros" ) + ) i_spm_mem ( + .clk_i (clk_i ), + .rst_ni (rst_ni ), + .impl_i ('0 ), + .impl_o (/* Unused */ ), + .req_i (stack_valid[h] ), + .we_i (stack_we[h] ), + .addr_i (stack_add[h] ), + .wdata_i (stack_wdata[h] ), + .be_i (stack_be[h] ), + .rdata_o (stack_rdata[h] ) + ); + + shift_reg #( + .dtype (stack_meta_t ), + .Depth (1 ) + ) i_req_meta_pipe ( + .clk_i (clk_i ), + .rst_ni(rst_ni ), + .d_i (stack_req_meta[h] ), + .d_o (stack_rsp_meta[h] ) + ); + + // Host's own dedicated scalar TCDM port to the cache. + reqrsp_to_tcdm #( + .AddrWidth (AddrWidth ), + .DataWidth (DataWidth ), + .BufDepth (4 ), + .reqrsp_req_t (dreq_t ), + .reqrsp_rsp_t (drsp_t ), + .UserWidth ($bits(tcdm_user_t) ), + .tcdm_req_t (tcdm_req_t ), + .tcdm_rsp_t (tcdm_rsp_t ) + ) i_reqrsp_to_tcdm ( + .clk_i (clk_i ), + .rst_ni (rst_ni ), + .reqrsp_req_i (mem_req[h][CacheMem] ), + .reqrsp_rsp_o (mem_rsp[h][CacheMem] ), + .tcdm_req_o (tcdm_req_o[NumMemPortsPerSpatz+h] ), + .tcdm_rsp_i (tcdm_rsp_i[NumMemPortsPerSpatz+h] ) + ); + // Scalar port: left on its existing unconditional-accept behavior for now. + assign tcdm_rsp_ready_o[NumMemPortsPerSpatz+h] = 1'b1; + end + +endmodule diff --git a/hardware/src/cachepool_cluster.sv b/hardware/src/cachepool_cluster.sv index ee777efc..ee6c9c36 100644 --- a/hardware/src/cachepool_cluster.sv +++ b/hardware/src/cachepool_cluster.sv @@ -31,14 +31,14 @@ module cachepool_cluster parameter logic [31:0] BootAddr = 32'h0, /// Address to indicate start of UART parameter logic [31:0] UartAddr = 32'h0, - /// The total amount of cores. - parameter int unsigned NrCores = 8, + /// Number of Core Complex (CC) slots in this cluster. + parameter int unsigned NumCC = 8, /// Data/TCDM memory depth per cut (in words). parameter int unsigned TCDMDepth = 1024, /// Cluster peripheral address region size (in kB). parameter int unsigned ClusterPeriphSize = 64, /// Number of TCDM Banks. - parameter int unsigned NrBanks = 2 * NrCores, + parameter int unsigned NrBanks = 2 * NumCC, /// Width of a single icache line. parameter unsigned ICacheLineWidth = 0, /// Number of icache lines per set. @@ -91,11 +91,13 @@ module cachepool_cluster /*** ATTENTION: `NrSramCfg` should be changed if `L1NumDataBank` and `L1NumTagBank` is changed ***/ parameter int unsigned NrSramCfg = 1, /// Folded data bank configuration (0 = auto: min(4, L1AssoPerCtrl)). - parameter bit UseFoldedDataBanks = 1'b1, - parameter int unsigned FoldWayGroup = 0, - parameter bit UseHashWaySelect = 1'b1, + parameter bit UseFoldedDataBanks = 1'b1, + parameter int unsigned FoldWayGroup = 0, + parameter bit UseHashWaySelect = 1'b1, /// Enable the SRAM forwarding buffer (default on; requires UseHashWaySelect). - parameter bit UseForwardingBuffer = 1'b1 + parameter bit UseForwardingBuffer = 1'b1, + /// First hartid of the cluster; cores get hartids HartBaseId..HartBaseId+NumCC-1. + parameter logic [9:0] HartBaseId = 10'h0 ) ( /// System clock. input logic clk_i, @@ -117,10 +119,6 @@ module cachepool_cluster /// another core to facilitate inter-processor-interrupts. This signal is /// assumed to be _async_. input logic msip_i, - /// First hartid of the cluster. Cores of a cluster are monotonically - /// increasing without a gap, i.e., a cluster with 8 cores and a - /// `hart_base_id_i` of 5 get the hartids 5 - 12. - input logic [9:0] hart_base_id_i, /// Base address of cluster. TCDM and cluster peripheral location are derived from /// it. This signal is pseudo-static. input logic [AxiAddrWidth-1:0] cluster_base_addr_i, @@ -254,7 +252,7 @@ module cachepool_cluster .BootAddr ( BootAddr ), .UartAddr ( UartAddr ), .ClusterPeriphSize ( ClusterPeriphSize ), - .NrCores ( NumCoreGroup ), + .NumCC ( NumCoreGroup ), .TCDMDepth ( TCDMDepth ), .NrBanks ( NrBanks / NumGroups ), .ICacheLineWidth ( ICacheLineWidth ), @@ -290,7 +288,7 @@ module cachepool_cluster .meip_i ( meip_i ), .mtip_i ( mtip_i ), .msip_i ( msip_i ), - .hart_base_id_i ( hart_base_id_i + 10'(g * NumCoreGroup) ), + .hart_base_id_i ( HartBaseId + 10'(g * NumCoreGroup * NumScalarPerCC) ), .tile_base_id_i ( TileIDWidth'(g * NumTilesPerGroup) ), .cluster_base_addr_i ( cluster_base_addr_i ), .private_start_addr_i ( private_start_addr ), @@ -312,7 +310,6 @@ module cachepool_cluster // Peripherals .icache_events_o ( /* unused */ ), .icache_prefetch_enable_i ( icache_prefetch_enable ), - .cl_interrupt_i ( '0 ), .dynamic_offset_i ( dynamic_offset ), .l1d_private_i ( l1d_private ), .l1d_insn_i ( l1d_insn ), @@ -1264,7 +1261,7 @@ module cachepool_cluster .tcdm_start_address_i ( tcdm_start_address ), .tcdm_end_address_i ( tcdm_end_address ), .icache_prefetch_enable_o ( icache_prefetch_enable ), - .cluster_hart_base_id_i ( hart_base_id_i ), + .cluster_hart_base_id_i ( HartBaseId ), .cluster_probe_o ( cluster_probe_o ), .dynamic_offset_o ( dynamic_offset ), .private_start_addr_o ( private_start_addr ), diff --git a/hardware/src/cachepool_group.sv b/hardware/src/cachepool_group.sv index 9f9cdd47..4b6f737a 100644 --- a/hardware/src/cachepool_group.sv +++ b/hardware/src/cachepool_group.sv @@ -30,14 +30,14 @@ module cachepool_group parameter logic [31:0] BootAddr = 32'h0, /// Address to indicate start of UART parameter logic [31:0] UartAddr = 32'h0, - /// The total amount of cores. - parameter int unsigned NrCores = 8, + /// Number of Core Complex (CC) slots in this group. + parameter int unsigned NumCC = 8, /// Data/TCDM memory depth per cut (in words). parameter int unsigned TCDMDepth = 1024, /// Cluster peripheral address region size (in kB). parameter int unsigned ClusterPeriphSize = 64, /// Number of TCDM Banks. - parameter int unsigned NrBanks = 2 * NrCores, + parameter int unsigned NrBanks = 2 * NumCC, /// Width of a single icache line. parameter unsigned ICacheLineWidth = 0, /// Number of icache lines per set. @@ -144,9 +144,8 @@ module cachepool_group input floo_cachepool_noc_pkg::id_t l2_group_id_i, /// Peripheral signals - output icache_l1_events_t [NrCores-1:0] icache_events_o, + output icache_l1_events_t [NumCC-1:0] icache_events_o, input logic icache_prefetch_enable_i, - input logic [NrCores-1:0] cl_interrupt_i, input logic [$clog2(AxiAddrWidth)-1:0] dynamic_offset_i, input logic [$clog2(NumL1CtrlTile):0] l1d_private_i, input cache_insn_t l1d_insn_i, @@ -187,7 +186,7 @@ module cachepool_group // Constants // --------- // Per-group overrides of package-level constants that depend on NumTiles/NumCores. - localparam int unsigned NumL1CacheCtrlLocal = NrCores; + localparam int unsigned NumL1CacheCtrlLocal = NumCC; localparam int unsigned WideIdWidthIn = AxiIdWidthOut; @@ -547,7 +546,7 @@ module cachepool_group for (genvar t = 0; t < NumTilesPerGroup; t ++) begin : gen_tiles logic [9:0] hart_base_id; - assign hart_base_id = hart_base_id_i + t * NumCoresTile; + assign hart_base_id = hart_base_id_i + t * NumCoresTile * NumScalarPerCC; logic [TileIDWidth-1:0] tile_id; assign tile_id = tile_base_id_i + TileIDWidth'(t); @@ -562,7 +561,7 @@ module cachepool_group .BootAddr ( BootAddr ), .UartAddr ( UartAddr ), .ClusterPeriphSize ( ClusterPeriphSize ), - .NrCores ( NumCoresTile ), + .NumCC ( NumCoresTile ), .TCDMDepth ( TCDMDepth ), .NrBanks ( NrBanks ), .ICacheLineWidth ( ICacheLineWidth ), @@ -635,7 +634,6 @@ module cachepool_group // Peripherals .icache_events_o ( /* unused */ ), .icache_prefetch_enable_i ( icache_prefetch_enable_i ), - .cl_interrupt_i ( cl_interrupt_i [t*NumCoresTile+:NumCoresTile] ), .dynamic_offset_i ( dynamic_offset_i ), .l1d_insn_i ( l1d_insn_i ), .l1d_private_i ( l1d_private_i ), @@ -653,7 +651,7 @@ module cachepool_group .BootAddr ( BootAddr ), .UartAddr ( UartAddr ), .ClusterPeriphSize ( ClusterPeriphSize ), - .NrCores ( NumCoresTile ), + .NumCC ( NumCoresTile ), .TCDMDepth ( TCDMDepth ), .NrBanks ( NrBanks ), .ICacheLineWidth ( ICacheLineWidth ), @@ -726,7 +724,6 @@ module cachepool_group // Peripherals .icache_events_o ( /* unused */ ), .icache_prefetch_enable_i ( icache_prefetch_enable_i ), - .cl_interrupt_i ( cl_interrupt_i [t*NumCoresTile+:NumCoresTile] ), .dynamic_offset_i ( dynamic_offset_i ), .l1d_insn_i ( l1d_insn_i ), .l1d_private_i ( l1d_private_i ), diff --git a/hardware/src/cachepool_group_noc_wrapper.sv b/hardware/src/cachepool_group_noc_wrapper.sv index 0950eed9..b80daa21 100644 --- a/hardware/src/cachepool_group_noc_wrapper.sv +++ b/hardware/src/cachepool_group_noc_wrapper.sv @@ -27,10 +27,10 @@ module cachepool_group_noc_wrapper parameter int unsigned AxiUserWidth = 1, parameter logic [31:0] BootAddr = 32'h0, parameter logic [31:0] UartAddr = 32'h0, - parameter int unsigned NrCores = 0, + parameter int unsigned NumCC = 0, parameter int unsigned TCDMDepth = 1024, parameter int unsigned ClusterPeriphSize = 64, - parameter int unsigned NrBanks = 2 * NrCores, + parameter int unsigned NrBanks = 2 * NumCC, parameter int unsigned ICacheLineWidth = 0, parameter int unsigned ICacheLineCount = 0, parameter int unsigned ICacheSets = 0, @@ -90,9 +90,8 @@ module cachepool_group_noc_wrapper // L2 mesh: endpoint ID and source-routing table from floogen input floo_cachepool_noc_pkg::id_t l2_id_i, input floo_cachepool_noc_pkg::route_t [floo_cachepool_noc_pkg::RouteCfg.NumRoutes-1:0] l2_route_table_i, - output icache_l1_events_t [NrCores-1:0] icache_events_o, + output icache_l1_events_t [NumCC-1:0] icache_events_o, input logic icache_prefetch_enable_i, - input logic [NrCores-1:0] cl_interrupt_i, input logic [$clog2(AxiAddrWidth)-1:0] dynamic_offset_i, input logic [$clog2(NumL1CtrlTile):0] l1d_private_i, input cache_insn_t l1d_insn_i, @@ -742,7 +741,7 @@ module cachepool_group_noc_wrapper .BootAddr ( BootAddr ), .UartAddr ( UartAddr ), .ClusterPeriphSize ( ClusterPeriphSize ), - .NrCores ( NrCores ), + .NumCC ( NumCC ), .TCDMDepth ( TCDMDepth ), .NrBanks ( NrBanks ), .ICacheLineWidth ( ICacheLineWidth ), @@ -793,7 +792,6 @@ module cachepool_group_noc_wrapper .barrier_done_i ( barrier_done_q ), .icache_events_o ( icache_events_o ), .icache_prefetch_enable_i ( icache_prefetch_enable_i ), - .cl_interrupt_i ( cl_interrupt_i ), .dynamic_offset_i ( dynamic_offset_q ), .l1d_private_i ( l1d_private_q ), .l1d_insn_i ( l1d_insn_q ), diff --git a/hardware/src/cachepool_pkg.sv b/hardware/src/cachepool_pkg.sv index a7b38cd4..6af961ad 100644 --- a/hardware/src/cachepool_pkg.sv +++ b/hardware/src/cachepool_pkg.sv @@ -26,7 +26,8 @@ package cachepool_pkg; ////////////////// // GLOBAL HW // ////////////////// - localparam int unsigned NumCores = `ifdef NUM_CORES `NUM_CORES `else 0 `endif; + // Core Complex slot count (one Spatz per slot, shared by NumScalarPerCC harts). + localparam int unsigned NumCC = `ifdef NUM_CORES `NUM_CORES `else 0 `endif; localparam int unsigned NumTiles = `ifdef NUM_TILES `NUM_TILES `else 0 `endif; // TODO: not yet passed in through config, hardcode to 1 localparam int unsigned NumGroups = `ifdef NUM_GROUPS `NUM_GROUPS `else 1 `endif; @@ -42,6 +43,14 @@ package cachepool_pkg; localparam int unsigned NFpu = `ifdef SPATZ_NUM_FPU `SPATZ_NUM_FPU `else 0 `endif; localparam int unsigned NIpu = `ifdef SPATZ_NUM_IPU `SPATZ_NUM_IPU `else 1 `endif; + // Snitch scalar cores sharing one Spatz per CC (1 = cachepool_cc, 2 = cachepool_cc_dual). + // Whole-build choice, not per-tile; default 1 until config.mk wires a real value through. + localparam int unsigned NumScalarPerCC = `ifdef NUM_SCALAR_PER_CC `NUM_SCALAR_PER_CC `else 1 `endif; + + // Total hart count (NumCC CC-slots x NumScalarPerCC harts each) -- use for + // per-hart addressing; use NumCC for CC/Spatz-slot units (tile/group size). + localparam int unsigned NumCores = NumCC * NumScalarPerCC; + localparam int unsigned NumIntOutstandingLoads = `ifdef SNITCH_MAX_TRANS `SNITCH_MAX_TRANS `else 0 `endif; localparam int unsigned NumIntOutstandingMem = `ifdef SNITCH_MAX_TRANS `SNITCH_MAX_TRANS `else 0 `endif; localparam int unsigned NumSpatzOutstandingLoads = `ifdef SPATZ_MAX_TRANS `SPATZ_MAX_TRANS `else 0 `endif; @@ -52,7 +61,7 @@ package cachepool_pkg; // TILE CONFIG // /////////////////// // How many cores for each tile? - localparam int unsigned NumCoresTile = NumCores / NumTiles; + localparam int unsigned NumCoresTile = NumCC / NumTiles; // Intra-group remote ports per core (to other tiles in the same group). localparam int unsigned NumLGPortCore = `ifdef LG_PORT_PER_CORE `LG_PORT_PER_CORE `else 0 `endif; @@ -60,8 +69,9 @@ package cachepool_pkg; // How many cores within a tile? This is used to select the ports within a tile. localparam int unsigned LogNumCoresTile = $clog2(NumCoresTile); - // 4 ports from Spatz + 1 shared port from Snitch/FPU - localparam int unsigned NrTCDMPortsPerCore = 5; + // Spatz vector-FU ports (shared, one CC-slot's worth) + one dedicated scalar + // port per hart sharing that Spatz. + localparam int unsigned NrTCDMPortsPerCore = (NFpu > NIpu ? NFpu : NIpu) + NumScalarPerCC; // Intra-group remote ports per tile, in total. localparam int unsigned NumLGPortTile = NumLGPortCore * NrTCDMPortsPerCore; @@ -73,7 +83,7 @@ package cachepool_pkg; localparam int unsigned NumTilesPerGroup = NumTiles / NumGroups; // How many cores for each group? - localparam int unsigned NumCoreGroup = NumCores / NumGroups; + localparam int unsigned NumCoreGroup = NumCC / NumGroups; // How many remote group ports for each tile? localparam int unsigned NumRemoteGroupPortCore = `ifdef RG_PORT_PER_CORE `RG_PORT_PER_CORE `else 0 `endif; @@ -144,7 +154,7 @@ package cachepool_pkg; localparam int unsigned NumBank = `ifdef L1D_NUM_BANKS `L1D_NUM_BANKS `else 0 `endif; // NOTE: these are used by AXI/L2 as well, keep here but ordered as "cluster-level cache topology" - localparam int unsigned NumL1CacheCtrl = NumCores; + localparam int unsigned NumL1CacheCtrl = NumCC; localparam int unsigned NumL1CtrlTile = NumL1CacheCtrl / NumTiles; // Number of data banks assigned to each cache controller diff --git a/hardware/src/cachepool_spatz_lock.sv b/hardware/src/cachepool_spatz_lock.sv new file mode 100644 index 00000000..3899f405 --- /dev/null +++ b/hardware/src/cachepool_spatz_lock.sv @@ -0,0 +1,434 @@ +// Copyright 2025 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 + +// Author: Diyou Shen + +`include "common_cells/assertions.svh" +`include "common_cells/registers.svh" + +/// Spatz ownership lock/switch for a dual-Snitch cachepool_cc_dual. Two +/// dedicated addresses (ACQUIRE/RELEASE) are intercepted as loads that +/// always complete immediately, encoding the outcome and current +/// owner/locked status in the returned word (see note.md). Pure arbiter: +/// does not route acc/data traffic itself (see acc_mux.sv); only decides +/// ownership and counts real Spatz acc handshakes to gate a switch until +/// fully drained. Also hosts the pass-through memory-path register cuts +/// (moved here from the per-core spill registers in cachepool_cc.sv). +module cachepool_spatz_lock + import cachepool_peripheral_reg_pkg::*; +#( + parameter int unsigned AddrWidth = 0, + parameter int unsigned NumHosts = 2, + parameter type dreq_t = logic, + parameter type drsp_t = logic, + parameter type dreq_chan_t = logic, + parameter type drsp_chan_t = logic, + parameter type user_t = logic, + parameter bit RegisterCoreReq = 1'b0, + parameter bit RegisterCoreRsp = 1'b0, + /// Derived parameter *Do not override* + parameter type addr_t = logic [AddrWidth-1:0] +) ( + input logic clk_i, + input logic rst_ni, + + // memory interface, used to attempt acquire/release + input dreq_t [NumHosts-1:0] in_req_i, + output drsp_t [NumHosts-1:0] in_rsp_o, + output dreq_t [NumHosts-1:0] out_req_o, + input drsp_t [NumHosts-1:0] out_rsp_i, + + // Real Spatz acc handshake fires, fed back by acc_mux purely for drain counting. + input logic req_fire_i, + input logic rsp_fire_i, + + // Spatz LSU drain tracking: vle/vse have no acc_rsp_t completion, so they need their own count. + input logic spatz_lsu_issue_fire_i, + input logic [1:0] spatz_mem_finished_i, + input logic spatz_st_rsp_done_i, + + // owner/lock status + output logic owner_id_o, + output logic locked_o, + output logic waiting_o, + // True during Locked/RelWait; lets acc_mux keep draining through RelWait after new issues stop. + output logic owner_active_o, + + // peripheral base address + input addr_t cluster_periph_start_address_i +); + + addr_t acquire_addr, release_addr; + assign acquire_addr = cluster_periph_start_address_i + CACHEPOOL_PERIPHERAL_SPATZ_LOCK_ACQUIRE_OFFSET; + assign release_addr = cluster_periph_start_address_i + CACHEPOOL_PERIPHERAL_SPATZ_LOCK_RELEASE_OFFSET; + + // Single lock FSM (host 0 is the implicit owner while Free). AcqWait/RelWait + // are drain-wait sub-states, resolved autonomously once drain_done -- no + // response is owed then, since the triggering request already got SUCCESS-WAIT. + // Free -(acquire, other host)-> AcqWait -(drain_done)-> Locked (new owner) + // Locked -(release, owner) -> RelWait -(drain_done)-> Free (owner = host 0) + // Every hit completes immediately with an outcome (FAIL/SUCCESS/SUCCESS-WAIT) + // instead of stalling; a hit during AcqWait/RelWait always gets FAIL(PENDING). + // Handshake timing: q_ready fires on the deciding cycle; p_valid is held + // starting the next cycle until the host takes it via p_ready. + typedef enum logic [1:0] { + Free, + AcqWait, + RelWait, + Locked + } lock_state_e; + + // Response payload bit layout (see note.md): [1:0]=outcome, [4:2]=reason + // (valid only on FAIL), [5]=owner, [6]=locked -- all post-decision. + typedef enum logic [1:0] { + OutcomeFail = 2'd0, + OutcomeSuccess = 2'd1, + OutcomeSuccessWait = 2'd2 + } lock_outcome_e; + + typedef enum logic [2:0] { + ReasonNotOwner = 3'd0, + ReasonBusy = 3'd1, + ReasonPending = 3'd2 + } lock_fail_reason_e; + + lock_state_e lock_d, lock_q; + logic owner_d, owner_q; + logic active_d, active_q; + + // Outstanding acc handshakes on the owner's path; must reach 0 before a wait can complete. + logic [7:0] outstanding_d, outstanding_q; + // Outstanding Spatz vle/vse ops; mirrors snitch.sv's own acc_mem_cnt_q. + logic [7:0] lsu_outstanding_d, lsu_outstanding_q; + logic drain_done; + + assign owner_id_o = owner_q; + assign locked_o = (lock_q == Locked); + assign waiting_o = (lock_q == AcqWait) || (lock_q == RelWait); + assign owner_active_o = (lock_q == Locked) || (lock_q == RelWait); + + assign drain_done = (outstanding_q == '0) && (lsu_outstanding_q == '0) && spatz_st_rsp_done_i; + + `ASSERT(NoOutstandingUnderflow, rsp_fire_i |-> (outstanding_q != '0)) + // Checked against the net balance for this cycle, not the pre-update register alone: + // a scalar FP store's finish can land in the same cycle as its own issue. + `ASSERT(NoLsuOutstandingUnderflow, + (spatz_mem_finished_i[0] || spatz_mem_finished_i[1]) |-> + (lsu_outstanding_q + spatz_lsu_issue_fire_i >= (spatz_mem_finished_i[0] + spatz_mem_finished_i[1]))) + + always_comb begin + outstanding_d = outstanding_q; + if (req_fire_i && !rsp_fire_i) begin + // one insn issued + outstanding_d = outstanding_q + 8'd1; + end else if (!req_fire_i && rsp_fire_i) begin + // one insn finished + outstanding_d = outstanding_q - 8'd1; + end + end + + always_comb begin + lsu_outstanding_d = lsu_outstanding_q; + if (spatz_lsu_issue_fire_i) begin + lsu_outstanding_d = lsu_outstanding_d + 8'd1; + end + if (spatz_mem_finished_i[0]) begin + lsu_outstanding_d = lsu_outstanding_d - 8'd1; + end + if (spatz_mem_finished_i[1]) begin + lsu_outstanding_d = lsu_outstanding_d - 8'd1; + end + end + + // Address-based op decode: a hit on either address is serviced regardless + // of q.write, since only loads carry a usable result back to software. + logic [1:0] acquire_hit, release_hit, lock_hit; + + for (genvar i = 0; i < 2; i++) begin + assign acquire_hit[i] = in_req_i[i].q_valid && (in_req_i[i].q.addr == acquire_addr); + assign release_hit[i] = in_req_i[i].q_valid && (in_req_i[i].q.addr == release_addr); + assign lock_hit[i] = acquire_hit[i] || release_hit[i]; + end + + // Owner's hit wins on a same-cycle tie; the loser just waits one cycle for resp_pending_q to clear. + logic hit_any, winner; + assign hit_any = lock_hit[0] || lock_hit[1]; + assign winner = lock_hit[owner_q] ? owner_q : ~owner_q; + + // Pending, not-yet-delivered completion response (one at a time; a new lock op is + // only arbitrated once the previous one's response has been taken). + logic resp_pending_d, resp_pending_q; + logic resp_host_d, resp_host_q; + user_t resp_user_d, resp_user_q; + lock_outcome_e resp_outcome_d, resp_outcome_q; + lock_fail_reason_e resp_reason_d, resp_reason_q; + logic resp_owner_d, resp_owner_q; + logic resp_locked_d, resp_locked_q; + + // Register cuts on the pass-through path (moved here from cachepool_cc.sv's per-core + // spill registers), placed at the module's output side. + logic [1:0] mem_req_valid, mem_req_ready, mem_rsp_valid; + drsp_chan_t [1:0] mem_rsp_chan; + + for (genvar i = 0; i < 2; i++) begin : gen_out_cut + assign mem_req_valid[i] = in_req_i[i].q_valid && !lock_hit[i]; + + spill_register #( + .T ( dreq_chan_t ), + .Bypass ( !RegisterCoreReq ) + ) i_spill_register_req ( + .clk_i, + .rst_ni, + .valid_i ( mem_req_valid[i] ), + .ready_o ( mem_req_ready[i] ), + .data_i ( in_req_i[i].q ), + .valid_o ( out_req_o[i].q_valid ), + .ready_i ( out_rsp_i[i].q_ready ), + .data_o ( out_req_o[i].q ) + ); + + spill_register #( + .T ( drsp_chan_t ), + .Bypass ( !RegisterCoreRsp ) + ) i_spill_register_rsp ( + .clk_i, + .rst_ni, + .valid_i ( out_rsp_i[i].p_valid ), + .ready_o ( out_req_o[i].p_ready ), + .data_i ( out_rsp_i[i].p ), + .valid_o ( mem_rsp_valid[i] ), + .ready_i ( in_req_i[i].p_ready ), + .data_o ( mem_rsp_chan[i] ) + ); + end + + always_comb begin + lock_d = lock_q; + owner_d = owner_q; + active_d = active_q; + resp_pending_d = resp_pending_q; + resp_host_d = resp_host_q; + resp_user_d = resp_user_q; + resp_outcome_d = resp_outcome_q; + resp_reason_d = resp_reason_q; + resp_owner_d = resp_owner_q; + resp_locked_d = resp_locked_q; + + // Default: pass through the (registered) memory path; blocked on a lock-address hit. + for (int i = 0; i < 2; i++) begin + in_rsp_o[i] = '0; + in_rsp_o[i].q_ready = lock_hit[i] ? 1'b0 : mem_req_ready[i]; + in_rsp_o[i].p_valid = mem_rsp_valid[i]; + in_rsp_o[i].p = mem_rsp_chan[i]; + end + + // Deliver a pending completion response, starting the cycle after it was accepted. + if (resp_pending_q) begin + in_rsp_o[resp_host_q] = '0; + in_rsp_o[resp_host_q].p_valid = 1'b1; + in_rsp_o[resp_host_q].p.user = resp_user_q; + in_rsp_o[resp_host_q].p.data[1:0] = resp_outcome_q; + in_rsp_o[resp_host_q].p.data[4:2] = resp_reason_q; + in_rsp_o[resp_host_q].p.data[5] = resp_owner_q; + in_rsp_o[resp_host_q].p.data[6] = resp_locked_q; + if (in_req_i[resp_host_q].p_ready) resp_pending_d = 1'b0; + end + + case (lock_q) + // Host 0 is the implicit owner. Its own release is a no-op (immediate grant); + // an acquire from either host grants immediately once drained, else waits. + Free: begin + if (hit_any && !resp_pending_q) begin + in_rsp_o[winner].q_ready = 1'b1; + resp_pending_d = 1'b1; + resp_host_d = winner; + resp_user_d = in_req_i[winner].q.user; + + if (acquire_hit[winner]) begin + if (drain_done) begin + lock_d = Locked; + owner_d = winner; + resp_outcome_d = OutcomeSuccess; + resp_owner_d = winner; + resp_locked_d = 1'b1; + end else begin + active_d = winner; + lock_d = AcqWait; + resp_outcome_d = OutcomeSuccessWait; + resp_owner_d = owner_q; + resp_locked_d = 1'b0; + end + end else begin + // release_hit[winner] + if (winner == owner_q) begin + // host 0 "self release": nothing to release, immediate no-op grant + resp_outcome_d = OutcomeSuccess; + resp_owner_d = owner_q; + resp_locked_d = 1'b0; + end else begin + // host 1 release: never held it + resp_outcome_d = OutcomeFail; + resp_reason_d = ReasonNotOwner; + resp_owner_d = owner_q; + resp_locked_d = 1'b0; + end + end + end + end + + // Non-owner acquire/release always fails outright, no queueing for a future handoff. + Locked: begin + if (hit_any && !resp_pending_q) begin + in_rsp_o[winner].q_ready = 1'b1; + resp_pending_d = 1'b1; + resp_host_d = winner; + resp_user_d = in_req_i[winner].q.user; + + if (acquire_hit[winner]) begin + resp_outcome_d = (winner == owner_q) ? OutcomeSuccess : OutcomeFail; + resp_reason_d = ReasonBusy; + resp_owner_d = owner_q; + resp_locked_d = 1'b1; + end else begin + // release_hit[winner] + if (winner == owner_q) begin + if (drain_done) begin + lock_d = Free; + owner_d = 1'b0; + resp_outcome_d = OutcomeSuccess; + resp_owner_d = 1'b0; + resp_locked_d = 1'b0; + end else begin + active_d = winner; + lock_d = RelWait; + resp_outcome_d = OutcomeSuccessWait; + resp_owner_d = owner_q; + resp_locked_d = 1'b1; + end + end else begin + resp_outcome_d = OutcomeFail; + resp_reason_d = ReasonNotOwner; + resp_owner_d = owner_q; + resp_locked_d = 1'b1; + end + end + end + end + + // Any hit here fails with PENDING; active_q's transition resolves on its own, no response owed. + AcqWait: begin + if (hit_any && !resp_pending_q) begin + in_rsp_o[winner].q_ready = 1'b1; + resp_pending_d = 1'b1; + resp_host_d = winner; + resp_user_d = in_req_i[winner].q.user; + resp_outcome_d = OutcomeFail; + resp_reason_d = ReasonPending; + resp_owner_d = owner_q; + resp_locked_d = 1'b0; + end + if (drain_done) begin + owner_d = active_q; + lock_d = Locked; + end + end + + RelWait: begin + if (hit_any && !resp_pending_q) begin + in_rsp_o[winner].q_ready = 1'b1; + resp_pending_d = 1'b1; + resp_host_d = winner; + resp_user_d = in_req_i[winner].q.user; + resp_outcome_d = OutcomeFail; + resp_reason_d = ReasonPending; + resp_owner_d = owner_q; + resp_locked_d = 1'b1; + end + if (drain_done) begin + owner_d = 1'b0; + lock_d = Free; + end + end + + default: lock_d = Free; + endcase + end + + `FF(lock_q, lock_d, Free, clk_i, rst_ni) + `FF(owner_q, owner_d, 1'b0, clk_i, rst_ni) + `FF(active_q, active_d, 1'b0, clk_i, rst_ni) + `FF(outstanding_q, outstanding_d, '0, clk_i, rst_ni) + `FF(lsu_outstanding_q, lsu_outstanding_d, '0, clk_i, rst_ni) + `FF(resp_pending_q, resp_pending_d, 1'b0, clk_i, rst_ni) + `FF(resp_host_q, resp_host_d, 1'b0, clk_i, rst_ni) + `FF(resp_user_q, resp_user_d, '0, clk_i, rst_ni) + `FF(resp_outcome_q, resp_outcome_d, OutcomeFail, clk_i, rst_ni) + `FF(resp_reason_q, resp_reason_d, ReasonNotOwner,clk_i, rst_ni) + `FF(resp_owner_q, resp_owner_d, 1'b0, clk_i, rst_ni) + `FF(resp_locked_q, resp_locked_d, 1'b0, clk_i, rst_ni) + +`ifndef TARGET_SYNTHESIS + // Debug-only: verbose state-transition/response log (+spatz_lock_verbose + // plusarg) and a stuck-in-wait watchdog. Added to debug the FPU-enabled + // dual-CC boot hang; temporary instrumentation, safe to strip once resolved. + bit spatz_lock_verbose = 1'b0; + initial begin + // verilog_lint: waive plusarg-assignment + spatz_lock_verbose = $test$plusargs("spatz_lock_verbose"); + end + + always_ff @(posedge clk_i) begin + if (rst_ni && spatz_lock_verbose) begin + if (lock_d != lock_q) begin + $display({"[SPATZ-LOCK %0t %m] STATE %s -> %s owner_d=%0d active_d=%0d ", + "outstanding_q=%0d lsu_outstanding_q=%0d"}, + $time, lock_q.name(), lock_d.name(), owner_d, active_d, + outstanding_q, lsu_outstanding_q); + end + if (hit_any && !resp_pending_q) begin + $display({"[SPATZ-LOCK %0t %m] HIT host=%0d acquire=%0b release=%0b ", + "state=%s outstanding_q=%0d lsu_outstanding_q=%0d ", + "spatz_st_rsp_done=%0b"}, + $time, winner, acquire_hit[winner], release_hit[winner], + lock_q.name(), outstanding_q, lsu_outstanding_q, spatz_st_rsp_done_i); + end + if (resp_pending_d && !resp_pending_q) begin + $display({"[SPATZ-LOCK %0t %m] RESP host=%0d outcome=%s reason=%s owner=%0d ", + "locked=%0b"}, + $time, resp_host_d, resp_outcome_d.name(), resp_reason_d.name(), + resp_owner_d, resp_locked_d); + end + end + end + + // Watchdog: warn if the FSM has been stuck in AcqWait/RelWait (waiting on + // drain_done) for more than SpatzLockWdogPs, re-warning every + // SpatzLockWdogPs while still stuck. + localparam longint unsigned SpatzLockWdogPs = 5_000_000; + logic [63:0] spatz_lock_last_progress_q, spatz_lock_last_warn_q; + + always_ff @(posedge clk_i or negedge rst_ni) begin + if (!rst_ni) begin + spatz_lock_last_progress_q <= '0; + spatz_lock_last_warn_q <= '0; + end else begin + if (lock_q != AcqWait && lock_q != RelWait) begin + spatz_lock_last_progress_q <= 64'($time); + spatz_lock_last_warn_q <= 64'($time); + end + if ((lock_q == AcqWait || lock_q == RelWait) && + (64'($time) - spatz_lock_last_progress_q) > SpatzLockWdogPs && + (64'($time) - spatz_lock_last_warn_q) > SpatzLockWdogPs) begin + spatz_lock_last_warn_q <= 64'($time); + $display({"[%0t] [SPATZ-LOCK %m] STUCK: state=%s active_q=%0d owner_q=%0d ", + "outstanding_q=%0d lsu_outstanding_q=%0d spatz_st_rsp_done=%0b ", + "drain_done=%0b"}, + $time, lock_q.name(), active_q, owner_q, outstanding_q, + lsu_outstanding_q, spatz_st_rsp_done_i, drain_done); + end + end + end +`endif // TARGET_SYNTHESIS + +endmodule diff --git a/hardware/src/cachepool_tile.sv b/hardware/src/cachepool_tile.sv index 9defdd60..19c02513 100644 --- a/hardware/src/cachepool_tile.sv +++ b/hardware/src/cachepool_tile.sv @@ -32,14 +32,18 @@ module cachepool_tile parameter logic [31:0] BootAddr = 32'h0, /// Address to indicate start of UART parameter logic [31:0] UartAddr = 32'h0, - /// The total amount of cores. - parameter int unsigned NrCores = 8, + /// Number of Core Complex (CC) slots in this tile. + parameter int unsigned NumCC = 8, + /// Number of Snitch scalar cores sharing one Spatz per CC (1 = today's + /// cachepool_cc, 2 = cachepool_cc_dual). Whole-build choice; defaults to + /// the package-level constant (single source of truth), not per-tile. + parameter int unsigned NumScalarPerCC = cachepool_pkg::NumScalarPerCC, /// Data/TCDM memory depth per cut (in words). parameter int unsigned TCDMDepth = 1024, /// Cluster peripheral address region size (in kB). parameter int unsigned ClusterPeriphSize = 64, /// Number of TCDM Banks. - parameter int unsigned NrBanks = 2 * NrCores, + parameter int unsigned NrBanks = 2 * NumCC, /// Width of a single icache line. parameter unsigned ICacheLineWidth = 0, /// Number of icache lines per set. @@ -110,7 +114,9 @@ module cachepool_tile parameter bit UseHashWaySelect = 1'b0, /// Enable the SRAM forwarding buffer (default on; requires UseHashWaySelect). parameter bit UseForwardingBuffer = 1'b1, - localparam int unsigned TotRGPorts = (NumRemoteGroupPortCore == 0) ? 0 : NumRemoteGroupPortCore*NrTCDMPortsPerCore-1 + localparam int unsigned TotRGPorts = (NumRemoteGroupPortCore == 0) ? 0 : NumRemoteGroupPortCore*NrTCDMPortsPerCore-1, + /// Derived parameter *Do not override*: true hart count (NumCC = CC-slot count). + localparam int unsigned NrHarts = NumCC * NumScalarPerCC ) ( /// System clock. input logic clk_i, @@ -170,9 +176,8 @@ module cachepool_tile input remote_group_req_t [TotRGPorts:0] remote_group_req_i, output remote_group_rsp_t [TotRGPorts:0] remote_group_rsp_o, /// Peripheral signals - output icache_l1_events_t [NrCores-1:0] icache_events_o, + output icache_l1_events_t [NrHarts-1:0] icache_events_o, input logic icache_prefetch_enable_i, - input logic [NrCores-1:0] cl_interrupt_i, input logic [$clog2(AxiAddrWidth)-1:0] dynamic_offset_i, input cache_insn_t l1d_insn_i, input logic [$clog2(NumL1CtrlTile):0] l1d_private_i, @@ -204,7 +209,7 @@ module cachepool_tile assign num_private_cache = l1d_private_i [$clog2(NumL1CtrlTile):0]; /// Minimum width to hold the core number. - // localparam int unsigned CoreIDWidth = cf_math_pkg::idx_width(NrCores); + // localparam int unsigned CoreIDWidth = cf_math_pkg::idx_width(NumCC); localparam int unsigned TCDMMemAddrWidth = $clog2(TCDMDepth); // Enlarge the address width for Spatz due to cache @@ -213,7 +218,7 @@ module cachepool_tile localparam int unsigned NrSuperBanks = NrBanks / BanksPerSuperBank; function automatic int unsigned get_tcdm_ports(int unsigned core); - return spatz_pkg::N_FU + 1; + return spatz_pkg::N_FU + NumScalarPerCC; endfunction function automatic int unsigned get_tcdm_port_offs(int unsigned core_idx); @@ -222,7 +227,7 @@ module cachepool_tile return n; endfunction - localparam int unsigned NrTCDMPortsCores = get_tcdm_port_offs(NrCores); + localparam int unsigned NrTCDMPortsCores = get_tcdm_port_offs(NumCC); localparam int unsigned NumTCDMIn = NrTCDMPortsCores + 1; localparam logic [AxiAddrWidth-1:0] TCDMMask = ~(TCDMSize-1); @@ -350,13 +355,12 @@ module cachepool_tile tcdm_req_t [NrTCDMPortsCores-1:0] tcdm_req; tcdm_rsp_t [NrTCDMPortsCores-1:0] tcdm_rsp; - core_events_t [NrCores-1:0] core_events; + // snitch_icache_pkg::icache_events_t [NumCC-1:0] icache_events; - // snitch_icache_pkg::icache_events_t [NrCores-1:0] icache_events; - - // 4. Memory Subsystem (Core side). - reqrsp_req_t [NrCores-1:0] core_req, filtered_core_req; - reqrsp_rsp_t [NrCores-1:0] core_rsp, filtered_core_rsp; + // 4. Memory Subsystem (Core side). Per-hart (not per-CC): each hart needs + // its own Periph/barrier port, one per hart sharing a CC included. + reqrsp_req_t [NrHarts-1:0] core_req, filtered_core_req; + reqrsp_rsp_t [NrHarts-1:0] core_rsp, filtered_core_rsp; // 8. L1 D$ @@ -372,8 +376,11 @@ module cachepool_tile tcdm_req_t [NrTCDMPortsPerCore-1:0][NumL1CtrlTile-1:0] cache_ctrl_req; tcdm_rsp_t [NrTCDMPortsPerCore-1:0][NumL1CtrlTile-1:0] cache_bank_rsp; - tcdm_req_t [NumL1CtrlTile-1:0] cache_amo_req; - tcdm_rsp_t [NumL1CtrlTile-1:0] cache_amo_rsp; + // One AMO-capable path per scalar port sharing this CC (indexed by the + // same j as cache_ctrl_req/cache_bank_rsp; only the last NumScalarPerCC + // planes are ever driven, see gen_cache_amo_connect). + tcdm_req_t [NrTCDMPortsPerCore-1:0][NumL1CtrlTile-1:0] cache_amo_req; + tcdm_rsp_t [NrTCDMPortsPerCore-1:0][NumL1CtrlTile-1:0] cache_amo_rsp; logic [NumL1CtrlTile-1:0][NrTCDMPortsPerCore-1:0] cache_req_valid; @@ -423,7 +430,7 @@ module cachepool_tile // Per-core response-side readiness, driven by each i_cachepool_cc instance. logic [NrTCDMPortsCores-1:0] tcdm_rsp_ready; logic [NrTCDMPortsPerCore-1:0][NumL1CtrlTile-1:0] cache_pready, cache_xbar_pready; - logic [NumL1CtrlTile-1:0] cache_amo_pready; + logic [NrTCDMPortsPerCore-1:0][NumL1CtrlTile-1:0] cache_amo_pready; always_comb begin : cache_flush_protection for (int j = 0; unsigned'(j) < NrTCDMPortsCores; j++) begin @@ -684,7 +691,7 @@ module cachepool_tile tcdm_cache_interco #( .NumTiles (NumTiles ), - .NumCores (NrCores ), + .NumCores (NumCC ), .NumCache (NumL1CtrlTile ), .NumTotCache (NumL1CacheCtrl ), .NumLGPort (NumLGPortCore ), @@ -716,7 +723,7 @@ module cachepool_tile // No inter-group remote ports: instantiate interco without inter-group remote ports (backward-compatible). tcdm_cache_interco #( .NumTiles (NumTiles ), - .NumCores (NrCores ), + .NumCores (NumCC ), .NumCache (NumL1CtrlTile ), .NumTotCache (NumL1CacheCtrl ), .NumLGPort (NumLGPortCore ), @@ -750,7 +757,10 @@ module cachepool_tile // Ports from Spatz can bypass this module for (genvar j = 0; j < NrTCDMPortsPerCore; j++) begin : gen_cache_amo_connect - if (j == NrTCDMPortsPerCore-1) begin : gen_amo + // One AMO-capable path per scalar port sharing this CC (the last + // NumScalarPerCC planes) -- each hart's own scalar port needs + // independent AMO capability, not just a single shared one. + if (j >= NrTCDMPortsPerCore - NumScalarPerCC) begin : gen_amo spatz_cache_amo #( .DataWidth ( DataWidth ), .CoreIDWidth ( CoreIDWidth ), @@ -765,9 +775,9 @@ module cachepool_tile .core_req_i (cache_ctrl_req [j][cb] ), .core_rsp_ready_i (cache_xbar_pready[j][cb] ), .core_rsp_o (cache_bank_rsp [j][cb] ), - .mem_req_o (cache_amo_req [cb] ), - .mem_rsp_ready_o (cache_amo_pready [cb] ), - .mem_rsp_i (cache_amo_rsp [cb] ) + .mem_req_o (cache_amo_req [j][cb] ), + .mem_rsp_ready_o (cache_amo_pready [j][cb] ), + .mem_rsp_i (cache_amo_rsp [j][cb] ) ); tcdm_req_t cache_req_reg; @@ -778,27 +788,27 @@ module cachepool_tile .Bypass ( 1'b0 ) ) i_spill_reg_cache_req ( .clk_i , - .rst_ni ( rst_ni ), - .valid_i ( cache_amo_req[cb].q_valid ), - .ready_o ( cache_amo_rsp[cb].q_ready ), - .data_i ( cache_amo_req[cb].q ), - .valid_o ( cache_req_reg.q_valid ), - .ready_i ( cache_rsp_reg.q_ready ), - .data_o ( cache_req_reg.q ) + .rst_ni ( rst_ni ), + .valid_i ( cache_amo_req[j][cb].q_valid ), + .ready_o ( cache_amo_rsp[j][cb].q_ready ), + .data_i ( cache_amo_req[j][cb].q ), + .valid_o ( cache_req_reg.q_valid ), + .ready_i ( cache_rsp_reg.q_ready ), + .data_o ( cache_req_reg.q ) ); spill_register #( .T ( tcdm_rsp_chan_t ), .Bypass ( 1'b1 ) ) i_spill_reg_cache_rsp ( - .clk_i ( clk_i ), - .rst_ni ( rst_ni ), - .valid_i ( cache_rsp_reg.p_valid ), - .ready_o ( cache_rsp_ready [cb][j] ), - .data_i ( cache_rsp_reg.p ), - .valid_o ( cache_amo_rsp [cb].p_valid), - .ready_i ( cache_amo_pready[cb] ), - .data_o ( cache_amo_rsp [cb].p ) + .clk_i ( clk_i ), + .rst_ni ( rst_ni ), + .valid_i ( cache_rsp_reg.p_valid ), + .ready_o ( cache_rsp_ready [cb][j] ), + .data_i ( cache_rsp_reg.p ), + .valid_o ( cache_amo_rsp [j][cb].p_valid), + .ready_i ( cache_amo_pready[j][cb] ), + .data_o ( cache_amo_rsp [j][cb].p ) ); assign cache_req_valid[cb][j] = cache_req_reg.q_valid; @@ -1448,92 +1458,182 @@ module cachepool_tile end end - hive_req_t [NrCores-1:0] hive_req; - hive_rsp_t [NrCores-1:0] hive_rsp; + hive_req_t [NrHarts-1:0] hive_req; + hive_rsp_t [NrHarts-1:0] hive_rsp; - for (genvar i = 0; i < NrCores; i++) begin : gen_core + for (genvar i = 0; i < NumCC; i++) begin : gen_cc localparam int unsigned TcdmPorts = get_tcdm_ports(i); localparam int unsigned TcdmPortsOffs = get_tcdm_port_offs(i); + // First hart index served by this CC slot (the only one if + // NumScalarPerCC==1; cachepool_cc_dual below wires up HartIdx+1 too). + localparam int unsigned HartIdx = i * NumScalarPerCC; - interrupts_t irq; + interrupts_t irq0; sync #(.STAGES (2)) - i_sync_debug (.clk_i, .rst_ni, .serial_i (debug_req_i), .serial_o (irq.debug)); + i_sync_debug0 (.clk_i, .rst_ni, .serial_i (debug_req_i), .serial_o (irq0.debug)); sync #(.STAGES (2)) - i_sync_meip (.clk_i, .rst_ni, .serial_i (meip_i), .serial_o (irq.meip)); + i_sync_meip0 (.clk_i, .rst_ni, .serial_i (meip_i), .serial_o (irq0.meip)); sync #(.STAGES (2)) - i_sync_mtip (.clk_i, .rst_ni, .serial_i (mtip_i), .serial_o (irq.mtip)); + i_sync_mtip0 (.clk_i, .rst_ni, .serial_i (mtip_i), .serial_o (irq0.mtip)); sync #(.STAGES (2)) - i_sync_msip (.clk_i, .rst_ni, .serial_i (msip_i), .serial_o (irq.msip)); - assign irq.mcip = cl_interrupt_i[i]; + i_sync_msip0 (.clk_i, .rst_ni, .serial_i (msip_i), .serial_o (irq0.msip)); + assign irq0.mcip = 1'b0; tcdm_req_t [TcdmPorts-1:0] tcdm_req_wo_user; - logic [31:0] hart_id; - assign hart_id = hart_base_id_i + i; - - cachepool_cc #( - .BootAddr (BootAddr ), - .UartAddr (UartAddr ), - .RVE (1'b0 ), - .RVF (RVF ), - .RVD (RVD ), - .RVV (RVV ), - .AddrWidth (AxiAddrWidth ), - .DataWidth (NarrowDataWidth ), - .UserWidth (AxiUserWidth ), - .SnitchPMACfg (SnitchPMACfg ), - .dreq_t (reqrsp_req_t ), - .drsp_t (reqrsp_rsp_t ), - .dreq_chan_t (reqrsp_req_chan_t ), - .drsp_chan_t (reqrsp_rsp_chan_t ), - .tcdm_req_t (tcdm_req_t ), - .tcdm_user_t (tcdm_user_t ), - .tcdm_req_chan_t (tcdm_req_chan_t ), - .tcdm_rsp_t (tcdm_rsp_t ), - .tcdm_rsp_chan_t (tcdm_rsp_chan_t ), - .axi_req_t (axi_mst_tile_wide_req_t ), - .axi_ar_chan_t (axi_mst_tile_wide_ar_chan_t), - .axi_aw_chan_t (axi_mst_tile_wide_aw_chan_t), - .axi_rsp_t (axi_mst_tile_wide_resp_t ), - .hive_req_t (hive_req_t ), - .hive_rsp_t (hive_rsp_t ), - .acc_issue_req_t (acc_issue_req_t ), - .acc_issue_rsp_t (acc_issue_rsp_t ), - .acc_rsp_t (acc_rsp_t ), - .XDivSqrt (1'b0 ), - .XF16 (1'b1 ), - .XF16ALT (1'b0 ), - .XF8 (1'b1 ), - .XF8ALT (1'b0 ), - .IsoCrossing (1'b0 ), - .NumIntOutstandingLoads (NumIntOutstandingLoads ), - .NumIntOutstandingMem (NumIntOutstandingMem ), - .NumSpatzOutstandingLoads(NumSpatzOutstandingLoads ), - .FPUImplementation (FPUImplementation ), - .SpmStackDepth (SpmStackDepth ), - .RegisterOffloadRsp (RegisterOffloadRsp ), - .RegisterCoreReq (RegisterCoreReq ), - .RegisterCoreRsp (RegisterCoreRsp ), - .NumSpatzFPUs (NumSpatzFPUs ), - .NumSpatzIPUs (NumSpatzIPUs ), - .TCDMAddrWidth (SPMAddrWidth ) - ) i_cachepool_cc ( - .clk_i (clk_i ), - .rst_ni (rst_ni ), - .testmode_i (1'b0 ), - .hart_id_i (hart_id ), - .hive_req_o (hive_req[i] ), - .hive_rsp_i (hive_rsp[i] ), - .irq_i (irq ), - .data_req_o (core_req[i] ), - .data_rsp_i (core_rsp[i] ), - .tcdm_req_o (tcdm_req_wo_user ), - .tcdm_rsp_i (tcdm_rsp[TcdmPortsOffs +: TcdmPorts] ), - .tcdm_rsp_ready_o (tcdm_rsp_ready[TcdmPortsOffs +: TcdmPorts] ), - .core_events_o (core_events[i] ), - .tcdm_addr_base_i (tcdm_start_address ) - ); + logic [31:0] hart_id0; + assign hart_id0 = hart_base_id_i + HartIdx; + + if (NumScalarPerCC == 1) begin : gen_single + cachepool_cc #( + .BootAddr (BootAddr ), + .UartAddr (UartAddr ), + .RVE (1'b0 ), + .RVF (RVF ), + .RVD (RVD ), + .RVV (RVV ), + .AddrWidth (AxiAddrWidth ), + .DataWidth (NarrowDataWidth ), + .UserWidth (AxiUserWidth ), + .SnitchPMACfg (SnitchPMACfg ), + .dreq_t (reqrsp_req_t ), + .drsp_t (reqrsp_rsp_t ), + .dreq_chan_t (reqrsp_req_chan_t ), + .drsp_chan_t (reqrsp_rsp_chan_t ), + .tcdm_req_t (tcdm_req_t ), + .tcdm_user_t (tcdm_user_t ), + .tcdm_req_chan_t (tcdm_req_chan_t ), + .tcdm_rsp_t (tcdm_rsp_t ), + .tcdm_rsp_chan_t (tcdm_rsp_chan_t ), + .axi_req_t (axi_mst_tile_wide_req_t ), + .axi_ar_chan_t (axi_mst_tile_wide_ar_chan_t), + .axi_aw_chan_t (axi_mst_tile_wide_aw_chan_t), + .axi_rsp_t (axi_mst_tile_wide_resp_t ), + .hive_req_t (hive_req_t ), + .hive_rsp_t (hive_rsp_t ), + .acc_issue_req_t (acc_issue_req_t ), + .acc_issue_rsp_t (acc_issue_rsp_t ), + .acc_rsp_t (acc_rsp_t ), + .XDivSqrt (1'b0 ), + .XF16 (1'b1 ), + .XF16ALT (1'b0 ), + .XF8 (1'b1 ), + .XF8ALT (1'b0 ), + .IsoCrossing (1'b0 ), + .NumIntOutstandingLoads (NumIntOutstandingLoads ), + .NumIntOutstandingMem (NumIntOutstandingMem ), + .NumSpatzOutstandingLoads(NumSpatzOutstandingLoads ), + .FPUImplementation (FPUImplementation ), + .SpmStackDepth (SpmStackDepth ), + .RegisterOffloadRsp (RegisterOffloadRsp ), + .RegisterCoreReq (RegisterCoreReq ), + .RegisterCoreRsp (RegisterCoreRsp ), + .NumSpatzFPUs (NumSpatzFPUs ), + .NumSpatzIPUs (NumSpatzIPUs ), + .TCDMAddrWidth (SPMAddrWidth ) + ) i_cachepool_cc ( + .clk_i (clk_i ), + .rst_ni (rst_ni ), + .testmode_i (1'b0 ), + .hart_id_i (hart_id0 ), + .hive_req_o (hive_req[HartIdx] ), + .hive_rsp_i (hive_rsp[HartIdx] ), + .irq_i (irq0 ), + .data_req_o (core_req[HartIdx] ), + .data_rsp_i (core_rsp[HartIdx] ), + .tcdm_req_o (tcdm_req_wo_user ), + .tcdm_rsp_i (tcdm_rsp[TcdmPortsOffs +: TcdmPorts] ), + .tcdm_rsp_ready_o (tcdm_rsp_ready[TcdmPortsOffs +: TcdmPorts] ), + .core_events_o (/* unused */ ), + .tcdm_addr_base_i (tcdm_start_address ) + ); + end else if (NumScalarPerCC == 2) begin : gen_dual + // Second hart's own interrupt sync + irq assembly (mirrors hart 0's above). + interrupts_t irq1; + + sync #(.STAGES (2)) + i_sync_debug1 (.clk_i, .rst_ni, .serial_i (debug_req_i), .serial_o (irq1.debug)); + sync #(.STAGES (2)) + i_sync_meip1 (.clk_i, .rst_ni, .serial_i (meip_i), .serial_o (irq1.meip)); + sync #(.STAGES (2)) + i_sync_mtip1 (.clk_i, .rst_ni, .serial_i (mtip_i), .serial_o (irq1.mtip)); + sync #(.STAGES (2)) + i_sync_msip1 (.clk_i, .rst_ni, .serial_i (msip_i), .serial_o (irq1.msip)); + assign irq1.mcip = 1'b0; + + interrupts_t [1:0] irq_pair; + assign irq_pair[0] = irq0; + assign irq_pair[1] = irq1; + + logic [1:0][31:0] hart_id_pair; + assign hart_id_pair[0] = hart_id0; + assign hart_id_pair[1] = hart_base_id_i + HartIdx + 1; + + cachepool_cc_dual #( + .AddrWidth (AxiAddrWidth ), + .DataWidth (NarrowDataWidth ), + .UserWidth (AxiUserWidth ), + .SpmStackDepth (SpmStackDepth ), + .dreq_t (reqrsp_req_t ), + .drsp_t (reqrsp_rsp_t ), + .dreq_chan_t (reqrsp_req_chan_t ), + .drsp_chan_t (reqrsp_rsp_chan_t ), + .tcdm_req_t (tcdm_req_t ), + .tcdm_user_t (tcdm_user_t ), + .tcdm_req_chan_t (tcdm_req_chan_t ), + .tcdm_rsp_t (tcdm_rsp_t ), + .tcdm_rsp_chan_t (tcdm_rsp_chan_t ), + .TCDMAddrWidth (SPMAddrWidth ), + .hive_req_t (hive_req_t ), + .hive_rsp_t (hive_rsp_t ), + .acc_issue_req_t (acc_issue_req_t ), + .acc_issue_rsp_t (acc_issue_rsp_t ), + .acc_rsp_t (acc_rsp_t ), + .FPUImplementation (FPUImplementation ), + .BootAddr (BootAddr ), + .UartAddr (UartAddr ), + .RVE (1'b0 ), + .RVF (RVF ), + .RVD (RVD ), + .XDivSqrt (1'b0 ), + .XF8 (1'b1 ), + .XF16 (1'b1 ), + .XF16ALT (1'b0 ), + .XF8ALT (1'b0 ), + .NumIntOutstandingLoads (NumIntOutstandingLoads ), + .NumIntOutstandingMem (NumIntOutstandingMem ), + .NumSpatzOutstandingLoads(NumSpatzOutstandingLoads ), + .RVV (RVV ), + .NumSpatzFPUs (NumSpatzFPUs ), + .NumSpatzIPUs (NumSpatzIPUs ), + .IsoCrossing (1'b0 ), + .RegisterOffloadRsp (RegisterOffloadRsp ), + .RegisterCoreReq (RegisterCoreReq ), + .RegisterCoreRsp (RegisterCoreRsp ), + .SnitchPMACfg (SnitchPMACfg ) + ) i_cachepool_cc_dual ( + .clk_i (clk_i ), + .rst_ni (rst_ni ), + .testmode_i (1'b0 ), + .hart_id_i (hart_id_pair ), + .irq_i (irq_pair ), + .hive_req_o (hive_req[HartIdx +: 2] ), + .hive_rsp_i (hive_rsp[HartIdx +: 2] ), + .data_req_o (core_req[HartIdx +: 2] ), + .data_rsp_i (core_rsp[HartIdx +: 2] ), + .tcdm_req_o (tcdm_req_wo_user ), + .tcdm_rsp_i (tcdm_rsp[TcdmPortsOffs +: TcdmPorts] ), + .tcdm_rsp_ready_o (tcdm_rsp_ready[TcdmPortsOffs +: TcdmPorts] ), + .tcdm_addr_base_i (tcdm_start_address ), + .cluster_periph_start_address_i (cluster_periph_start_address), + .owner_id_o (/* debug only, unused */ ) + ); + end else begin : gen_unsupported + `ASSERT_INIT(NumScalarPerCCSupported, 1'b0, + "cachepool_tile: NumScalarPerCC configuration not supported (only 1 or 2)") + end + for (genvar j = 0; j < TcdmPorts; j++) begin : gen_tcdm_user always_comb begin tcdm_req[TcdmPortsOffs+j].q = tcdm_req_wo_user[j].q; @@ -1547,16 +1647,16 @@ module cachepool_tile // Instruction Cache // ---------------- - addr_t [NrCores-1:0] inst_addr; - logic [NrCores-1:0] inst_cacheable; - logic [NrCores-1:0][31:0] inst_data; - logic [NrCores-1:0] inst_valid; - logic [NrCores-1:0] inst_ready; - logic [NrCores-1:0] inst_error; - logic [NrCores-1:0] flush_valid; - logic [NrCores-1:0] flush_ready; + addr_t [NrHarts-1:0] inst_addr; + logic [NrHarts-1:0] inst_cacheable; + logic [NrHarts-1:0][31:0] inst_data; + logic [NrHarts-1:0] inst_valid; + logic [NrHarts-1:0] inst_ready; + logic [NrHarts-1:0] inst_error; + logic [NrHarts-1:0] flush_valid; + logic [NrHarts-1:0] flush_ready; - for (genvar i = 0; i < NrCores; i++) begin : gen_unpack_icache + for (genvar i = 0; i < NrHarts; i++) begin : gen_unpack_icache assign inst_addr[i] = hive_req[i].inst_addr; assign inst_cacheable[i] = hive_req[i].inst_cacheable; assign inst_valid[i] = hive_req[i].inst_valid; @@ -1571,7 +1671,7 @@ module cachepool_tile end snitch_icache #( - .NR_FETCH_PORTS ( NrCores ), + .NR_FETCH_PORTS ( NrHarts ), .L0_LINE_COUNT ( 8 ), .LINE_WIDTH ( ICacheLineWidth ), .LINE_COUNT ( ICacheLineCount ), @@ -1618,7 +1718,7 @@ module cachepool_tile // First-level barrier for CachePool system cachepool_tile_barrier #( .AddrWidth (AxiAddrWidth ), - .NrPorts (NrCores ), + .NrPorts (NrHarts ), .dreq_t (reqrsp_req_t ), .drsp_t (reqrsp_rsp_t ), .user_t (tcdm_user_t ) @@ -1638,7 +1738,7 @@ module cachepool_tile reqrsp_rsp_t core_to_periph_rsp; reqrsp_mux #( - .NrPorts (NrCores ), + .NrPorts (NrHarts ), .AddrWidth (AxiAddrWidth ), .DataWidth (NarrowDataWidth ), .UserWidth ($bits(tcdm_user_t)), diff --git a/hardware/tb/cachepool_cluster_wrapper.sv b/hardware/tb/cachepool_cluster_wrapper.sv index 6c4d7913..33b3b911 100644 --- a/hardware/tb/cachepool_cluster_wrapper.sv +++ b/hardware/tb/cachepool_cluster_wrapper.sv @@ -73,7 +73,7 @@ module cachepool_cluster_wrapper .BootAddr (BootAddr ), .UartAddr (UartAddr ), .ClusterPeriphSize (64 ), - .NrCores (NumCores ), + .NumCC (NumCC ), .TCDMDepth (TCDMDepth ), .NrBanks (NumBank ), .ICacheLineWidth (ICacheLineWidth ), @@ -99,7 +99,8 @@ module cachepool_cluster_wrapper .UseFoldedDataBanks (UseFoldedDataBanks ), .FoldWayGroup (FoldWayGroup ), .UseHashWaySelect (UseHashWaySelect ), - .UseForwardingBuffer (UseForwardingBuffer ) + .UseForwardingBuffer (UseForwardingBuffer ), + .HartBaseId (10'h0 ) ) i_cluster ( .clk_i , .rst_ni , @@ -110,7 +111,6 @@ module cachepool_cluster_wrapper .meip_i (meip_i ), .mtip_i (mtip_i ), .msip_i (msip_i ), - .hart_base_id_i (10'h0 ), .cluster_base_addr_i (TCDMStartAddr ), .cluster_probe_o (cluster_probe_o ), // REQRSP peripheral in-port: passed straight through from wrapper port. diff --git a/hardware/tb/cachepool_monitor.sv b/hardware/tb/cachepool_monitor.sv index 0ddd0d4e..10af768d 100644 --- a/hardware/tb/cachepool_monitor.sv +++ b/hardware/tb/cachepool_monitor.sv @@ -29,8 +29,25 @@ module cachepool_monitor `define TILE_PATH(gy, gx, t) \ i_cluster_wrapper.i_cluster.gen_group_y[gy].gen_group_x[gx].i_group.i_group.gen_tiles[t].gen_tile.i_tile - `define CC_PATH(gy, gx, t, c) \ - `TILE_PATH(gy, gx, t).gen_core[c].i_cachepool_cc + // CACHEPOOL_DUAL_CC (plain definedness flag, set by the Makefile iff + // num_scalar_per_core==2) picks which sub-instance gen_cc wraps, since + // the toolchain's preprocessor doesn't support `if value-comparisons. + // CC_SNITCH_PATH takes a hart index h; only h=0 is probed below today + // (the monitor's c-loop still indexes CC slots 1:1 with harts, so + // per-hart dual-mode stat collection is unstarted, deferred). i_spatz + // sits at the same relative depth in both flavors, so CC_PATH's + // existing `.i_spatz.*` usages need no change either way. + `ifdef CACHEPOOL_DUAL_CC + `define CC_PATH(gy, gx, t, c) \ + `TILE_PATH(gy, gx, t).gen_cc[c].gen_dual.i_cachepool_cc_dual + `define CC_SNITCH_PATH(gy, gx, t, c, h) \ + `CC_PATH(gy, gx, t, c).gen_snitch[h].i_snitch + `else + `define CC_PATH(gy, gx, t, c) \ + `TILE_PATH(gy, gx, t).gen_cc[c].gen_single.i_cachepool_cc + `define CC_SNITCH_PATH(gy, gx, t, c, h) \ + `CC_PATH(gy, gx, t, c).i_snitch + `endif `define CLUSTER_PATH i_cluster_wrapper.i_cluster @@ -618,21 +635,21 @@ module cachepool_monitor automatic logic inst_valid, inst_ready, stall; automatic logic is_no_instr, is_itlb, is_hazard, is_lsu, is_acc, is_fence; - inst_valid = `CC_PATH(gy, gx, t, c).i_snitch.inst_valid_o; - inst_ready = `CC_PATH(gy, gx, t, c).i_snitch.inst_ready_i; - stall = `CC_PATH(gy, gx, t, c).i_snitch.stall; + inst_valid = `CC_SNITCH_PATH(gy, gx, t, c, 0).inst_valid_o; + inst_ready = `CC_SNITCH_PATH(gy, gx, t, c, 0).inst_ready_i; + stall = `CC_SNITCH_PATH(gy, gx, t, c, 0).stall; is_no_instr = ~(inst_valid & inst_ready); - is_itlb = `CC_PATH(gy, gx, t, c).i_snitch.trans_active & - ~(`CC_PATH(gy, gx, t, c).i_snitch.itlb_valid & - `CC_PATH(gy, gx, t, c).i_snitch.itlb_ready); - is_hazard = ~(`CC_PATH(gy, gx, t, c).i_snitch.operands_ready & - `CC_PATH(gy, gx, t, c).i_snitch.dst_ready); - is_lsu = `CC_PATH(gy, gx, t, c).i_snitch.lsu_stall; - is_acc = `CC_PATH(gy, gx, t, c).i_snitch.acc_stall; - is_fence = `CC_PATH(gy, gx, t, c).i_snitch.fence_snitch_stall | - `CC_PATH(gy, gx, t, c).i_snitch.fence_spatz_stall | - `CC_PATH(gy, gx, t, c).i_snitch.fence_stall; + is_itlb = `CC_SNITCH_PATH(gy, gx, t, c, 0).trans_active & + ~(`CC_SNITCH_PATH(gy, gx, t, c, 0).itlb_valid & + `CC_SNITCH_PATH(gy, gx, t, c, 0).itlb_ready); + is_hazard = ~(`CC_SNITCH_PATH(gy, gx, t, c, 0).operands_ready & + `CC_SNITCH_PATH(gy, gx, t, c, 0).dst_ready); + is_lsu = `CC_SNITCH_PATH(gy, gx, t, c, 0).lsu_stall; + is_acc = `CC_SNITCH_PATH(gy, gx, t, c, 0).acc_stall; + is_fence = `CC_SNITCH_PATH(gy, gx, t, c, 0).fence_snitch_stall | + `CC_SNITCH_PATH(gy, gx, t, c, 0).fence_spatz_stall | + `CC_SNITCH_PATH(gy, gx, t, c, 0).fence_stall; inst_valid_cyc_d = session_edge ? '0 : inst_valid_cyc_q + (inst_valid ? 1 : 0); inst_accepted_d = session_edge ? '0 : @@ -660,13 +677,13 @@ module cachepool_monitor // outstanding. logic snitch_load_req_accept, snitch_load_resp_commit; assign snitch_load_req_accept = - `CC_PATH(gy, gx, t, c).i_snitch.data_req_o.q_valid & - `CC_PATH(gy, gx, t, c).i_snitch.data_rsp_i.q_ready & - ~`CC_PATH(gy, gx, t, c).i_snitch.data_req_o.q.write; + `CC_SNITCH_PATH(gy, gx, t, c, 0).data_req_o.q_valid & + `CC_SNITCH_PATH(gy, gx, t, c, 0).data_rsp_i.q_ready & + ~`CC_SNITCH_PATH(gy, gx, t, c, 0).data_req_o.q.write; assign snitch_load_resp_commit = - `CC_PATH(gy, gx, t, c).i_snitch.data_rsp_i.p_valid & - `CC_PATH(gy, gx, t, c).i_snitch.data_req_o.p_ready & - ~`CC_PATH(gy, gx, t, c).i_snitch.data_rsp_i.p.write; + `CC_SNITCH_PATH(gy, gx, t, c, 0).data_rsp_i.p_valid & + `CC_SNITCH_PATH(gy, gx, t, c, 0).data_req_o.p_ready & + ~`CC_SNITCH_PATH(gy, gx, t, c, 0).data_rsp_i.p.write; cnt_t load_req_ts_q[$]; cnt_t snitch_load_lat_sum_q, snitch_load_lat_cnt_q; @@ -866,6 +883,7 @@ module cachepool_monitor `undef TILE_PATH `undef CC_PATH + `undef CC_SNITCH_PATH `undef CLUSTER_PATH `endif diff --git a/iis-env.sh b/iis-env.sh index 787ba0cb..341ee936 100644 --- a/iis-env.sh +++ b/iis-env.sh @@ -13,7 +13,14 @@ export INSTALL_DIR=/home/dishen/cachepool-32b/install # Python deps for hardware code generation (make generate) - venv instead of uv, # to match the existing local dev flow. -python3 -m venv cachepool +# python3.12 (not the default python3) is required: floogen needs Python >=3.10. +PYTHON=python3.12 +export PYTHON +$PYTHON -m venv cachepool source cachepool/bin/activate python3 -m pip install --quiet --upgrade pip python3 -m pip install --quiet -r requirements.txt + +# floogen (FlooNoC code generator) is pulled in via bender, not requirements.txt - +# install it editable into this venv so `floogen` on PATH resolves here. +make install-floogen PYTHON=python3 --no-print-directory diff --git a/requirements.txt b/requirements.txt index 3e978375..ef40567c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ numpy # CPU-only torch for data generation scripts (gen_data.py in software/tests/) --extra-index-url https://download.pytorch.org/whl/cpu torch +pyyaml diff --git a/sim/scripts/vsim_core.tcl b/sim/scripts/vsim_core.tcl index 30ee61ac..f6bf0316 100644 --- a/sim/scripts/vsim_core.tcl +++ b/sim/scripts/vsim_core.tcl @@ -18,8 +18,16 @@ if {$argc > 5 && "${6}" != ""} { quietly lappend parent_grp -group ${6} } -# The {*} syntax safely expands the list. +# Detect flavor at runtime (gen_single vs gen_dual) instead of relying on a +# compile-time flag, so this script works regardless of num_scalar_per_core. +if {[llength [find instances -nodu ${core_path}/gen_dual/i_cachepool_cc_dual]] > 0} { + do sim/scripts/vsim_core_dual.tcl ${1} ${2} ${3} ${4} ${5} ${6} + return +} + +# The {*} syntax safely expands the list. # If $parent_grp is empty, it safely ignores it instead of passing "". +quietly set core_path ${core_path}/gen_single add wave -noupdate {*}$parent_grp -group ${name} -group scalar_xbar ${core_path}/i_cachepool_cc/i_scalar_xbar/* add wave -noupdate {*}$parent_grp -group ${name} -group Params ${core_path}/i_cachepool_cc/BootAddr diff --git a/sim/scripts/vsim_core_dual.tcl b/sim/scripts/vsim_core_dual.tcl new file mode 100644 index 00000000..bbf1e34d --- /dev/null +++ b/sim/scripts/vsim_core_dual.tcl @@ -0,0 +1,76 @@ +# Copyright 2026 ETH Zurich and University of Bologna. +# Solderpad Hardware License, Version 0.51, see LICENSE for details. +# SPDX-License-Identifier: SHL-0.51 + +# Create group for a dual-CC Core Complex (2 Snitch harts sharing 1 Spatz). +# Not a mechanical mirror of vsim_core.tcl's single-CC dump: cachepool_cc_dual +# has a different internal structure (2x gen_snitch, 1 shared i_spatz, plus +# the lock/acc_mux), so this covers both harts' top-level Snitch signals, the +# shared Spatz, and the ownership lock/mux state, rather than replicating +# every internal Snitch/Spatz signal group per hart. +onerror {resume} +quietly WaveActivateNextPane {} 0 + +quietly set core_path ${4}/gen_dual +quietly set name g_${1}_t_${2}_c_${3} + +quietly set parent_grp [list] +if {$argc > 4 && "${5}" != ""} { + quietly lappend parent_grp -group ${5} +} +if {$argc > 5 && "${6}" != ""} { + quietly lappend parent_grp -group ${6} +} + +# Lock/switch state: which host owns Spatz, and the acc_mux fake-completion state. +add wave -noupdate {*}$parent_grp -group ${name} -group Lock ${core_path}/i_cachepool_cc_dual/i_spatz_lock/* +add wave -noupdate {*}$parent_grp -group ${name} -group AccMux ${core_path}/i_cachepool_cc_dual/i_acc_mux/* + +for {set h 0} {$h < 2} {incr h} { + quietly set snitch_path ${core_path}/i_cachepool_cc_dual/gen_snitch[${h}]/i_snitch + + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} -radix unsigned ${snitch_path}/hart_id_i + + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} -divider Instructions + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} ${snitch_path}/inst_addr_o + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} ${snitch_path}/inst_data_i + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} ${snitch_path}/inst_valid_o + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} ${snitch_path}/inst_ready_i + + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} -divider Load/Store + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} ${snitch_path}/data_req_o + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} ${snitch_path}/data_rsp_i + + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} -divider Accelerator + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} ${snitch_path}/acc_qreq_o + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} ${snitch_path}/acc_qrsp_i + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} ${snitch_path}/acc_qvalid_o + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} ${snitch_path}/acc_qready_i + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} ${snitch_path}/acc_prsp_i + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} ${snitch_path}/acc_pvalid_i + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} ${snitch_path}/acc_pready_o + + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} -group Internal ${snitch_path}/* +} + +quietly set spatz_path ${core_path}/i_cachepool_cc_dual/i_spatz + +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz ${spatz_path}/issue_valid_i +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz ${spatz_path}/issue_ready_o +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz ${spatz_path}/issue_req_i +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz ${spatz_path}/issue_rsp_o +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz ${spatz_path}/rsp_valid_o +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz ${spatz_path}/rsp_ready_i +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz ${spatz_path}/rsp_o +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz ${spatz_path}/spatz_mem_req_o +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz ${spatz_path}/spatz_mem_req_valid_o +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz ${spatz_path}/spatz_mem_req_ready_i +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz ${spatz_path}/spatz_mem_rsp_i +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz ${spatz_path}/spatz_mem_rsp_valid_i + +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz -group VLSU ${spatz_path}/i_vlsu/* +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz -group VSLDU ${spatz_path}/i_vsldu/* +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz -group VFU ${spatz_path}/i_vfu/* +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz -group Controller ${spatz_path}/i_controller/* + +add wave -noupdate {*}$parent_grp -group ${name} -group Internal ${core_path}/i_cachepool_cc_dual/* diff --git a/sim/scripts/vsim_wave.tcl b/sim/scripts/vsim_wave.tcl index 8a1db670..74ee5ef5 100644 --- a/sim/scripts/vsim_wave.tcl +++ b/sim/scripts/vsim_wave.tcl @@ -61,7 +61,7 @@ for {set g 0} {$g < $NUM_GROUPS} {incr g} { # 4. Plot all cores grouped under their tile for {set core 0} {$core < $NUM_CORES} {incr core} { - quietly set core_path ${tile_path}/i_tile/gen_core[${core}] + quietly set core_path ${tile_path}/i_tile/gen_cc[${core}] do sim/scripts/vsim_core.tcl $g $tile $core ${core_path} "${gwp_name}" "tile[${tile}]" } } diff --git a/software/CMakeLists.txt b/software/CMakeLists.txt index 88de4857..64b8ed7e 100644 --- a/software/CMakeLists.txt +++ b/software/CMakeLists.txt @@ -15,6 +15,13 @@ project(snitch_cluster LANGUAGES C ASM) include(SnitchUtilities) enable_testing() + +# Set here (not just in snRuntime/CMakeLists.txt) so sibling directories +# added below, e.g. software/tests, also see this define -- add_compile_definitions +# is a directory property and does not propagate to sibling add_subdirectory trees. +set(NUM_SCALAR_PER_CORE "1" CACHE STRING "Snitch harts sharing one Spatz per Core Complex") +add_compile_definitions(SNRT_NUM_SCALAR_PER_CORE=${NUM_SCALAR_PER_CORE}) + add_subdirectory(${SNITCH_SOFTWARE_DIR}/snRuntime snRuntime) # add_subdirectory(${SPATZ_SOFTWARE_DIR}/riscvTests riscvTests) # add_subdirectory(${SPATZ_SOFTWARE_DIR}/spatzBenchmarks spatzBenchmarks) @@ -22,5 +29,15 @@ add_subdirectory(${SNITCH_SOFTWARE_DIR}/snRuntime snRuntime) option(ENABLE_CACHEPOOL_TESTS "Enable CachePool tests for spatz" OFF) if (ENABLE_CACHEPOOL_TESTS) message(STATUS "Adding CachePool tests") + # Shared macros (add_spatz_test_*) and common libraries (benchmark, + # spin_lock, mcs_lock); must be included before the group subdirectories + # below, since they use these macros/libraries in their own CMakeLists. + include(${CACHEPOOL_DIR}/software/cmake/CachePoolTests.cmake) + + add_subdirectory(${CACHEPOOL_DIR}/software/cache cache) + add_subdirectory(${CACHEPOOL_DIR}/software/sync sync) + add_subdirectory(${CACHEPOOL_DIR}/software/kernels/fp kernels_fp) + add_subdirectory(${CACHEPOOL_DIR}/software/kernels/int kernels_int) + add_subdirectory(${CACHEPOOL_DIR}/software/rlc rlc) add_subdirectory(${CACHEPOOL_DIR}/software/tests CachePoolTests) endif() diff --git a/software/README.md b/software/README.md new file mode 100644 index 00000000..589e53d8 --- /dev/null +++ b/software/README.md @@ -0,0 +1,63 @@ +# CachePool Software + +``` +software/ +├── snRuntime/ # bare-metal C runtime (barriers, l1 cache control, printf, +│ # allocator, Spatz lock, ...), linked into every test +├── cache/ # L1D cache correctness/stress tests +├── sync/ # locks, barriers, and other concurrency primitives +├── kernels/ +│ ├── fp/ # floating-point vector kernels (fdotp, fmatmul, gemv, fft, ...) +│ └── int/ # integer vector kernels (idotp, ...) +├── rlc/ # RLC (multi-producer/single-consumer) application test(s) +├── tests/ # smoke tests / one-off repros that don't fit another group, +│ # plus the shared benchmark/spin_lock/mcs_lock helper libraries +│ # and headers (tests/benchmark, tests/include) used by all groups +└── cmake/ # shared CMake macros and toolchain files +``` + +Each test lives in its own directory (e.g. `kernels/fp/fdotp-32b/`): +`main.c`, plus, for data-driven kernels, `kernel/` (kernel source), `data/` +(generated headers, gitignored), and `script/` (`gen_data.py` + +`data_.json` golden-value definitions). + +## Building + +From the repository root: + +``` +make sw config= # build all registered tests +``` + +Binaries land in `software/build/CachePoolTests/`, named +`test-`. Run one against the RTL simulator with: + +``` +./sim/bin/cachepool_cluster.vsim software/build/CachePoolTests/test- +``` + +## Adding a test + +1. Create the test's directory under the group it belongs to (`cache/`, + `sync/`, `kernels/fp/`, `kernels/int/`, `rlc/`, or `tests/` for + ungrouped smoke tests/repros). +2. Register it in that group's `CMakeLists.txt` with one of the + `add_spatz_test_{zeroParam,oneParam,twoParam,threeParam}` macros + (`zeroParam` for a fixed test, `oneParam`/`twoParam`/`threeParam` for + tests swept over 1-3 named parameters via `DATAHEADER`) — mirror an + existing entry in that file. +3. For a data-driven kernel, add `script/gen_data.py` and one + `script/data_.json` per parameter combination; golden headers + under `data/` are generated automatically by `make gen-data` (also run + as part of `make sw`). + +## Adding a new group + +Add a directory under `software/`, give it a `CMakeLists.txt` with its own +`add_spatz_test_*` calls, and add an `add_subdirectory(...)` line for it in +`software/CMakeLists.txt`, inside the `ENABLE_CACHEPOOL_TESTS` block. + +## Removing a test + +Delete its directory and the corresponding `add_spatz_test_*` line(s) from +the group's `CMakeLists.txt`. diff --git a/software/cache/CMakeLists.txt b/software/cache/CMakeLists.txt new file mode 100644 index 00000000..4c3c6282 --- /dev/null +++ b/software/cache/CMakeLists.txt @@ -0,0 +1,14 @@ +# Copyright 2025 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +add_spatz_test_zeroParam(byte-enable byte-enable/main.c) +add_spatz_test_zeroParam(cache-line-rw-smoke cache-line-rw-smoke/main.c) +add_spatz_test_zeroParam(cache-test-scalar cache-test-scalar/main.c) +add_spatz_test_zeroParam(cache-test-vector cache-test-vector/main.c) +add_spatz_test_zeroParam(cache-mix-smoke cache-mix-smoke/main.c) +add_spatz_test_zeroParam(cache-mix-pressure cache-mix-pressure/main.c) +add_spatz_test_zeroParam(cache-rlc-mimic cache-rlc-mimic/main.c) +add_spatz_test_zeroParam(cache-vector-rw cache-vector-rw/main.c) +add_spatz_test_zeroParam(cache-coverage cache-coverage/main.c) +add_spatz_test_zeroParam(cache-coverage-min cache-coverage-min/main.c) diff --git a/software/tests/byte-enable/main.c b/software/cache/byte-enable/main.c similarity index 100% rename from software/tests/byte-enable/main.c rename to software/cache/byte-enable/main.c diff --git a/software/tests/cache-coverage-min/main.c b/software/cache/cache-coverage-min/main.c similarity index 100% rename from software/tests/cache-coverage-min/main.c rename to software/cache/cache-coverage-min/main.c diff --git a/software/tests/cache-coverage/main.c b/software/cache/cache-coverage/main.c similarity index 100% rename from software/tests/cache-coverage/main.c rename to software/cache/cache-coverage/main.c diff --git a/software/tests/cache-line-rw-smoke/main.c b/software/cache/cache-line-rw-smoke/main.c similarity index 100% rename from software/tests/cache-line-rw-smoke/main.c rename to software/cache/cache-line-rw-smoke/main.c diff --git a/software/tests/cache-mix-pressure/main.c b/software/cache/cache-mix-pressure/main.c similarity index 100% rename from software/tests/cache-mix-pressure/main.c rename to software/cache/cache-mix-pressure/main.c diff --git a/software/tests/cache-mix-smoke/main.c b/software/cache/cache-mix-smoke/main.c similarity index 100% rename from software/tests/cache-mix-smoke/main.c rename to software/cache/cache-mix-smoke/main.c diff --git a/software/tests/cache-rlc-mimic/main.c b/software/cache/cache-rlc-mimic/main.c similarity index 100% rename from software/tests/cache-rlc-mimic/main.c rename to software/cache/cache-rlc-mimic/main.c diff --git a/software/tests/cache-test-scalar/main.c b/software/cache/cache-test-scalar/main.c similarity index 100% rename from software/tests/cache-test-scalar/main.c rename to software/cache/cache-test-scalar/main.c diff --git a/software/tests/cache-test-vector/main.c b/software/cache/cache-test-vector/main.c similarity index 100% rename from software/tests/cache-test-vector/main.c rename to software/cache/cache-test-vector/main.c diff --git a/software/tests/cache-vector-rw/main.c b/software/cache/cache-vector-rw/main.c similarity index 100% rename from software/tests/cache-vector-rw/main.c rename to software/cache/cache-vector-rw/main.c diff --git a/software/cmake/CachePoolTests.cmake b/software/cmake/CachePoolTests.cmake new file mode 100644 index 00000000..fef08168 --- /dev/null +++ b/software/cmake/CachePoolTests.cmake @@ -0,0 +1,57 @@ +# Copyright 2020 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +# Shared test-registration macros and common libraries for all CachePool +# test groups (cache/, sync/, kernels/fp/, kernels/int/, rlc/, tests/). +# Included once from the top-level software/CMakeLists.txt, before any +# group's add_subdirectory(), so its macros/libraries are visible to them. + +set(CACHEPOOL_TESTS_COMMON_DIR ${CACHEPOOL_DIR}/software/tests) + +# All test groups' binaries land in one place, software/build/CachePoolTests/, +# same as before the cache/sync/kernels/rlc split -- CI (.gitlab-ci.yml) and +# util/auto-benchmark reference that fixed path regardless of which group +# CMakeLists.txt registered the test. +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/CachePoolTests) + +include_directories(${CACHEPOOL_TESTS_COMMON_DIR}/include) +include_directories(${SNRUNTIME_INCLUDE_DIRS}) + +add_compile_options(-O3 -g -ffunction-sections) +add_compile_options(-DELEN=64) + +add_library(benchmark ${CACHEPOOL_TESTS_COMMON_DIR}/benchmark/benchmark.c) +add_library(spin_lock ${CACHEPOOL_TESTS_COMMON_DIR}/benchmark/spin_lock.c) +add_library(mcs_lock ${CACHEPOOL_TESTS_COMMON_DIR}/benchmark/mcs_lock.c) + +enable_testing() +set(SNITCH_TEST_PREFIX "") + +# Macro to generate golden values +macro(add_spatz_test_zeroParam name file) + set(target_name ${name}) + add_snitch_test(${target_name} ${file}) + target_link_libraries(test-${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) +endmacro() + +macro(add_spatz_test_oneParam name file param1) + set(target_name ${name}_M${param1}) + add_snitch_test(${target_name} ${file}) + target_link_libraries(test-${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) + target_compile_definitions(test-${SNITCH_TEST_PREFIX}${target_name} PUBLIC DATAHEADER="data/data_${param1}.h") +endmacro() + +macro(add_spatz_test_twoParam name file param1 param2) + set(target_name ${name}_M${param1}_N${param2}) + add_snitch_test(${target_name} ${file}) + target_link_libraries(test-${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) + target_compile_definitions(test-${SNITCH_TEST_PREFIX}${target_name} PUBLIC DATAHEADER="data/data_${param1}_${param2}.h") +endmacro() + +macro(add_spatz_test_threeParam name file param1 param2 param3) + set(target_name ${name}_M${param1}_N${param2}_K${param3}) + add_snitch_test(${target_name} ${file}) + target_link_libraries(test-${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) + target_compile_definitions(test-${SNITCH_TEST_PREFIX}${target_name} PUBLIC DATAHEADER="data/data_${param1}_${param2}_${param3}.h") +endmacro() diff --git a/software/kernels/fp/CMakeLists.txt b/software/kernels/fp/CMakeLists.txt new file mode 100644 index 00000000..f1053a29 --- /dev/null +++ b/software/kernels/fp/CMakeLists.txt @@ -0,0 +1,30 @@ +# Copyright 2025 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +add_library(fdotp-32b fdotp-32b/kernel/fdotp.c) +add_library(fmatmul-32b fmatmul-32b/kernel/fmatmul.c) + +add_spatz_test_oneParam(fdotp-32b fdotp-32b/main.c 8192) +add_spatz_test_oneParam(fdotp-32b fdotp-32b/main.c 32768) +add_spatz_test_oneParam(fdotp-32b fdotp-32b/main.c 65536) + +add_spatz_test_threeParam(gemv gemv/main.c 128 128 32) +add_spatz_test_threeParam(gemv gemv/main.c 256 128 32) +add_spatz_test_threeParam(gemv gemv/main.c 512 128 32) +add_spatz_test_threeParam(gemv gemv/main.c 1024 128 32) + +add_spatz_test_threeParam(gemv-opt gemv-opt/main.c 128 128 32) +add_spatz_test_threeParam(gemv-opt gemv-opt/main.c 256 128 32) +add_spatz_test_threeParam(gemv-opt gemv-opt/main.c 512 128 32) +add_spatz_test_threeParam(gemv-opt gemv-opt/main.c 1024 128 32) + +add_spatz_test_threeParam(fmatmul-32b fmatmul-32b/main.c 32 32 32) +add_spatz_test_threeParam(fmatmul-32b fmatmul-32b/main.c 64 64 64) +add_spatz_test_threeParam(fmatmul-32b fmatmul-32b/main.c 128 128 128) +add_spatz_test_threeParam(fmatmul-32b fmatmul-32b/main.c 1024 32 32) +add_spatz_test_threeParam(fmatmul-32b fmatmul-32b/main.c 1024 64 64) + +add_spatz_test_twoParam(fft-32b fft-32b/main.c 256 4) +add_spatz_test_twoParam(fft-32b fft-32b/main.c 1024 4) +add_spatz_test_twoParam(fft-32b fft-32b/main.c 1024 16) diff --git a/software/tests/fdotp-32b/data/.gitignore b/software/kernels/fp/fdotp-32b/data/.gitignore similarity index 100% rename from software/tests/fdotp-32b/data/.gitignore rename to software/kernels/fp/fdotp-32b/data/.gitignore diff --git a/software/tests/fdotp-32b/data/layer.h b/software/kernels/fp/fdotp-32b/data/layer.h similarity index 100% rename from software/tests/fdotp-32b/data/layer.h rename to software/kernels/fp/fdotp-32b/data/layer.h diff --git a/software/tests/fdotp-32b/kernel/fdotp.c b/software/kernels/fp/fdotp-32b/kernel/fdotp.c similarity index 100% rename from software/tests/fdotp-32b/kernel/fdotp.c rename to software/kernels/fp/fdotp-32b/kernel/fdotp.c diff --git a/software/tests/fdotp-32b/kernel/fdotp.h b/software/kernels/fp/fdotp-32b/kernel/fdotp.h similarity index 100% rename from software/tests/fdotp-32b/kernel/fdotp.h rename to software/kernels/fp/fdotp-32b/kernel/fdotp.h diff --git a/software/tests/fdotp-32b/main.c b/software/kernels/fp/fdotp-32b/main.c similarity index 83% rename from software/tests/fdotp-32b/main.c rename to software/kernels/fp/fdotp-32b/main.c index 5492ff31..1f10da72 100644 --- a/software/tests/fdotp-32b/main.c +++ b/software/kernels/fp/fdotp-32b/main.c @@ -24,8 +24,9 @@ #include "kernel/fdotp.c" int main() { - const uint32_t num_cores = snrt_cluster_core_num(); - const uint32_t cid = snrt_cluster_core_idx(); + const uint32_t num_cores = snrt_cluster_vpu_num(); + const uint32_t cid = snrt_cluster_vpu_idx(); + const int is_primary = snrt_cluster_is_primary(); const uint32_t measure_iter = 3; @@ -54,7 +55,7 @@ int main() { } else if (lmul_max == 1) { lmul = 1; } else { - if (cid == 0) { + if (is_primary && cid == 0) { printf("FATAL: Problem size too small!\n"); return -2; } @@ -67,7 +68,7 @@ int main() { if ((elem_per_round * num_cores) < (l2_block_elem * l2_channel)) { - if (cid == 0) { + if (is_primary && cid == 0) { printf("Warning: Current scheme cannot utilize all bandwidth!\n"); } } @@ -93,14 +94,14 @@ int main() { float *a_int = dotp_A_dram + cid * elem_per_round; float *b_int = dotp_B_dram + cid * elem_per_round; - if (cid == 0) { + if (is_primary && cid == 0) { printf("lmul:%u, elem:%u, offs:%u, iter:%u\n", lmul, elem_per_round, elem_jump_per_round, rounds); } for (int iter = 0; iter < measure_iter; iter ++) { // Start dump - if (cid == 0) + if (is_primary && cid == 0) start_kernel(); snrt_cluster_hw_barrier(); @@ -111,18 +112,20 @@ int main() { // Calculate dotp float acc; - if (lmul >= 8) - acc = fdotp_v32b_lmul8(a_int, b_int, elem_jump_per_round, elem_per_round, rounds); - else if (lmul >= 4) - acc = fdotp_v32b_lmul4(a_int, b_int, elem_jump_per_round, elem_per_round, rounds); - else if (lmul >= 2) - acc = fdotp_v32b_lmul2(a_int, b_int, elem_jump_per_round, elem_per_round, rounds); - else if (lmul >= 1) - acc = fdotp_v32b_lmul1(a_int, b_int, elem_jump_per_round, elem_per_round, rounds); - else - return -3; + if (is_primary) { + if (lmul >= 8) + acc = fdotp_v32b_lmul8(a_int, b_int, elem_jump_per_round, elem_per_round, rounds); + else if (lmul >= 4) + acc = fdotp_v32b_lmul4(a_int, b_int, elem_jump_per_round, elem_per_round, rounds); + else if (lmul >= 2) + acc = fdotp_v32b_lmul2(a_int, b_int, elem_jump_per_round, elem_per_round, rounds); + else if (lmul >= 1) + acc = fdotp_v32b_lmul1(a_int, b_int, elem_jump_per_round, elem_per_round, rounds); + else + return -3; - result[cid] = acc; + result[cid] = acc; + } // Make sure spatz has finished writing snrt_fence_spatz(); @@ -131,7 +134,7 @@ int main() { snrt_cluster_hw_barrier(); // End timer and check if new best runtime - if (cid == 0) { + if (is_primary && cid == 0) { timer_tmp = benchmark_get_cycle() - timer_tmp; timer = (timer < timer_tmp) ? timer : timer_tmp; if (iter == 0) @@ -144,7 +147,7 @@ int main() { const uint32_t red_group = 4; // Level 1: lead core of each group accumulates its group - if (cid % red_group == 0) { + if (is_primary && cid % red_group == 0) { for (uint32_t i = 1; i < red_group && (cid + i) < num_cores; ++i) acc += result[cid + i]; result[cid] = acc; @@ -153,7 +156,7 @@ int main() { snrt_cluster_hw_barrier(); // Level 2: core 0 sums all group results - if (cid == 0) { + if (is_primary && cid == 0) { for (uint32_t g = red_group; g < num_cores; g += red_group) acc += result[g]; result[0] = acc; @@ -161,10 +164,8 @@ int main() { } - snrt_cluster_hw_barrier(); - // Check and display results - if (cid == 0) { + if (is_primary && cid == 0) { // The timer did not count the reduction time uint32_t performance = 1000 * 2 * dotp_l.M / timer; uint32_t perf_iter1 = 1000 * 2 * dotp_l.M / timer_iter1; @@ -181,7 +182,7 @@ int main() { performance, utilization); } - if (cid == 0) { + if (is_primary && cid == 0) { if (fp_check(result[0], dotp_result*measure_iter)) { printf("Check Failed!\n"); printf("Calc:"); diff --git a/software/tests/fdotp-32b/script/data_32768.json b/software/kernels/fp/fdotp-32b/script/data_32768.json similarity index 100% rename from software/tests/fdotp-32b/script/data_32768.json rename to software/kernels/fp/fdotp-32b/script/data_32768.json diff --git a/software/tests/fdotp-32b/script/data_65536.json b/software/kernels/fp/fdotp-32b/script/data_65536.json similarity index 100% rename from software/tests/fdotp-32b/script/data_65536.json rename to software/kernels/fp/fdotp-32b/script/data_65536.json diff --git a/software/tests/fdotp-32b/script/data_8192.json b/software/kernels/fp/fdotp-32b/script/data_8192.json similarity index 100% rename from software/tests/fdotp-32b/script/data_8192.json rename to software/kernels/fp/fdotp-32b/script/data_8192.json diff --git a/software/tests/fdotp-32b/script/gen_data.py b/software/kernels/fp/fdotp-32b/script/gen_data.py similarity index 100% rename from software/tests/fdotp-32b/script/gen_data.py rename to software/kernels/fp/fdotp-32b/script/gen_data.py diff --git a/software/tests/fft-32b/data/.gitignore b/software/kernels/fp/fft-32b/data/.gitignore similarity index 100% rename from software/tests/fft-32b/data/.gitignore rename to software/kernels/fp/fft-32b/data/.gitignore diff --git a/software/tests/fft-32b/data/data_1024_4.h b/software/kernels/fp/fft-32b/data/data_1024_4.h similarity index 100% rename from software/tests/fft-32b/data/data_1024_4.h rename to software/kernels/fp/fft-32b/data/data_1024_4.h diff --git a/software/tests/fft-32b/kernel/fft.c b/software/kernels/fp/fft-32b/kernel/fft.c similarity index 100% rename from software/tests/fft-32b/kernel/fft.c rename to software/kernels/fp/fft-32b/kernel/fft.c diff --git a/software/tests/fft-32b/kernel/fft.h b/software/kernels/fp/fft-32b/kernel/fft.h similarity index 100% rename from software/tests/fft-32b/kernel/fft.h rename to software/kernels/fp/fft-32b/kernel/fft.h diff --git a/software/tests/fft-32b/main.c b/software/kernels/fp/fft-32b/main.c similarity index 93% rename from software/tests/fft-32b/main.c rename to software/kernels/fp/fft-32b/main.c index 8976a9ce..233ee261 100644 --- a/software/tests/fft-32b/main.c +++ b/software/kernels/fp/fft-32b/main.c @@ -48,9 +48,10 @@ int main() { const int measure_iter = 3; // twiddle layout: [re_p1, im_p1, re_p2, im_p2] - const uint32_t num_cores = snrt_cluster_core_num(); - const uint32_t cid = snrt_cluster_core_idx(); - + const uint32_t num_cores = snrt_cluster_vpu_num(); + const uint32_t cid = snrt_cluster_vpu_idx(); + const int is_primary = snrt_cluster_is_primary(); + snrt_cluster_hw_barrier(); const uint32_t NFFTpc = NFFT / active_cores; // 32-bit floating, 4 byte distance in memory @@ -89,7 +90,7 @@ int main() { uint32_t ierror = 0; for (int iter = 0; iter < measure_iter; iter++) { - if (cid == 0) { + if (is_primary && cid == 0) { start_kernel(); } @@ -97,12 +98,12 @@ int main() { snrt_cluster_hw_barrier(); // Start timer - if (cid == 0) { + if (is_primary && cid == 0) { timer_tmp = benchmark_get_cycle(); } for (uint32_t i = 0; i < log2_nfft1; i ++) { - if (cid < active_cores) { + if (is_primary && cid < active_cores) { fft_p1(src_p1, buf_p1, twi_p1, NFFT, NTWI_P1, cid, active_cores, i, len); // each round will use half the twiddle than previous round // the first round needs re/im NFFT/2 twiddles @@ -116,7 +117,7 @@ int main() { snrt_cluster_hw_barrier(); } - if (cid < active_cores) { + if (is_primary && cid < active_cores) { // Fall back into the single-core case // Each core just do a FFT on (NFFT >> stage_in_P1) data if (p2_switch) { @@ -131,7 +132,7 @@ int main() { snrt_cluster_hw_barrier(); // End timer and check if new best runtime - if (cid == 0) { + if (is_primary && cid == 0) { timer_tmp = benchmark_get_cycle() - timer_tmp; timer = (timer < timer_tmp) ? timer : timer_tmp; if (iter == 0) @@ -145,7 +146,7 @@ int main() { l1d_cluster_flush(); } - if (cid == 0) { + if (is_primary && cid == 0) { if ((iter == 0) && CHECK) { // Verify the real part for (unsigned int i = 0; i < NFFT; i++) { @@ -168,9 +169,9 @@ int main() { snrt_cluster_hw_barrier(); } - + // Display runtime - if (cid == 0) { + if (is_primary && cid == 0) { // Each stage requires: // 2 add, 2 sub, 2 mul, 2 macc/msac // in total 10 operations on NFFT/2 real and NFFT/2 im elements diff --git a/software/tests/fft-32b/script/data_1024_16.json b/software/kernels/fp/fft-32b/script/data_1024_16.json similarity index 100% rename from software/tests/fft-32b/script/data_1024_16.json rename to software/kernels/fp/fft-32b/script/data_1024_16.json diff --git a/software/tests/fft-32b/script/data_256_4.json b/software/kernels/fp/fft-32b/script/data_256_4.json similarity index 100% rename from software/tests/fft-32b/script/data_256_4.json rename to software/kernels/fp/fft-32b/script/data_256_4.json diff --git a/software/tests/fft-32b/script/fft_1024_4.json b/software/kernels/fp/fft-32b/script/fft_1024_4.json similarity index 100% rename from software/tests/fft-32b/script/fft_1024_4.json rename to software/kernels/fp/fft-32b/script/fft_1024_4.json diff --git a/software/tests/fft-32b/script/gen_data.py b/software/kernels/fp/fft-32b/script/gen_data.py similarity index 100% rename from software/tests/fft-32b/script/gen_data.py rename to software/kernels/fp/fft-32b/script/gen_data.py diff --git a/software/tests/fmatmul-32b/data/.gitignore b/software/kernels/fp/fmatmul-32b/data/.gitignore similarity index 100% rename from software/tests/fmatmul-32b/data/.gitignore rename to software/kernels/fp/fmatmul-32b/data/.gitignore diff --git a/software/tests/fmatmul-32b/data/layer.h b/software/kernels/fp/fmatmul-32b/data/layer.h similarity index 100% rename from software/tests/fmatmul-32b/data/layer.h rename to software/kernels/fp/fmatmul-32b/data/layer.h diff --git a/software/tests/fmatmul-32b/kernel/fmatmul.c b/software/kernels/fp/fmatmul-32b/kernel/fmatmul.c similarity index 100% rename from software/tests/fmatmul-32b/kernel/fmatmul.c rename to software/kernels/fp/fmatmul-32b/kernel/fmatmul.c diff --git a/software/tests/fmatmul-32b/kernel/fmatmul.h b/software/kernels/fp/fmatmul-32b/kernel/fmatmul.h similarity index 100% rename from software/tests/fmatmul-32b/kernel/fmatmul.h rename to software/kernels/fp/fmatmul-32b/kernel/fmatmul.h diff --git a/software/tests/fmatmul-32b/main.c b/software/kernels/fp/fmatmul-32b/main.c similarity index 85% rename from software/tests/fmatmul-32b/main.c rename to software/kernels/fp/fmatmul-32b/main.c index c59db8ca..fa308a0b 100644 --- a/software/tests/fmatmul-32b/main.c +++ b/software/kernels/fp/fmatmul-32b/main.c @@ -59,9 +59,10 @@ int verify_matrix(float *matrix, const float *checksum, } int main() { - const unsigned int num_cores = snrt_cluster_core_num(); - const unsigned int num_cores_per_tile = snrt_cluster_core_per_tile(); - const unsigned int cid = snrt_cluster_core_idx(); + const unsigned int num_cores = snrt_cluster_vpu_num(); + const unsigned int num_cores_per_tile = snrt_cluster_vpu_per_tile(); + const unsigned int cid = snrt_cluster_vpu_idx(); + const int is_primary = snrt_cluster_is_primary(); #if MEAS_1ITER == 1 const int measure_iter = 1; @@ -83,7 +84,7 @@ int main() { // Allocate and zero the error array while the cache is still in shared mode, // so the pointer write and slot initialisation are visible to all cores. - if (cid == 0) { + if (is_primary && cid == 0) { error_arr = (int *)snrt_malloc(active_cores * sizeof(int)); for (unsigned int i = 0; i < active_cores; i++) error_arr[i] = 0; @@ -117,7 +118,7 @@ int main() { // Initialize matrices #ifdef DEBUG - if (cid == 0) { + if (is_primary && cid == 0) { printf ("a:%x\n", a); printf ("b:%x\n", b); printf ("c:%x\n", c); @@ -134,21 +135,23 @@ int main() { // Calculate matmul for (unsigned int i = 0; i < measure_iter; ++i) { // Start dump - if (cid == 0) { + if (is_primary && cid == 0) { start_kernel(); } // Start timer timer_start = benchmark_get_cycle(); - if (kernel_size == 2) { - matmul_2xVL(gemm_C_dram, gemm_A_dram, gemm_B_dram, m_start, m_end, gemm_l.K, gemm_l.N, p_start, p_end); - } else if (kernel_size == 4) { - matmul_4xVL(gemm_C_dram, gemm_A_dram, gemm_B_dram, m_start, m_end, gemm_l.K, gemm_l.N, p_start, p_end); - } else if (kernel_size == 8) { - matmul_8xVL(gemm_C_dram, gemm_A_dram, gemm_B_dram, m_start, m_end, gemm_l.K, gemm_l.N, p_start, p_end); - } else { - return -1; + if (is_primary) { + if (kernel_size == 2) { + matmul_2xVL(gemm_C_dram, gemm_A_dram, gemm_B_dram, m_start, m_end, gemm_l.K, gemm_l.N, p_start, p_end); + } else if (kernel_size == 4) { + matmul_4xVL(gemm_C_dram, gemm_A_dram, gemm_B_dram, m_start, m_end, gemm_l.K, gemm_l.N, p_start, p_end); + } else if (kernel_size == 8) { + matmul_8xVL(gemm_C_dram, gemm_A_dram, gemm_B_dram, m_start, m_end, gemm_l.K, gemm_l.N, p_start, p_end); + } else { + return -1; + } } // Wait for all cores to finish @@ -157,7 +160,7 @@ int main() { // End timer and check if new best runtime timer_end = benchmark_get_cycle(); unsigned int timer_temp = timer_end - timer_start; - if (cid == 0) { + if (is_primary && cid == 0) { if (timer_temp < timer) { timer = timer_temp; if (i == 0) @@ -167,7 +170,7 @@ int main() { } if (i == 0) { - if (cid < active_cores) { + if (is_primary && cid < active_cores) { float *check_C = gemm_C_dram + cid * (gemm_l.M / active_cores) * gemm_l.N; float *check_gold = (float *)gemm_checksum + cid * (gemm_l.M / active_cores); @@ -177,7 +180,7 @@ int main() { snrt_cluster_hw_barrier(); - if (cid == 0) { + if (is_primary && cid == 0) { if (error_arr[0] != 0) printf("Core 0 error %d\n", error_arr[0]); @@ -196,7 +199,7 @@ int main() { } // Check and display results - if (cid == 0) { + if (is_primary && cid == 0) { long unsigned int performance = 1000 * 2 * gemm_l.M * gemm_l.N * gemm_l.K / timer; long unsigned int utilization = performance / (2 * active_cores * 4); diff --git a/software/tests/fmatmul-32b/script/data_1024_32_32.json b/software/kernels/fp/fmatmul-32b/script/data_1024_32_32.json similarity index 100% rename from software/tests/fmatmul-32b/script/data_1024_32_32.json rename to software/kernels/fp/fmatmul-32b/script/data_1024_32_32.json diff --git a/software/tests/fmatmul-32b/script/data_1024_64_64.json b/software/kernels/fp/fmatmul-32b/script/data_1024_64_64.json similarity index 100% rename from software/tests/fmatmul-32b/script/data_1024_64_64.json rename to software/kernels/fp/fmatmul-32b/script/data_1024_64_64.json diff --git a/software/tests/fmatmul-32b/script/data_128_128_128.json b/software/kernels/fp/fmatmul-32b/script/data_128_128_128.json similarity index 100% rename from software/tests/fmatmul-32b/script/data_128_128_128.json rename to software/kernels/fp/fmatmul-32b/script/data_128_128_128.json diff --git a/software/tests/fmatmul-32b/script/data_32_32_32.json b/software/kernels/fp/fmatmul-32b/script/data_32_32_32.json similarity index 100% rename from software/tests/fmatmul-32b/script/data_32_32_32.json rename to software/kernels/fp/fmatmul-32b/script/data_32_32_32.json diff --git a/software/tests/fmatmul-32b/script/data_64_64_64.json b/software/kernels/fp/fmatmul-32b/script/data_64_64_64.json similarity index 100% rename from software/tests/fmatmul-32b/script/data_64_64_64.json rename to software/kernels/fp/fmatmul-32b/script/data_64_64_64.json diff --git a/software/tests/fmatmul-32b/script/gen_data.py b/software/kernels/fp/fmatmul-32b/script/gen_data.py similarity index 100% rename from software/tests/fmatmul-32b/script/gen_data.py rename to software/kernels/fp/fmatmul-32b/script/gen_data.py diff --git a/software/tests/gemv-opt/data/.gitignore b/software/kernels/fp/gemv-opt/data/.gitignore similarity index 100% rename from software/tests/gemv-opt/data/.gitignore rename to software/kernels/fp/gemv-opt/data/.gitignore diff --git a/software/tests/gemv-opt/data/layer.h b/software/kernels/fp/gemv-opt/data/layer.h similarity index 100% rename from software/tests/gemv-opt/data/layer.h rename to software/kernels/fp/gemv-opt/data/layer.h diff --git a/software/tests/gemv-opt/kernel/gemv.c b/software/kernels/fp/gemv-opt/kernel/gemv.c similarity index 100% rename from software/tests/gemv-opt/kernel/gemv.c rename to software/kernels/fp/gemv-opt/kernel/gemv.c diff --git a/software/tests/gemv-opt/kernel/gemv.h b/software/kernels/fp/gemv-opt/kernel/gemv.h similarity index 100% rename from software/tests/gemv-opt/kernel/gemv.h rename to software/kernels/fp/gemv-opt/kernel/gemv.h diff --git a/software/tests/gemv-opt/main.c b/software/kernels/fp/gemv-opt/main.c similarity index 89% rename from software/tests/gemv-opt/main.c rename to software/kernels/fp/gemv-opt/main.c index 7136b597..ae1b2dc3 100644 --- a/software/tests/gemv-opt/main.c +++ b/software/kernels/fp/gemv-opt/main.c @@ -65,8 +65,9 @@ int main() { T *b; T *result; - const uint32_t num_cores = snrt_cluster_core_num(); - const uint32_t cid = snrt_cluster_core_idx(); + const uint32_t num_cores = snrt_cluster_vpu_num(); + const uint32_t cid = snrt_cluster_vpu_idx(); + const int is_primary = snrt_cluster_is_primary(); // How many column of data each core will work on // This will determine the vlen of the calculation @@ -90,7 +91,7 @@ int main() { } else if (lmul_max == 1) { lmul = 1; } else { - if (cid == 0) { + if (is_primary && cid == 0) { printf("FATAL: Problem size too small!\n"); } snrt_cluster_hw_barrier(); @@ -103,7 +104,7 @@ int main() { // offset bits in unit of byte (address) uint32_t offset = 31 - __builtin_clz(m_core * elem_width/8); - if (cid == 0) { + if (is_primary && cid == 0) { result_dram = (T *)snrt_malloc(gemv_l.M * sizeof(T)); } @@ -131,7 +132,7 @@ int main() { // If our block size is a 4-times multiple of l2 block size // Then we will always visiting the same L2 channel if (block_elem_core < l2_block_elem) { - if (cid == 0) { + if (is_primary && cid == 0) { printf("FATAL: Current scheme cannot utilize all bandwidth!\n"); printf("Core block size:%u, DRAM block size:%u\n", block_elem_core, l2_block_elem); } @@ -166,7 +167,7 @@ int main() { uint32_t n_core = gemv_l.N - col_shift; // first segment length uint32_t comp_size = col_shift; // second segment length - if (cid == 0) { + if (is_primary && cid == 0) { printf("lmul:%u, mcore:%u\n", lmul, m_core); } @@ -177,26 +178,28 @@ int main() { for (int i = 0; i < 3; i++) { // Start dump - if (cid == 0) { + if (is_primary && cid == 0) { start_kernel(); // Start timer timer_start = benchmark_get_cycle(); } // Calculate gemv - if (lmul == 4) { - gemv_v32b_m4(a_core, b_core, r_core, a_offset, b_offset, gemv_l.M, n_core, m_core, comp_size); - } else if (lmul == 2) { - gemv_v32b_m2(a_core, b_core, r_core, a_offset, b_offset, gemv_l.M, n_core, m_core, comp_size); - } else if (lmul == 1) { - gemv_v32b_m1(a_core, b_core, r_core, a_offset, b_offset, gemv_l.M, n_core, m_core, comp_size); + if (is_primary) { + if (lmul == 4) { + gemv_v32b_m4(a_core, b_core, r_core, a_offset, b_offset, gemv_l.M, n_core, m_core, comp_size); + } else if (lmul == 2) { + gemv_v32b_m2(a_core, b_core, r_core, a_offset, b_offset, gemv_l.M, n_core, m_core, comp_size); + } else if (lmul == 1) { + gemv_v32b_m1(a_core, b_core, r_core, a_offset, b_offset, gemv_l.M, n_core, m_core, comp_size); + } } // Wait for all cores to finish snrt_cluster_hw_barrier(); - if (cid == 0) { + if (is_primary && cid == 0) { // End timer and check if new best runtime timer_end = benchmark_get_cycle(); unsigned int timer_temp = timer_end - timer_start; @@ -226,7 +229,7 @@ int main() { } // Check and display results - if (cid == 0) { + if (is_primary && cid == 0) { long unsigned int performance = 1000 * 2 * gemv_l.M * gemv_l.N / timer; long unsigned int utilization = performance / (2 * num_cores * 4 * (4 / sizeof(T))); diff --git a/software/tests/gemv-opt/script/data_1024_128_32.json b/software/kernels/fp/gemv-opt/script/data_1024_128_32.json similarity index 100% rename from software/tests/gemv-opt/script/data_1024_128_32.json rename to software/kernels/fp/gemv-opt/script/data_1024_128_32.json diff --git a/software/tests/gemv-opt/script/data_128_128_32.json b/software/kernels/fp/gemv-opt/script/data_128_128_32.json similarity index 100% rename from software/tests/gemv-opt/script/data_128_128_32.json rename to software/kernels/fp/gemv-opt/script/data_128_128_32.json diff --git a/software/tests/gemv-opt/script/data_256_128_32.json b/software/kernels/fp/gemv-opt/script/data_256_128_32.json similarity index 100% rename from software/tests/gemv-opt/script/data_256_128_32.json rename to software/kernels/fp/gemv-opt/script/data_256_128_32.json diff --git a/software/tests/gemv-opt/script/data_512_128_32.json b/software/kernels/fp/gemv-opt/script/data_512_128_32.json similarity index 100% rename from software/tests/gemv-opt/script/data_512_128_32.json rename to software/kernels/fp/gemv-opt/script/data_512_128_32.json diff --git a/software/tests/gemv-opt/script/gen_data.py b/software/kernels/fp/gemv-opt/script/gen_data.py similarity index 100% rename from software/tests/gemv-opt/script/gen_data.py rename to software/kernels/fp/gemv-opt/script/gen_data.py diff --git a/software/tests/gemv/data/.gitignore b/software/kernels/fp/gemv/data/.gitignore similarity index 100% rename from software/tests/gemv/data/.gitignore rename to software/kernels/fp/gemv/data/.gitignore diff --git a/software/tests/gemv/data/layer.h b/software/kernels/fp/gemv/data/layer.h similarity index 100% rename from software/tests/gemv/data/layer.h rename to software/kernels/fp/gemv/data/layer.h diff --git a/software/tests/gemv/kernel/gemv.c b/software/kernels/fp/gemv/kernel/gemv.c similarity index 100% rename from software/tests/gemv/kernel/gemv.c rename to software/kernels/fp/gemv/kernel/gemv.c diff --git a/software/tests/gemv/kernel/gemv.h b/software/kernels/fp/gemv/kernel/gemv.h similarity index 100% rename from software/tests/gemv/kernel/gemv.h rename to software/kernels/fp/gemv/kernel/gemv.h diff --git a/software/tests/gemv/main.c b/software/kernels/fp/gemv/main.c similarity index 86% rename from software/tests/gemv/main.c rename to software/kernels/fp/gemv/main.c index 1cb5b89b..36a63280 100644 --- a/software/tests/gemv/main.c +++ b/software/kernels/fp/gemv/main.c @@ -55,8 +55,9 @@ int main() { T *b; T *result; - const unsigned int num_cores = snrt_cluster_core_num(); - const unsigned int cid = snrt_cluster_core_idx(); + const unsigned int num_cores = snrt_cluster_vpu_num(); + const unsigned int cid = snrt_cluster_vpu_idx(); + const int is_primary = snrt_cluster_is_primary(); unsigned int m_core = gemv_l.M / num_cores; @@ -71,7 +72,7 @@ int main() { a = gemv_A_dram; b = gemv_B_dram; - if (cid == 0) { + if (is_primary && cid == 0) { result_dram = (T *)snrt_malloc(gemv_l.M * sizeof(T)); } @@ -96,26 +97,28 @@ int main() { for (int i = 0; i < 3; i++) { // Start dump - if (cid == 0) { + if (is_primary && cid == 0) { start_kernel(); // Start timer timer_start = benchmark_get_cycle(); } // Calculate gemv - if (sizeof(T) == 8) - // does not support 64b - return -2; - else if (sizeof(T) == 4) - gemv_v32b_m4(a_core, b, result_core, gemv_l.M, m_core, gemv_l.N); - else - gemv_v16b_m4(a_core, b, result_core, gemv_l.M, m_core, gemv_l.N); + if (is_primary) { + if (sizeof(T) == 8) + // does not support 64b + return -2; + else if (sizeof(T) == 4) + gemv_v32b_m4(a_core, b, result_core, gemv_l.M, m_core, gemv_l.N); + else + gemv_v16b_m4(a_core, b, result_core, gemv_l.M, m_core, gemv_l.N); + } // Wait for all cores to finish snrt_cluster_hw_barrier(); - if (cid == 0) { + if (is_primary && cid == 0) { // End timer and check if new best runtime timer_end = benchmark_get_cycle(); unsigned int timer_temp = timer_end - timer_start; @@ -137,7 +140,7 @@ int main() { l1d_cluster_flush(); } - if (cid == 0) { + if (is_primary && cid == 0) { if (i == 0) { for (uint32_t j = 0; j < gemv_l.M; j++) { if (fp_check(&result[j], &gemv_result[j])) { @@ -156,7 +159,7 @@ int main() { } // Check and display results - if (cid == 0) { + if (is_primary && cid == 0) { long unsigned int performance = 1000 * 2 * gemv_l.M * gemv_l.N / timer; long unsigned int utilization = performance / (2 * num_cores * 4 * (4 / sizeof(T))); diff --git a/software/tests/gemv/script/data_1024_128_32.json b/software/kernels/fp/gemv/script/data_1024_128_32.json similarity index 100% rename from software/tests/gemv/script/data_1024_128_32.json rename to software/kernels/fp/gemv/script/data_1024_128_32.json diff --git a/software/tests/gemv/script/data_128_128_32.json b/software/kernels/fp/gemv/script/data_128_128_32.json similarity index 100% rename from software/tests/gemv/script/data_128_128_32.json rename to software/kernels/fp/gemv/script/data_128_128_32.json diff --git a/software/tests/gemv/script/data_256_128_32.json b/software/kernels/fp/gemv/script/data_256_128_32.json similarity index 100% rename from software/tests/gemv/script/data_256_128_32.json rename to software/kernels/fp/gemv/script/data_256_128_32.json diff --git a/software/tests/gemv/script/data_512_128_32.json b/software/kernels/fp/gemv/script/data_512_128_32.json similarity index 100% rename from software/tests/gemv/script/data_512_128_32.json rename to software/kernels/fp/gemv/script/data_512_128_32.json diff --git a/software/tests/gemv/script/gen_data.py b/software/kernels/fp/gemv/script/gen_data.py similarity index 100% rename from software/tests/gemv/script/gen_data.py rename to software/kernels/fp/gemv/script/gen_data.py diff --git a/software/kernels/int/CMakeLists.txt b/software/kernels/int/CMakeLists.txt new file mode 100644 index 00000000..2c01e58f --- /dev/null +++ b/software/kernels/int/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2025 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +add_library(idotp-32b idotp-32b/kernel/idotp.c) + +add_spatz_test_oneParam(idotp-32b idotp-32b/main.c 8192) +add_spatz_test_oneParam(idotp-32b idotp-32b/main.c 32768) diff --git a/software/tests/idotp-32b/data/.gitignore b/software/kernels/int/idotp-32b/data/.gitignore similarity index 100% rename from software/tests/idotp-32b/data/.gitignore rename to software/kernels/int/idotp-32b/data/.gitignore diff --git a/software/tests/idotp-32b/data/layer.h b/software/kernels/int/idotp-32b/data/layer.h similarity index 100% rename from software/tests/idotp-32b/data/layer.h rename to software/kernels/int/idotp-32b/data/layer.h diff --git a/software/tests/idotp-32b/kernel/idotp.c b/software/kernels/int/idotp-32b/kernel/idotp.c similarity index 100% rename from software/tests/idotp-32b/kernel/idotp.c rename to software/kernels/int/idotp-32b/kernel/idotp.c diff --git a/software/tests/idotp-32b/kernel/idotp.h b/software/kernels/int/idotp-32b/kernel/idotp.h similarity index 100% rename from software/tests/idotp-32b/kernel/idotp.h rename to software/kernels/int/idotp-32b/kernel/idotp.h diff --git a/software/tests/idotp-32b/main.c b/software/kernels/int/idotp-32b/main.c similarity index 83% rename from software/tests/idotp-32b/main.c rename to software/kernels/int/idotp-32b/main.c index 576e92bd..160bc393 100644 --- a/software/tests/idotp-32b/main.c +++ b/software/kernels/int/idotp-32b/main.c @@ -18,6 +18,7 @@ #include #include +#include #include #include DATAHEADER @@ -29,6 +30,12 @@ int main() { const uint32_t num_cores = snrt_cluster_core_num(); const uint32_t cid = snrt_cluster_core_idx(); + // Diagnostic: have host 0 of each pair hold the Spatz lock for the whole + // kernel, without gating any of the execution below on it. + if (snrt_cluster_is_primary()) { + spatz_lock_acquire(); + } + const int measure_iter = 2; // Byte-level interleaving for DRAM @@ -50,13 +57,13 @@ int main() { uint32_t offset = 31 - __builtin_clz(dim * sizeof(int)); // Set xbar policy - l1d_xbar_config(offset); + // l1d_xbar_config(offset); if (cid == 0) { printf ("round:%u, lmul:%u, dim:%u\n", rounds, lmul, dim); } - snrt_cluster_hw_barrier(); + snrt_cluster_host0_barrier(); // Reset timer uint32_t timer = (uint32_t)-1; @@ -66,12 +73,13 @@ int main() { int *a_int = dotp_A_dram + dim * cid; int *b_int = dotp_B_dram + dim * cid; + for (int iter = 0; iter < measure_iter; iter ++) { // Start dump if (cid == 0) start_kernel(); - snrt_cluster_hw_barrier(); + // snrt_cluster_hw_barrier(); // Start timer timer_tmp = benchmark_get_cycle(); @@ -92,8 +100,9 @@ int main() { result[cid] = acc; - // Wait for all cores to finish - snrt_cluster_hw_barrier(); + // Host 0 only: this kernel doesn't care about host 1's progress or + // data at all, so no barrier or read ever involves it. + snrt_cluster_host0_barrier(); // End timer and check if new best runtime if (cid == 0) { @@ -105,10 +114,16 @@ int main() { stop_kernel(); } - // Final reduction +#if SNRT_NUM_SCALAR_PER_CORE == 2 + const uint32_t pair_stride = 2; +#else + const uint32_t pair_stride = 1; +#endif + + // Final reduction over host 0's own slots only. if (cid == 0) { - // timer_tmp = benchmark_get_cycle() - timer_tmp; - for (uint32_t i = 1; i < num_cores; ++i) + acc = result[0]; + for (uint32_t i = pair_stride; i < num_cores; i += pair_stride) acc += result[i]; result[0] = acc; @@ -116,10 +131,9 @@ int main() { printf("results:%u\n", result[0]); #endif } - } - snrt_cluster_hw_barrier(); + snrt_cluster_host0_barrier(); // Check and display results if (cid == 0) { @@ -150,7 +164,7 @@ int main() { } // Wait for core 0 to display the results - snrt_cluster_hw_barrier(); + snrt_cluster_host0_barrier(); return 0; } \ No newline at end of file diff --git a/software/tests/idotp-32b/script/data_32768.json b/software/kernels/int/idotp-32b/script/data_32768.json similarity index 100% rename from software/tests/idotp-32b/script/data_32768.json rename to software/kernels/int/idotp-32b/script/data_32768.json diff --git a/software/tests/idotp-32b/script/data_8192.json b/software/kernels/int/idotp-32b/script/data_8192.json similarity index 100% rename from software/tests/idotp-32b/script/data_8192.json rename to software/kernels/int/idotp-32b/script/data_8192.json diff --git a/software/tests/idotp-32b/script/gen_data.py b/software/kernels/int/idotp-32b/script/gen_data.py similarity index 100% rename from software/tests/idotp-32b/script/gen_data.py rename to software/kernels/int/idotp-32b/script/gen_data.py diff --git a/software/rlc/CMakeLists.txt b/software/rlc/CMakeLists.txt new file mode 100644 index 00000000..a9495e07 --- /dev/null +++ b/software/rlc/CMakeLists.txt @@ -0,0 +1,16 @@ +# Copyright 2025 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +add_library(printf_lock multi_producer_single_consumer_double_linked_list/kernel/printf_lock.c) +add_library(llist multi_producer_single_consumer_double_linked_list/kernel/llist.c) +add_library(mm multi_producer_single_consumer_double_linked_list/kernel/mm.c) +add_library(rlc multi_producer_single_consumer_double_linked_list/kernel/rlc.c) +add_library(data_move_vec multi_producer_single_consumer_double_linked_list/kernel/data_move_vec.c) + +# add_snitch_test(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c) +# add_spatz_test_threeParam(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c 1 1350 1000) +# add_spatz_test_threeParam(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c 1 2044 100) +add_spatz_test_threeParam(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c 1 1350 300) +add_spatz_test_threeParam(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c 1 1350 100) +add_spatz_test_threeParam(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c 1 1350 10) diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/data/data_1_1350_10.h b/software/rlc/multi_producer_single_consumer_double_linked_list/data/data_1_1350_10.h similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/data/data_1_1350_10.h rename to software/rlc/multi_producer_single_consumer_double_linked_list/data/data_1_1350_10.h diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/data/data_1_1350_100.h b/software/rlc/multi_producer_single_consumer_double_linked_list/data/data_1_1350_100.h similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/data/data_1_1350_100.h rename to software/rlc/multi_producer_single_consumer_double_linked_list/data/data_1_1350_100.h diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/data/data_1_1350_1000.h b/software/rlc/multi_producer_single_consumer_double_linked_list/data/data_1_1350_1000.h similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/data/data_1_1350_1000.h rename to software/rlc/multi_producer_single_consumer_double_linked_list/data/data_1_1350_1000.h diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/data/data_1_1350_300.h b/software/rlc/multi_producer_single_consumer_double_linked_list/data/data_1_1350_300.h similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/data/data_1_1350_300.h rename to software/rlc/multi_producer_single_consumer_double_linked_list/data/data_1_1350_300.h diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/data/data_1_2044_100.h b/software/rlc/multi_producer_single_consumer_double_linked_list/data/data_1_2044_100.h similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/data/data_1_2044_100.h rename to software/rlc/multi_producer_single_consumer_double_linked_list/data/data_1_2044_100.h diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/data/rlc_flowchart b/software/rlc/multi_producer_single_consumer_double_linked_list/data/rlc_flowchart similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/data/rlc_flowchart rename to software/rlc/multi_producer_single_consumer_double_linked_list/data/rlc_flowchart diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/data/rlc_flowchart.png b/software/rlc/multi_producer_single_consumer_double_linked_list/data/rlc_flowchart.png similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/data/rlc_flowchart.png rename to software/rlc/multi_producer_single_consumer_double_linked_list/data/rlc_flowchart.png diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/kernel/data_move_vec.c b/software/rlc/multi_producer_single_consumer_double_linked_list/kernel/data_move_vec.c similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/kernel/data_move_vec.c rename to software/rlc/multi_producer_single_consumer_double_linked_list/kernel/data_move_vec.c diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/kernel/data_move_vec.h b/software/rlc/multi_producer_single_consumer_double_linked_list/kernel/data_move_vec.h similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/kernel/data_move_vec.h rename to software/rlc/multi_producer_single_consumer_double_linked_list/kernel/data_move_vec.h diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/kernel/llist.c b/software/rlc/multi_producer_single_consumer_double_linked_list/kernel/llist.c similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/kernel/llist.c rename to software/rlc/multi_producer_single_consumer_double_linked_list/kernel/llist.c diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/kernel/llist.h b/software/rlc/multi_producer_single_consumer_double_linked_list/kernel/llist.h similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/kernel/llist.h rename to software/rlc/multi_producer_single_consumer_double_linked_list/kernel/llist.h diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/kernel/mm.c b/software/rlc/multi_producer_single_consumer_double_linked_list/kernel/mm.c similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/kernel/mm.c rename to software/rlc/multi_producer_single_consumer_double_linked_list/kernel/mm.c diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/kernel/mm.h b/software/rlc/multi_producer_single_consumer_double_linked_list/kernel/mm.h similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/kernel/mm.h rename to software/rlc/multi_producer_single_consumer_double_linked_list/kernel/mm.h diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/kernel/printf_lock.c b/software/rlc/multi_producer_single_consumer_double_linked_list/kernel/printf_lock.c similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/kernel/printf_lock.c rename to software/rlc/multi_producer_single_consumer_double_linked_list/kernel/printf_lock.c diff --git a/software/tests/mcs-lock/kernel/printf_lock.h b/software/rlc/multi_producer_single_consumer_double_linked_list/kernel/printf_lock.h similarity index 100% rename from software/tests/mcs-lock/kernel/printf_lock.h rename to software/rlc/multi_producer_single_consumer_double_linked_list/kernel/printf_lock.h diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/kernel/rlc.c b/software/rlc/multi_producer_single_consumer_double_linked_list/kernel/rlc.c similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/kernel/rlc.c rename to software/rlc/multi_producer_single_consumer_double_linked_list/kernel/rlc.c diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/kernel/rlc.h b/software/rlc/multi_producer_single_consumer_double_linked_list/kernel/rlc.h similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/kernel/rlc.h rename to software/rlc/multi_producer_single_consumer_double_linked_list/kernel/rlc.h diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/main.c b/software/rlc/multi_producer_single_consumer_double_linked_list/main.c similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/main.c rename to software/rlc/multi_producer_single_consumer_double_linked_list/main.c diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/script/flowchart.py b/software/rlc/multi_producer_single_consumer_double_linked_list/script/flowchart.py similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/script/flowchart.py rename to software/rlc/multi_producer_single_consumer_double_linked_list/script/flowchart.py diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/script/generate_pdcp_pkg.py b/software/rlc/multi_producer_single_consumer_double_linked_list/script/generate_pdcp_pkg.py similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/script/generate_pdcp_pkg.py rename to software/rlc/multi_producer_single_consumer_double_linked_list/script/generate_pdcp_pkg.py diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/script/pdcp_pkg.json b/software/rlc/multi_producer_single_consumer_double_linked_list/script/pdcp_pkg.json similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/script/pdcp_pkg.json rename to software/rlc/multi_producer_single_consumer_double_linked_list/script/pdcp_pkg.json diff --git a/software/snRuntime/CMakeLists.txt b/software/snRuntime/CMakeLists.txt index 5c6b3cba..a489b963 100644 --- a/software/snRuntime/CMakeLists.txt +++ b/software/snRuntime/CMakeLists.txt @@ -49,6 +49,12 @@ if(RUNTIME_TRACE) add_compile_definitions(__SNRT_USE_TRACE) endif() +# Also set/applied in the parent software/CMakeLists.txt so sibling +# directories (e.g. software/tests) see the same define; kept here too for +# standalone snRuntime builds. +set(NUM_SCALAR_PER_CORE "1" CACHE STRING "Snitch harts sharing one Spatz per Core Complex") +add_compile_definitions(SNRT_NUM_SCALAR_PER_CORE=${NUM_SCALAR_PER_CORE}) + include_directories( include vendor @@ -65,6 +71,7 @@ set(sources src/alloc.c src/interrupt.c src/l1cache.c + src/spatz_lock.c ) # platform specific sources diff --git a/software/snRuntime/README.md b/software/snRuntime/README.md index b804918d..5d1208ed 100644 --- a/software/snRuntime/README.md +++ b/software/snRuntime/README.md @@ -9,6 +9,7 @@ snRuntime/ ├── include/ # Public headers — include these in application code │ ├── snrt.h # Master header: topology, barriers, DMA, allocation │ ├── l1cache.h # CachePool L1 data cache management API +│ ├── spatz_lock.h # Dual-scalar Spatz ownership lock API │ ├── cachepool_peripheral.h # Register offsets for the cluster peripheral │ ├── perf_cnt.h # Performance counter API │ ├── team.h # Team/cluster descriptor structs @@ -23,6 +24,7 @@ snRuntime/ │ ├── team.c # Team/topology initialisation │ ├── barrier.c # Hardware and software barrier implementations │ ├── l1cache.c # CachePool L1 cache management (flush, partition, xbar) +│ ├── spatz_lock.c # Dual-scalar Spatz ownership lock (see spatz_lock.h) │ ├── alloc.c # L1 TCDM bump allocator + DRAM linked-list allocator │ ├── memcpy.c # Optimised memcpy │ ├── perf_cnt.c # Performance counter helpers @@ -103,6 +105,47 @@ void l1d_wait(); void l1d_init(uint32_t size); ``` +### Dual-Scalar Spatz Lock — CachePool-specific (`spatz_lock.h`) + +Only meaningful on a `num_scalar_per_core=2` build (`cachepool_cc_dual`), where 2 Snitch +harts share 1 Spatz unit. On a single-scalar-per-CC build every call below is a harmless +no-op that always reports success. + +```c +void spatz_lock_acquire(); // blocks (retries in software) until this hart owns Spatz +void spatz_lock_release(); // must be called by the current owner only +``` + +Both are plain, non-blocking-in-hardware retries under the hood — see `spatz_lock_outcome_t` +below — so a hart can never hang in hardware waiting on the other hart to release. After +`spatz_lock_acquire()` returns, it is always safe to issue vector/FP work immediately: if the +ownership switch is still draining, `acc_mux` (RTL) withholds Spatz access until it completes, +so the next vector/FP instruction just blocks there instead, exactly as if the acquire itself +had blocked. + +For finer control (e.g. to do other work instead of retrying), use the non-blocking primitives +directly: + +```c +typedef enum { + SPATZ_LOCK_FAIL = 0, // denied; hardware made no reservation, safe to retry + SPATZ_LOCK_SUCCESS = 1, // granted now + SPATZ_LOCK_SUCCESS_WAIT = 2, // accepted, completes on its own once drained +} spatz_lock_outcome_t; + +uint32_t spatz_lock_try_acquire(); // single, always-immediate attempt +uint32_t spatz_lock_try_release(); +spatz_lock_outcome_t spatz_lock_outcome(uint32_t raw); // decode the above +``` + +Related topology/sync helpers in `snrt.h`: + +```c +int snrt_cluster_is_primary(); // true for the pair's default owner (even cid) +void snrt_cluster_host0_barrier(); // partial barrier over default owners only +void snrt_cluster_host1_barrier(); // partial barrier over their partners only +``` + ### Performance Counters (`perf_cnt.h`) *TODO: REMOVE* ```c diff --git a/software/snRuntime/include/cachepool_peripheral.h b/software/snRuntime/include/cachepool_peripheral.h index 8bad67a8..9eb2b8eb 100644 --- a/software/snRuntime/include/cachepool_peripheral.h +++ b/software/snRuntime/include/cachepool_peripheral.h @@ -24,22 +24,28 @@ extern "C" { // cores have #define CACHEPOOL_PERIPHERAL_HW_BARRIER_REG_OFFSET 0x0 +// Spatz ownership acquire for dual-Snitch core complexes. Load +#define CACHEPOOL_PERIPHERAL_SPATZ_LOCK_ACQUIRE_REG_OFFSET 0x4 + +// Spatz ownership release for dual-Snitch core complexes. Load +#define CACHEPOOL_PERIPHERAL_SPATZ_LOCK_RELEASE_REG_OFFSET 0x8 + // Controls prefetching of the instruction cache. -#define CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE_REG_OFFSET 0x4 +#define CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE_REG_OFFSET 0xc #define CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE_ICACHE_PREFETCH_ENABLE_BIT 0 // Sets the status of the Spatz cluster. -#define CACHEPOOL_PERIPHERAL_SPATZ_STATUS_REG_OFFSET 0x8 +#define CACHEPOOL_PERIPHERAL_SPATZ_STATUS_REG_OFFSET 0x10 #define CACHEPOOL_PERIPHERAL_SPATZ_STATUS_SPATZ_CLUSTER_PROBE_BIT 0 // Store cycle counts of kernels -#define CACHEPOOL_PERIPHERAL_SPATZ_CYCLE_REG_OFFSET 0xc +#define CACHEPOOL_PERIPHERAL_SPATZ_CYCLE_REG_OFFSET 0x14 // Controls the cluster boot process. -#define CACHEPOOL_PERIPHERAL_CLUSTER_BOOT_CONTROL_REG_OFFSET 0x10 +#define CACHEPOOL_PERIPHERAL_CLUSTER_BOOT_CONTROL_REG_OFFSET 0x18 // End of computation and exit status register -#define CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_REG_OFFSET 0x14 +#define CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_REG_OFFSET 0x1c #define CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_EOC_EXIT_MASK 0xf #define CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_EOC_EXIT_OFFSET 0 #define CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_EOC_EXIT_FIELD \ @@ -48,7 +54,7 @@ extern "C" { .index = CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_EOC_EXIT_OFFSET}) // Controls the configurations of L1 DCache SPM size. -#define CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_REG_OFFSET 0x18 +#define CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_REG_OFFSET 0x20 #define CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_SPM_SIZE_MASK 0x3ff #define CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_SPM_SIZE_OFFSET 0 #define CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_SPM_SIZE_FIELD \ @@ -57,7 +63,7 @@ extern "C" { .index = CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_SPM_SIZE_OFFSET}) // Controls the L1 DCache flushing and invalidation. -#define CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_REG_OFFSET 0x1c +#define CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_REG_OFFSET 0x24 #define CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_INSN_MASK 0x3 #define CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_INSN_OFFSET 0 #define CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_INSN_FIELD \ @@ -71,10 +77,10 @@ extern "C" { #define CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_MULTIREG_COUNT 2 // One-hot tile selection mask for private-partition flush. -#define CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_0_REG_OFFSET 0x20 +#define CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_0_REG_OFFSET 0x28 // One-hot tile selection mask for private-partition flush. -#define CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_1_REG_OFFSET 0x24 +#define CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_1_REG_OFFSET 0x2c // Tile participation mask for the cluster-level hardware barrier. #define CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_TILE_FIELD_WIDTH 32 @@ -82,25 +88,25 @@ extern "C" { #define CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_MULTIREG_COUNT 2 // Tile participation mask for the cluster-level hardware barrier. -#define CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_0_REG_OFFSET 0x28 +#define CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_0_REG_OFFSET 0x30 // Tile participation mask for the cluster-level hardware barrier. -#define CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_1_REG_OFFSET 0x2c +#define CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_1_REG_OFFSET 0x34 // Controls the L1 DCache flushing and invalidation. -#define CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT_REG_OFFSET 0x30 +#define CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT_REG_OFFSET 0x38 #define CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT_COMMIT_BIT 0 // Controls the L1 DCache flushing and invalidation. -#define CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT_REG_OFFSET 0x34 +#define CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT_REG_OFFSET 0x3c #define CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT_COMMIT_BIT 0 // Indicate the status of flushing -#define CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_REG_OFFSET 0x38 +#define CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_REG_OFFSET 0x40 #define CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_STATUS_BIT 0 // Number of private banks configured per tile -#define CACHEPOOL_PERIPHERAL_L1D_PRIVATE_REG_OFFSET 0x3c +#define CACHEPOOL_PERIPHERAL_L1D_PRIVATE_REG_OFFSET 0x44 #define CACHEPOOL_PERIPHERAL_L1D_PRIVATE_NUMBER_MASK 0xf #define CACHEPOOL_PERIPHERAL_L1D_PRIVATE_NUMBER_OFFSET 0 #define CACHEPOOL_PERIPHERAL_L1D_PRIVATE_NUMBER_FIELD \ @@ -109,10 +115,10 @@ extern "C" { CACHEPOOL_PERIPHERAL_L1D_PRIVATE_NUMBER_OFFSET}) // Starting address of private L1D partition -#define CACHEPOOL_PERIPHERAL_L1D_ADDR_REG_OFFSET 0x40 +#define CACHEPOOL_PERIPHERAL_L1D_ADDR_REG_OFFSET 0x48 // Cache xbar offset setting -#define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_REG_OFFSET 0x44 +#define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_REG_OFFSET 0x4c #define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET_MASK 0x1f #define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET_OFFSET 0 #define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET_FIELD \ @@ -121,7 +127,7 @@ extern "C" { CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET_OFFSET}) // Cache xbar offset setting -#define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT_REG_OFFSET 0x48 +#define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT_REG_OFFSET 0x50 #define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT_COMMIT_BIT 0 #ifdef __cplusplus diff --git a/software/snRuntime/include/snrt.h b/software/snRuntime/include/snrt.h index 886d552a..d76c7c70 100644 --- a/software/snRuntime/include/snrt.h +++ b/software/snRuntime/include/snrt.h @@ -82,6 +82,14 @@ extern uint32_t snrt_cluster_partial_barrier_mask(const uint32_t *cids, uint32_t /// by snrt_cluster_partial_barrier_mask()). O(1) — a single store. extern void snrt_cluster_partial_barrier(uint32_t local_mask); +/// Barrier across only host-0 (primary) harts. Callable unconditionally +/// from any hart; host-1 harts return immediately without participating. +extern void snrt_cluster_host0_barrier(); + +/// Barrier across only host-1 (secondary) harts. Callable unconditionally +/// from any hart; host-0 harts return immediately without participating. +extern void snrt_cluster_host1_barrier(); + static inline uint32_t __attribute__((pure)) snrt_hartid(); struct snrt_team_root *snrt_current_team(); extern struct snrt_peripherals *snrt_peripherals(); @@ -97,6 +105,15 @@ extern uint32_t snrt_cluster_core_per_tile(); extern uint32_t snrt_cluster_idx(); extern uint32_t snrt_cluster_num(); +/// whether this hart is host 0 of its Core Complex pair +extern int snrt_cluster_is_primary(); + +/// Physical vector-unit count/index/per-tile count, halved vs. the hart-based +/// counterparts in dual-scalar configs (one Spatz per CC pair), unchanged otherwise. +extern uint32_t snrt_cluster_vpu_num(); +extern uint32_t snrt_cluster_vpu_idx(); +extern uint32_t snrt_cluster_vpu_per_tile(); + /// get pointer to barrier register extern uint32_t _snrt_barrier_reg_ptr(); diff --git a/software/snRuntime/include/spatz_lock.h b/software/snRuntime/include/spatz_lock.h new file mode 100644 index 00000000..b6985cae --- /dev/null +++ b/software/snRuntime/include/spatz_lock.h @@ -0,0 +1,41 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. + +// SPDX-License-Identifier: Apache-2.0 + +#include + +#include "cachepool_peripheral.h" +#include "team.h" + +extern __thread struct snrt_team *_snrt_team_current; + +// Outcome of a single spatz_lock_try_acquire()/try_release() attempt -- +// bits [1:0] of the raw register value, see cachepool_spatz_lock.sv. +typedef enum { + SPATZ_LOCK_FAIL = 0, // denied; hardware made no reservation + SPATZ_LOCK_SUCCESS = 1, // granted now + SPATZ_LOCK_SUCCESS_WAIT = 2, // accepted, completes on its own once drained +} spatz_lock_outcome_t; + +// Single, always-immediate attempt (never blocks in hardware). Decode with +// spatz_lock_outcome(). +uint32_t spatz_lock_try_acquire(void); +uint32_t spatz_lock_try_release(void); + +static inline spatz_lock_outcome_t spatz_lock_outcome(uint32_t raw) { + return (spatz_lock_outcome_t)(raw & 0x3); +} + +// Acquire/release Spatz ownership on a dual-Snitch Core Complex. Retries +// spatz_lock_try_acquire()/try_release() until it stops failing: FAIL means +// genuine contention (nothing reserved, must retry); SUCCESS/SUCCESS_WAIT +// both return right away -- on SUCCESS_WAIT, the switch is still draining, +// but acc_mux already refuses any new Spatz issue until it's done, so the +// next real vector/FP instruction naturally blocks in hardware until the +// switch completes, same as before. Only call release after a successful +// acquire on the same hart, and only from harts sharing a Spatz +// (cachepool_cc_dual) -- on a single-scalar-per-CC build these access a +// harmless, unused peripheral register (every attempt returns SUCCESS). +void spatz_lock_acquire(void); +void spatz_lock_release(void); diff --git a/software/snRuntime/src/barrier.c b/software/snRuntime/src/barrier.c index c5485d42..0e39dc51 100644 --- a/software/snRuntime/src/barrier.c +++ b/software/snRuntime/src/barrier.c @@ -44,6 +44,30 @@ void snrt_cluster_partial_barrier(uint32_t local_mask) { *(volatile uint32_t *)_snrt_barrier_reg_ptr() = local_mask; } +/// Tile-local mask of every hart whose parity matches want_primary (host 0 +/// positions if 1, host 1 positions if 0), given sequential host0/host1 +/// pairing; with no pairing at all, every hart counts as host 0. +static uint32_t snrt_cc_role_mask(int want_primary) { + uint32_t cpt = snrt_cluster_core_per_tile(); + uint32_t mask = 0; +#if SNRT_NUM_SCALAR_PER_CORE == 2 + for (uint32_t i = (want_primary ? 0 : 1); i < cpt; i += 2) mask |= (1u << i); +#else + if (want_primary) for (uint32_t i = 0; i < cpt; i++) mask |= (1u << i); +#endif + return mask; +} + +void snrt_cluster_host0_barrier() { + if (!snrt_cluster_is_primary()) return; + snrt_cluster_partial_barrier(snrt_cc_role_mask(1)); +} + +void snrt_cluster_host1_barrier() { + if (snrt_cluster_is_primary()) return; + snrt_cluster_partial_barrier(snrt_cc_role_mask(0)); +} + /// Synchronize cores in a cluster with a software barrier void snrt_cluster_sw_barrier() { // Remember previous iteration diff --git a/software/snRuntime/src/spatz_lock.c b/software/snRuntime/src/spatz_lock.c new file mode 100644 index 00000000..6579ba2b --- /dev/null +++ b/software/snRuntime/src/spatz_lock.c @@ -0,0 +1,46 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. + +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +uint32_t spatz_lock_try_acquire(void) { +#if SNRT_NUM_SCALAR_PER_CORE == 2 + volatile uint32_t *lock = + (uint32_t *)(_snrt_team_current->root->cluster_mem.end + + CACHEPOOL_PERIPHERAL_SPATZ_LOCK_ACQUIRE_REG_OFFSET); + return *lock; +#else + // No lock module exists to intercept this on a single-scalar-per-CC + // build; report an unconditional grant rather than reading back the + // register's uninitialized reset value (which would decode as FAIL). + return SPATZ_LOCK_SUCCESS; +#endif +} + +uint32_t spatz_lock_try_release(void) { +#if SNRT_NUM_SCALAR_PER_CORE == 2 + volatile uint32_t *lock = + (uint32_t *)(_snrt_team_current->root->cluster_mem.end + + CACHEPOOL_PERIPHERAL_SPATZ_LOCK_RELEASE_REG_OFFSET); + return *lock; +#else + return SPATZ_LOCK_SUCCESS; +#endif +} + +void spatz_lock_acquire(void) { + spatz_lock_outcome_t outcome; + do { + outcome = spatz_lock_outcome(spatz_lock_try_acquire()); + } while (outcome == SPATZ_LOCK_FAIL); +} + +void spatz_lock_release(void) { + spatz_lock_outcome_t outcome; + do { + outcome = spatz_lock_outcome(spatz_lock_try_release()); + } while (outcome == SPATZ_LOCK_FAIL); +} diff --git a/software/snRuntime/src/team.c b/software/snRuntime/src/team.c index 4ab61e66..0b423722 100644 --- a/software/snRuntime/src/team.c +++ b/software/snRuntime/src/team.c @@ -54,10 +54,42 @@ uint32_t snrt_cluster_tile_idx() { uint32_t snrt_cluster_core_idx() { return _snrt_core_idx; } +int snrt_cluster_is_primary() { +#if SNRT_NUM_SCALAR_PER_CORE == 2 + return (_snrt_core_idx % 2) == 0; +#else + return 1; +#endif +} + uint32_t snrt_cluster_core_num() { return _snrt_team_current->root->cluster_core_num; } +uint32_t snrt_cluster_vpu_num() { +#if SNRT_NUM_SCALAR_PER_CORE == 2 + return snrt_cluster_core_num() / 2; +#else + return snrt_cluster_core_num(); +#endif +} + +uint32_t snrt_cluster_vpu_idx() { +#if SNRT_NUM_SCALAR_PER_CORE == 2 + return snrt_cluster_core_idx() / 2; +#else + return snrt_cluster_core_idx(); +#endif +} + +uint32_t snrt_cluster_vpu_per_tile() { +#if SNRT_NUM_SCALAR_PER_CORE == 2 + return snrt_cluster_core_per_tile() / 2; +#else + return snrt_cluster_core_per_tile(); +#endif +} + uint32_t snrt_cluster_core_per_tile() { uint32_t c = _snrt_team_current->root->cluster_core_num; uint32_t t = SNRT_BOOT_TILE_COUNT; diff --git a/software/sync/CMakeLists.txt b/software/sync/CMakeLists.txt new file mode 100644 index 00000000..018f9d13 --- /dev/null +++ b/software/sync/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2025 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +add_spatz_test_zeroParam(spin-lock spin-lock/main.c) +add_spatz_test_zeroParam(mcs-lock mcs-lock/main.c) +add_spatz_test_zeroParam(partial_barrier partial_barrier/main.c)# SOCMIPO +add_spatz_test_zeroParam(spatz-lock-handoff spatz-lock-handoff/main.c) +add_spatz_test_zeroParam(false-sharing false-sharing/main.c) diff --git a/software/tests/false-sharing/main.c b/software/sync/false-sharing/main.c similarity index 100% rename from software/tests/false-sharing/main.c rename to software/sync/false-sharing/main.c diff --git a/software/tests/mcs-lock/kernel/printf_lock.c b/software/sync/mcs-lock/kernel/printf_lock.c similarity index 100% rename from software/tests/mcs-lock/kernel/printf_lock.c rename to software/sync/mcs-lock/kernel/printf_lock.c diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/kernel/printf_lock.h b/software/sync/mcs-lock/kernel/printf_lock.h similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/kernel/printf_lock.h rename to software/sync/mcs-lock/kernel/printf_lock.h diff --git a/software/tests/mcs-lock/main.c b/software/sync/mcs-lock/main.c similarity index 100% rename from software/tests/mcs-lock/main.c rename to software/sync/mcs-lock/main.c diff --git a/software/tests/partial_barrier/main.c b/software/sync/partial_barrier/main.c similarity index 100% rename from software/tests/partial_barrier/main.c rename to software/sync/partial_barrier/main.c diff --git a/software/sync/spatz-lock-handoff/main.c b/software/sync/spatz-lock-handoff/main.c new file mode 100644 index 00000000..3df38863 --- /dev/null +++ b/software/sync/spatz-lock-handoff/main.c @@ -0,0 +1,109 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// SPDX-License-Identifier: Apache-2.0 + +// Lock handoff test: proves the Spatz lock actually serializes host0/host1, +// not just that Spatz produces correct data for independent requesters. +// v0 is physical Spatz register state shared across the lock handoff -- host0 +// loads a pattern into v0, then both hosts (racing for the lock) add a fixed +// role constant (host0: 1, host1: 2) directly to v0, and host0 stores the +// final result. Fixed constants (not cid) keep the expected sum identical +// across every pair, so a failure is easy to spot and debug. A lost update +// (lock failed to serialize) yields a wrong final sum, which a +// per-hart-disjoint-slice test could never detect. + +#include +#include +#include +#include +#include + +#define MAX_PAIRS 128U +#define TEST_LEN 64U // vector elements per pair, fits one vsetvli group +#define PATTERN 1U + +#if SNRT_NUM_SCALAR_PER_CORE == 2 +#define PAIR_SIZE 2U +#define EXPECT (PATTERN + 3U) // host0 adds 1, host1 adds 2 +#else +#define PAIR_SIZE 1U +#define EXPECT (PATTERN + 1U) // host0 only, adds 1 +#endif + +static uint32_t src_dram[TEST_LEN] __attribute__((section(".data"))) + = {[0 ... TEST_LEN - 1] = PATTERN}; +static uint32_t dst_dram[MAX_PAIRS][TEST_LEN] __attribute__((section(".data"))); +static uint32_t pair_errors[MAX_PAIRS] __attribute__((section(".data"))); + +int main() { + const uint32_t cid = snrt_cluster_core_idx(); + const uint32_t pair = cid / PAIR_SIZE; + const uint32_t add_val = snrt_cluster_is_primary() ? 1U : 2U; + uint32_t vlen; + + if (cid == 0) { + printf("*** spatz-lock-handoff test ***\n"); + printf("PATTERN=%u PAIR_SIZE=%u EXPECT=%u\n", PATTERN, PAIR_SIZE, EXPECT); + } + snrt_cluster_hw_barrier(); + + // Phase 1 (host0 only): load the seed pattern into v0 and hold the lock + // open just long enough to establish it -- no store, v0 carries the state. + if (snrt_cluster_is_primary()) { + spatz_lock_acquire(); + asm volatile("vsetvli %0, %1, e32, m8, ta, ma" : "=r"(vlen) : "r"(TEST_LEN)); + asm volatile("vle32.v v0, (%0)" : : "r"(src_dram)); + spatz_lock_release(); + } + snrt_cluster_hw_barrier(); + + // Phase 2 (both hosts): racing add directly on v0, serialized by the lock + // -- a lost update here means the lock failed to exclude concurrent access. + spatz_lock_acquire(); + asm volatile("vsetvli %0, %1, e32, m8, ta, ma" : "=r"(vlen) : "r"(TEST_LEN)); + asm volatile("vadd.vx v0, v0, %0" : : "r"(add_val)); + spatz_lock_release(); + snrt_cluster_hw_barrier(); + + // Phase 3 (host0 only): store the final accumulated v0 for checking. + if (snrt_cluster_is_primary()) { + spatz_lock_acquire(); + asm volatile("vsetvli %0, %1, e32, m8, ta, ma" : "=r"(vlen) : "r"(TEST_LEN)); + asm volatile("vse32.v v0, (%0)" : : "r"(dst_dram[pair])); + spatz_lock_release(); + + uint32_t errs = 0; + for (uint32_t i = 0; i < TEST_LEN; i++) { + if (dst_dram[pair][i] != EXPECT) { + errs++; + } + } + pair_errors[pair] = errs; + } + snrt_cluster_hw_barrier(); + + if (cid == 0) { + const uint32_t num_cores = snrt_cluster_core_num(); + const uint32_t num_pairs = num_cores / PAIR_SIZE; + uint32_t total_err = 0; + for (uint32_t p = 0; p < num_pairs; p++) { + total_err += pair_errors[p]; + } + + if (total_err == 0) { + printf("[PASS] spatz-lock-handoff: pairs=%u len=%u\n", num_pairs, TEST_LEN); + } else { + printf("[FAIL] spatz-lock-handoff: errors=%u pairs=%u len=%u\n", + total_err, num_pairs, TEST_LEN); + printf(" expect=%u\n", EXPECT); + for (uint32_t p = 0; p < num_pairs; p++) { + if (pair_errors[p] != 0) { + printf(" pair %u: dst_dram[0]=%u errs=%u\n", p, dst_dram[p][0], + pair_errors[p]); + } + } + } + } + snrt_cluster_hw_barrier(); + + return 0; +} diff --git a/software/tests/spin-lock/main.c b/software/sync/spin-lock/main.c similarity index 100% rename from software/tests/spin-lock/main.c rename to software/sync/spin-lock/main.c diff --git a/software/tests/CMakeLists.txt b/software/tests/CMakeLists.txt index d82b0494..0c133516 100644 --- a/software/tests/CMakeLists.txt +++ b/software/tests/CMakeLists.txt @@ -2,143 +2,7 @@ # Licensed under the Apache License, Version 2.0, see LICENSE for details. # SPDX-License-Identifier: Apache-2.0 -cmake_minimum_required(VERSION 3.13) - -# Allow spatzBenchmarks to be built as a standalone library. -if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) - list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../cmake) - set(CMAKE_TOOLCHAIN_FILE toolchain-gcc CACHE STRING "Toolchain to use") - - project(Benchmarks LANGUAGES C ASM) - include(SnitchUtilities) - - # Build the runtime. - add_subdirectory(${SPATZ_DIR}/sw/snRuntime snRuntime) -endif() - -include_directories(include) -include_directories(${SNRUNTIME_INCLUDE_DIRS}) - -add_compile_options(-O3 -g -ffunction-sections) - - - -# Macro to generate golden values -macro(add_spatz_test_zeroParam name file) - set(target_name ${name}) - add_snitch_test(${target_name} ${file}) - target_link_libraries(test-${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) - # target_compile_definitions(test-${SNITCH_TEST_PREFIX}${target_name}) -endmacro() - -macro(add_spatz_test_oneParam name file param1) - set(target_name ${name}_M${param1}) - add_snitch_test(${target_name} ${file}) - target_link_libraries(test-${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) - target_compile_definitions(test-${SNITCH_TEST_PREFIX}${target_name} PUBLIC DATAHEADER="data/data_${param1}.h") -endmacro() - -macro(add_spatz_test_twoParam name file param1 param2) - set(target_name ${name}_M${param1}_N${param2}) - add_snitch_test(${target_name} ${file}) - target_link_libraries(test-${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) - target_compile_definitions(test-${SNITCH_TEST_PREFIX}${target_name} PUBLIC DATAHEADER="data/data_${param1}_${param2}.h") -endmacro() - -macro(add_spatz_test_threeParam name file param1 param2 param3) - set(target_name ${name}_M${param1}_N${param2}_K${param3}) - add_snitch_test(${target_name} ${file}) - target_link_libraries(test-${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) - target_compile_definitions(test-${SNITCH_TEST_PREFIX}${target_name} PUBLIC DATAHEADER="data/data_${param1}_${param2}_${param3}.h") -endmacro() - - - - -# Benchmark library -add_library(benchmark benchmark/benchmark.c) -add_library(spin_lock benchmark/spin_lock.c) -add_library(mcs_lock benchmark/mcs_lock.c) - -add_compile_options(-O3 -g -ffunction-sections) -add_compile_options(-DELEN=64) - -include_directories(include) -include_directories(${SNRUNTIME_INCLUDE_DIRS}) - -# Kernels -## Vector Kernels -add_library(fdotp-32b fdotp-32b/kernel/fdotp.c) -add_library(idotp-32b idotp-32b/kernel/idotp.c) -add_library(fmatmul-32b fmatmul-32b/kernel/fmatmul.c) - - -## RLC -add_library(printf_lock multi_producer_single_consumer_double_linked_list/kernel/printf_lock.c) -add_library(llist multi_producer_single_consumer_double_linked_list/kernel/llist.c) -add_library(mm multi_producer_single_consumer_double_linked_list/kernel/mm.c) -add_library(rlc multi_producer_single_consumer_double_linked_list/kernel/rlc.c) -add_library(data_move_vec multi_producer_single_consumer_double_linked_list/kernel/data_move_vec.c) - -# Tests -enable_testing() -set(SNITCH_TEST_PREFIX cachepool-) - -## RLC -add_spatz_test_zeroParam(spin-lock spin-lock/main.c) -add_spatz_test_zeroParam(partial_barrier partial_barrier/main.c)# SOCMIPO -add_spatz_test_zeroParam(partial_barrier_benchmark partial_barrier_benchmark/main.c)# SOCMIPO -add_spatz_test_zeroParam(mcs-lock mcs-lock/main.c) -add_spatz_test_zeroParam(byte-enable byte-enable/main.c) -add_spatz_test_zeroParam(cache-line-rw-smoke cache-line-rw-smoke/main.c) add_spatz_test_zeroParam(minimal-tile0-repro minimal-tile0-repro/main.c) -add_spatz_test_zeroParam(cache-test-scalar cache-test-scalar/main.c) -add_spatz_test_zeroParam(cache-test-vector cache-test-vector/main.c) -add_spatz_test_zeroParam(cache-mix-smoke cache-mix-smoke/main.c) -add_spatz_test_zeroParam(cache-mix-pressure cache-mix-pressure/main.c) -add_spatz_test_zeroParam(cache-rlc-mimic cache-rlc-mimic/main.c) -add_spatz_test_zeroParam(cache-vector-rw cache-vector-rw/main.c) -add_spatz_test_zeroParam(cache-coverage cache-coverage/main.c) -add_spatz_test_zeroParam(cache-coverage-min cache-coverage-min/main.c) - -# add_snitch_test(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c) -# add_spatz_test_threeParam(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c 1 1350 1000) -# add_spatz_test_threeParam(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c 1 2044 100) -add_spatz_test_threeParam(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c 1 1350 300) -add_spatz_test_threeParam(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c 1 1350 100) -add_spatz_test_threeParam(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c 1 1350 10) - -## Vector -### Floating-Point -add_spatz_test_oneParam(fdotp-32b fdotp-32b/main.c 8192) -add_spatz_test_oneParam(fdotp-32b fdotp-32b/main.c 32768) -add_spatz_test_oneParam(fdotp-32b fdotp-32b/main.c 65536) - -add_spatz_test_threeParam(gemv gemv/main.c 128 128 32) -add_spatz_test_threeParam(gemv gemv/main.c 256 128 32) -add_spatz_test_threeParam(gemv gemv/main.c 512 128 32) -add_spatz_test_threeParam(gemv gemv/main.c 1024 128 32) - -add_spatz_test_threeParam(gemv-opt gemv-opt/main.c 128 128 32) -add_spatz_test_threeParam(gemv-opt gemv-opt/main.c 256 128 32) -add_spatz_test_threeParam(gemv-opt gemv-opt/main.c 512 128 32) -add_spatz_test_threeParam(gemv-opt gemv-opt/main.c 1024 128 32) - -add_spatz_test_threeParam(fmatmul-32b fmatmul-32b/main.c 32 32 32) -add_spatz_test_threeParam(fmatmul-32b fmatmul-32b/main.c 64 64 64) -add_spatz_test_threeParam(fmatmul-32b fmatmul-32b/main.c 128 128 128) -add_spatz_test_threeParam(fmatmul-32b fmatmul-32b/main.c 1024 32 32) -add_spatz_test_threeParam(fmatmul-32b fmatmul-32b/main.c 1024 64 64) - -add_spatz_test_twoParam(fft-32b fft-32b/main.c 256 4) -add_spatz_test_twoParam(fft-32b fft-32b/main.c 1024 4) -add_spatz_test_twoParam(fft-32b fft-32b/main.c 1024 16) - -### Integer-Point -add_spatz_test_oneParam(idotp-32b idotp-32b/main.c 8192) -add_spatz_test_oneParam(idotp-32b idotp-32b/main.c 32768) - - add_spatz_test_zeroParam(load-store load-store/main.c) - +add_spatz_test_zeroParam(partial_barrier_benchmark partial_barrier_benchmark/main.c)# SOCMIPO add_spatz_test_zeroParam(bandwidth bandwidth/main.c) diff --git a/util/auto-benchmark/configs.sh b/util/auto-benchmark/configs.sh index b7e424f6..f5fe28fe 100755 --- a/util/auto-benchmark/configs.sh +++ b/util/auto-benchmark/configs.sh @@ -5,5 +5,5 @@ # Configs and kernel suffixes (without prefix) CONFIGS="cachepool_fpu_4g" KERNELS="fdotp-32b_M65536 gemv_M1024_N128_K32 fmatmul-32b_M1024_N32_K32 fft-32b_M1024_N16 " -PREFIX="test-cachepool-" # common prefix for all kernels +PREFIX="test-" # common prefix for all kernels ROOT_PATH=../.. # adjust if needed (path to repo root) diff --git a/util/scripts/gen_spatz_cfg.py b/util/scripts/gen_spatz_cfg.py index 030695c3..e957868a 100644 --- a/util/scripts/gen_spatz_cfg.py +++ b/util/scripts/gen_spatz_cfg.py @@ -38,7 +38,11 @@ def main(): # Build dynamic values that depend on others num_cores = int(os.environ.get("num_cores", "4")) num_tiles = int(os.environ.get("num_tiles", "1")) - cores_array = ",".join(['{ $ref: "#/compute_core_template" }'] * num_cores) + num_scalar_per_core = int(os.environ.get("num_scalar_per_core", "1")) + # cluster.cores must list one entry per hart, not per CC slot, since + # generate_bootdata.py derives SNRT_BOOT_CORE_COUNT from its length. + num_harts = num_cores * num_scalar_per_core + cores_array = ",".join(['{ $ref: "#/compute_core_template" }'] * num_harts) # Convert spatz_fpu_en -> spatz_fpu_bool for HJSON spatz_fpu_bool = bool_str(os.environ.get("spatz_fpu_en", "0"))