From 4ab50ecbd1d4d8b2b6d928d96bce399522b4d109 Mon Sep 17 00:00:00 2001 From: Diyou Shen Date: Thu, 15 Oct 2026 09:38:42 +0200 Subject: [PATCH 1/5] [SW] Add a fence for dual core configuration to avoid triggering a consistency issue in NoC. --- software/kernels/fp/fft-32b/kernel/fft.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/software/kernels/fp/fft-32b/kernel/fft.c b/software/kernels/fp/fft-32b/kernel/fft.c index 2723e98f..38a95369 100644 --- a/software/kernels/fp/fft-32b/kernel/fft.c +++ b/software/kernels/fp/fft-32b/kernel/fft.c @@ -250,5 +250,11 @@ void fft_p2(float *s, float *buf, const float *twi, float *out, asm volatile("vsuxei16.v v28, (%0), v12" ::"r"(im_l_o)); } } +#if SNRT_NUM_SCALAR_PER_CORE == 2 + // The next stage loads these indexed stores through other Spatz ports, whose path to the + // L1 (NoC channel) is not ordered against the stores' path: drain the stores first. + if (bf != log2_nfft - 1) + snrt_fence_spatz(); +#endif } } From 32329c2d94db56a9800786c7ff01d0a7fa6b373c Mon Sep 17 00:00:00 2001 From: Diyou Shen Date: Tue, 20 Oct 2026 13:27:29 +0200 Subject: [PATCH 2/5] [SRC][L1] Add cache bypass datapath --- Bender.lock | 2 +- Bender.yml | 1 + config/README.md | 4 +- config/cachepool_4g.mk | 2 +- config/cachepool_dual_4g.mk | 2 +- config/cachepool_dual_fpu_16g.mk | 2 +- config/cachepool_dual_fpu_4g.mk | 2 +- config/cachepool_fpu_16g.mk | 2 +- config/cachepool_fpu_16g_tiny.mk | 2 +- config/cachepool_fpu_4g.mk | 2 +- config/config.mk | 22 +- .../cachepool_peripheral.sv | 13 +- .../cachepool_peripheral_reg.hjson.tmpl | 14 + .../cachepool_peripheral_reg_pkg.sv | 46 ++- .../cachepool_peripheral_reg_top.sv | 47 ++- hardware/src/cachepool_cluster.sv | 156 ++++---- hardware/src/cachepool_group.sv | 11 +- hardware/src/cachepool_group_cfg.sv | 5 +- hardware/src/cachepool_group_noc_wrapper.sv | 3 + hardware/src/cachepool_pkg.sv | 55 ++- hardware/src/cachepool_reqrsp_to_axi.sv | 236 +++++++++++ hardware/src/cachepool_tile.sv | 86 ++-- hardware/tb/tb_cachepool.sv | 2 +- .../snRuntime/include/cachepool_peripheral.h | 7 +- software/snRuntime/include/l1cache.h | 4 + software/snRuntime/link/common.ld.in | 8 + software/snRuntime/src/l1cache.c | 16 + software/tests/CMakeLists.txt | 1 + software/tests/l1-bypass/main.c | 371 ++++++++++++++++++ 29 files changed, 953 insertions(+), 171 deletions(-) create mode 100644 hardware/src/cachepool_reqrsp_to_axi.sv create mode 100644 software/tests/l1-bypass/main.c diff --git a/Bender.lock b/Bender.lock index 982bb86e..8a6045c1 100644 --- a/Bender.lock +++ b/Bender.lock @@ -109,7 +109,7 @@ packages: - obi - register_interface insitu-cache: - revision: d7a526ebc965c3c9808f348b377d06f4479cc4ab + revision: 9a4fe83884fd94baf87bba6ea4e8ad75d4f4893f version: null source: Git: https://github.com/pulp-platform/Insitu-Cache.git diff --git a/Bender.yml b/Bender.yml index 5cc78d46..37d7c5f1 100644 --- a/Bender.yml +++ b/Bender.yml @@ -26,6 +26,7 @@ export_include_dirs: sources: # Level 0 + - hardware/src/cachepool_reqrsp_to_axi.sv - hardware/src/reqrsp_to_reg.sv - hardware/src/reqrsp_xbar.sv - hardware/src/tcdm_cache_interco.sv diff --git a/config/README.md b/config/README.md index 857dc60a..c38313ef 100644 --- a/config/README.md +++ b/config/README.md @@ -108,13 +108,13 @@ subset; the "Define" column gives the SystemVerilog macro the `Makefile` emits | `addr_width` | `32` | `ADDR_WIDTH` | Core address width | | `bootrom_data_width` | `32` | — | Bootrom word width (used by `generate_bootrom.py`) | | `num_barrier_slots` | `2` | `NUM_BARRIER_SLOTS` | Independent hardware barrier slots | -| `dram_type` | `HBM2` | `DRAM_TYPE` | DRAMSys model. One of `DDR3`, `DDR4`, `LPDDR4`, `HBM2`. Also sets the default `refill_data_width` | +| `dram_type` | `HBM2` | `DRAM_TYPE` | DRAMSys model. One of `DDR3`, `DDR4`, `LPDDR4`, `HBM2` | ### Tile and L1 data cache | Parameter | Default | Define | Description | | --- | --- | --- | --- | -| `refill_data_width` | auto from `dram_type` | `REFILL_DATA_WIDTH` | Refill interconnect data width. Default is 512 (HBM2), 256 (LPDDR4), 128 (otherwise) | +| `refill_data_width` | 512 | `REFILL_DATA_WIDTH` | Refill interconnect data width. Must equal `l1d_cacheline_width`: refills are single beats (checked in `cachepool_cluster.sv`) | | `l1d_cacheline_width` | `512` | `L1D_CACHELINE_WIDTH` | Cache line width in bits | | `l1d_bank_factor` | `1` | — | Banks per core (derives `l1d_num_banks`) | | `l1d_coal_window` | `2` | `L1D_COAL_WINDOW` | Coalescing window of the request coalescer | diff --git a/config/cachepool_4g.mk b/config/cachepool_4g.mk index 765b53ba..74bdcbdd 100644 --- a/config/cachepool_4g.mk +++ b/config/cachepool_4g.mk @@ -91,7 +91,7 @@ snitch_max_trans ?= 16 ##################### ## L2 Main Memory ## ##################### -# DRAM type (selects DRAMSys config and sets default refill_data_width) +# DRAM type (selects DRAMSys config) dram_type ?= HBM2 # L2 number of channels diff --git a/config/cachepool_dual_4g.mk b/config/cachepool_dual_4g.mk index 7c428d45..cb6276e2 100644 --- a/config/cachepool_dual_4g.mk +++ b/config/cachepool_dual_4g.mk @@ -97,7 +97,7 @@ num_barrier_slots ?= 4 ##################### ## L2 Main Memory ## ##################### -# DRAM type (selects DRAMSys config and sets default refill_data_width) +# DRAM type (selects DRAMSys config) dram_type ?= HBM2 # L2 number of channels diff --git a/config/cachepool_dual_fpu_16g.mk b/config/cachepool_dual_fpu_16g.mk index 6d52089e..f2fffa1d 100644 --- a/config/cachepool_dual_fpu_16g.mk +++ b/config/cachepool_dual_fpu_16g.mk @@ -97,7 +97,7 @@ num_barrier_slots ?= 4 ##################### ## L2 Main Memory ## ##################### -# DRAM type (selects DRAMSys config and sets default refill_data_width) +# DRAM type (selects DRAMSys config) dram_type ?= HBM2 # L2 number of channels diff --git a/config/cachepool_dual_fpu_4g.mk b/config/cachepool_dual_fpu_4g.mk index db7d40f1..7ecd817a 100644 --- a/config/cachepool_dual_fpu_4g.mk +++ b/config/cachepool_dual_fpu_4g.mk @@ -97,7 +97,7 @@ num_barrier_slots ?= 4 ##################### ## L2 Main Memory ## ##################### -# DRAM type (selects DRAMSys config and sets default refill_data_width) +# DRAM type (selects DRAMSys config) dram_type ?= HBM2 # L2 number of channels diff --git a/config/cachepool_fpu_16g.mk b/config/cachepool_fpu_16g.mk index fa04b844..55a0bb64 100644 --- a/config/cachepool_fpu_16g.mk +++ b/config/cachepool_fpu_16g.mk @@ -91,7 +91,7 @@ snitch_max_trans ?= 16 ##################### ## L2 Main Memory ## ##################### -# DRAM type (selects DRAMSys config and sets default refill_data_width) +# DRAM type (selects DRAMSys config) dram_type ?= HBM2 # L2 number of channels diff --git a/config/cachepool_fpu_16g_tiny.mk b/config/cachepool_fpu_16g_tiny.mk index d8c4a668..8ba68263 100644 --- a/config/cachepool_fpu_16g_tiny.mk +++ b/config/cachepool_fpu_16g_tiny.mk @@ -91,7 +91,7 @@ snitch_max_trans ?= 16 ##################### ## L2 Main Memory ## ##################### -# DRAM type (selects DRAMSys config and sets default refill_data_width) +# DRAM type (selects DRAMSys config) dram_type ?= HBM2 # L2 number of channels diff --git a/config/cachepool_fpu_4g.mk b/config/cachepool_fpu_4g.mk index a71911d4..50c71243 100644 --- a/config/cachepool_fpu_4g.mk +++ b/config/cachepool_fpu_4g.mk @@ -91,7 +91,7 @@ snitch_max_trans ?= 16 ##################### ## L2 Main Memory ## ##################### -# DRAM type (selects DRAMSys config and sets default refill_data_width) +# DRAM type (selects DRAMSys config) dram_type ?= HBM2 # L2 number of channels diff --git a/config/config.mk b/config/config.mk index 26c0c259..d40499fb 100644 --- a/config/config.mk +++ b/config/config.mk @@ -26,20 +26,14 @@ include $(CACHEPOOL_DIR)/config/$(config).mk ## DRAM Type ## ##################### -# DRAM type used by DRAMSys and to auto-set refill_data_width. +# DRAM type used by DRAMSys. # Supported values: DDR3, DDR4, LPDDR4, HBM2 dram_type ?= HBM2 -# Auto-derive default refill data width from dram_type. -# HBM variants: 512b (3600*16/1000 = 57.6B/cyc) -# DDR variants: 128b (1866*8/1000 = 15B/cyc) -ifeq ($(dram_type),HBM2) - refill_data_width_default := 512 -else ifeq ($(dram_type),LPDDR4) - refill_data_width_default := 256 -else - refill_data_width_default := 128 -endif +# Refill data width: always one full 512b cache line per beat. The DRAM converters send +# single beats only, and the L1 bypass path packs data + routing info into one beat. +# DDR variants waste part of the width (1866*8/1000 = 15B/cyc of real bandwidth). +refill_data_width_default := 512 ######################### ## CachePool Cluster ## @@ -149,8 +143,8 @@ num_barrier_slots ?= 2 # in cachepool_cluster.sv, and the cluster zero-extends refill_user_t onto the # (wider) AXI user port at the reqrsp_to_axi call site. # -# refill_user_t = bank_id(BankIDWidth) + tile_id(TileIDWidth) -# + cache_info_t + burst_req_t +# refill_user_t = bypass(1) + bank_id(BankIDWidth) + tile_id(TileIDWidth) +# + l2_src_id + cache_info_t + burst_req_t # cache_info_t = {for_write_pend, depth, way} -- NOTE: NO tile_id inside. # # BankIDWidth depends only on cores-per-tile (constant across a tile-count @@ -158,7 +152,7 @@ num_barrier_slots ?= 2 # only tile-count-dependent term in refill_user_t is TileIDWidth = # clog2(NumTiles), and that's already added exactly once, downstream, by # cachepool_pkg.sv (SpatzAxiUserWidth = AXI_USER_WIDTH + clog2(NumTiles) + -# bits(l2_id_t)) -- no need to also grow axi_user_width with NumTiles here. +# bits(l2_id_t) + 1 for bypass) -- no need to also grow axi_user_width with NumTiles here. # # axi_user_base below is the width needed at NumTiles=1 (TileIDWidth=0), # i.e. bank_id + cache_info_t + burst_req_t only. diff --git a/hardware/cachepool_peripheral/cachepool_peripheral.sv b/hardware/cachepool_peripheral/cachepool_peripheral.sv index 46e079af..13a659b8 100644 --- a/hardware/cachepool_peripheral/cachepool_peripheral.sv +++ b/hardware/cachepool_peripheral/cachepool_peripheral.sv @@ -129,6 +129,7 @@ module cachepool_peripheral assign reg_cfg.private_addr = reg2hw.l1d_addr.q; assign reg_cfg.xbar_offset = reg2hw.xbar_offset.q; assign reg_cfg.prefetch_en = reg2hw.icache_prefetch_enable.q; + assign reg_cfg.bypass_en = reg2hw.l1d_bypass.q; // Committed settings, the copy every group-level register holds once a batch is done l1d_cfg_t cfg_d, cfg_q; @@ -143,13 +144,14 @@ module cachepool_peripheral // A setting differing from its committed copy is a pending update; only the L1D instruction // waits for an explicit commit - logic l1d_insn_commit, xbar_update, private_update, addr_update, prefetch_update; + logic l1d_insn_commit, xbar_update, private_update, addr_update, prefetch_update, bypass_update; assign l1d_insn_commit = reg2hw.l1d_insn_commit.q; assign xbar_update = (reg_cfg.xbar_offset != cfg_q.xbar_offset); assign private_update = (reg_cfg.l1d_private != cfg_q.l1d_private); assign addr_update = (reg_cfg.private_addr != cfg_q.private_addr); assign prefetch_update = (reg_cfg.prefetch_en != cfg_q.prefetch_en); + assign bypass_update = (reg_cfg.bypass_en != cfg_q.bypass_en); // A batch carries one register and is only started while no batch is in flight; anything // else stays pending until the previous batch is done. INSN first. @@ -208,6 +210,13 @@ module cachepool_peripheral data_d[0] = reg_cfg.prefetch_en; busy_d = 1'b1; start_d = 1'b1; + end else if (bypass_update) begin + cfg_d.bypass_en = reg_cfg.bypass_en; + reg_d = GroupCfgL1dBypass; + data_d = '0; + data_d[0] = reg_cfg.bypass_en; + busy_d = 1'b1; + start_d = 1'b1; end end @@ -229,7 +238,7 @@ module cachepool_peripheral // Busy from the cycle a commit or a changed setting is visible, so a status read right behind // the write never sees idle before the batch has started. assign hw2reg.l1d_flush_status.d = busy_q | l1d_insn_commit | xbar_update | private_update | - addr_update | prefetch_update; + addr_update | prefetch_update | bypass_update; // One 32-bit register per barrier slot; single register is enough at // group granularity (max 32 groups). diff --git a/hardware/cachepool_peripheral/cachepool_peripheral_reg.hjson.tmpl b/hardware/cachepool_peripheral/cachepool_peripheral_reg.hjson.tmpl index df17724a..983aee0e 100644 --- a/hardware/cachepool_peripheral/cachepool_peripheral_reg.hjson.tmpl +++ b/hardware/cachepool_peripheral/cachepool_peripheral_reg.hjson.tmpl @@ -266,6 +266,20 @@ desc: "Starting address of Private L1D Partition" }] }, + { + name: "L1D_BYPASS", + desc: '''L1D bypass: requests from L1D_ADDR up (below the stack) skip the L1 data cache. + Applied when the value changes, no commit needed; L1D_FLUSH_STATUS is busy until + every group has applied it. ''' + swaccess: "rw", + hwaccess: "hro", + resval: "0", + fields: [{ + bits: "0:0", + name: "EN", + desc: "1: the private range (L1D_ADDR up to the stack) bypasses the L1D and goes to DRAM" + }] + }, { name: "XBAR_OFFSET", desc: '''Cache xbar offset setting. Applied when the value changes, no commit diff --git a/hardware/cachepool_peripheral/cachepool_peripheral_reg_pkg.sv b/hardware/cachepool_peripheral/cachepool_peripheral_reg_pkg.sv index 31804b1c..a0927901 100644 --- a/hardware/cachepool_peripheral/cachepool_peripheral_reg_pkg.sv +++ b/hardware/cachepool_peripheral/cachepool_peripheral_reg_pkg.sv @@ -86,6 +86,10 @@ package cachepool_peripheral_reg_pkg; logic [31:0] q; } cachepool_peripheral_reg2hw_l1d_addr_reg_t; + typedef struct packed { + logic q; + } cachepool_peripheral_reg2hw_l1d_bypass_reg_t; + typedef struct packed { logic [4:0] q; } cachepool_peripheral_reg2hw_xbar_offset_reg_t; @@ -118,22 +122,23 @@ package cachepool_peripheral_reg_pkg; // Register -> HW type typedef struct packed { - cachepool_peripheral_reg2hw_hw_barrier_reg_t hw_barrier; // [348:317] - cachepool_peripheral_reg2hw_spatz_lock_acquire_reg_t spatz_lock_acquire; // [316:285] - cachepool_peripheral_reg2hw_spatz_lock_release_reg_t spatz_lock_release; // [284:253] - cachepool_peripheral_reg2hw_icache_prefetch_enable_reg_t icache_prefetch_enable; // [252:252] - cachepool_peripheral_reg2hw_spatz_status_reg_t spatz_status; // [251:251] - cachepool_peripheral_reg2hw_spatz_cycle_reg_t spatz_cycle; // [250:219] - cachepool_peripheral_reg2hw_cluster_boot_control_reg_t cluster_boot_control; // [218:187] - cachepool_peripheral_reg2hw_cluster_eoc_exit_reg_t cluster_eoc_exit; // [186:183] - cachepool_peripheral_reg2hw_cfg_l1d_spm_reg_t cfg_l1d_spm; // [182:173] - cachepool_peripheral_reg2hw_cfg_l1d_insn_reg_t cfg_l1d_insn; // [172:171] - cachepool_peripheral_reg2hw_cfg_l1d_tile_sel_mreg_t [1:0] cfg_l1d_tile_sel; // [170:107] - cachepool_peripheral_reg2hw_hw_barrier_participation_mask_mreg_t [1:0] hw_barrier_participation_mask; // [106:43] - cachepool_peripheral_reg2hw_l1d_spm_commit_reg_t l1d_spm_commit; // [42:42] - cachepool_peripheral_reg2hw_l1d_insn_commit_reg_t l1d_insn_commit; // [41:41] - cachepool_peripheral_reg2hw_l1d_private_reg_t l1d_private; // [40:37] - cachepool_peripheral_reg2hw_l1d_addr_reg_t l1d_addr; // [36:5] + cachepool_peripheral_reg2hw_hw_barrier_reg_t hw_barrier; // [349:318] + cachepool_peripheral_reg2hw_spatz_lock_acquire_reg_t spatz_lock_acquire; // [317:286] + cachepool_peripheral_reg2hw_spatz_lock_release_reg_t spatz_lock_release; // [285:254] + cachepool_peripheral_reg2hw_icache_prefetch_enable_reg_t icache_prefetch_enable; // [253:253] + cachepool_peripheral_reg2hw_spatz_status_reg_t spatz_status; // [252:252] + cachepool_peripheral_reg2hw_spatz_cycle_reg_t spatz_cycle; // [251:220] + cachepool_peripheral_reg2hw_cluster_boot_control_reg_t cluster_boot_control; // [219:188] + cachepool_peripheral_reg2hw_cluster_eoc_exit_reg_t cluster_eoc_exit; // [187:184] + cachepool_peripheral_reg2hw_cfg_l1d_spm_reg_t cfg_l1d_spm; // [183:174] + cachepool_peripheral_reg2hw_cfg_l1d_insn_reg_t cfg_l1d_insn; // [173:172] + cachepool_peripheral_reg2hw_cfg_l1d_tile_sel_mreg_t [1:0] cfg_l1d_tile_sel; // [171:108] + cachepool_peripheral_reg2hw_hw_barrier_participation_mask_mreg_t [1:0] hw_barrier_participation_mask; // [107:44] + cachepool_peripheral_reg2hw_l1d_spm_commit_reg_t l1d_spm_commit; // [43:43] + cachepool_peripheral_reg2hw_l1d_insn_commit_reg_t l1d_insn_commit; // [42:42] + cachepool_peripheral_reg2hw_l1d_private_reg_t l1d_private; // [41:38] + cachepool_peripheral_reg2hw_l1d_addr_reg_t l1d_addr; // [37:6] + cachepool_peripheral_reg2hw_l1d_bypass_reg_t l1d_bypass; // [5:5] cachepool_peripheral_reg2hw_xbar_offset_reg_t xbar_offset; // [4:0] } cachepool_peripheral_reg2hw_t; @@ -167,7 +172,8 @@ package cachepool_peripheral_reg_pkg; parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_OFFSET = 7'h 40; parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_PRIVATE_OFFSET = 7'h 44; parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_ADDR_OFFSET = 7'h 48; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET = 7'h 4c; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_BYPASS_OFFSET = 7'h 4c; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET = 7'h 50; // Reset values for hwext registers and their fields parameter logic [31:0] CACHEPOOL_PERIPHERAL_HW_BARRIER_RESVAL = 32'h 0; @@ -197,11 +203,12 @@ package cachepool_peripheral_reg_pkg; CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS, CACHEPOOL_PERIPHERAL_L1D_PRIVATE, CACHEPOOL_PERIPHERAL_L1D_ADDR, + CACHEPOOL_PERIPHERAL_L1D_BYPASS, CACHEPOOL_PERIPHERAL_XBAR_OFFSET } cachepool_peripheral_id_e; // Register width information to check illegal writes - parameter logic [3:0] CACHEPOOL_PERIPHERAL_PERMIT [20] = '{ + parameter logic [3:0] CACHEPOOL_PERIPHERAL_PERMIT [21] = '{ 4'b 1111, // index[ 0] CACHEPOOL_PERIPHERAL_HW_BARRIER 4'b 1111, // index[ 1] CACHEPOOL_PERIPHERAL_SPATZ_LOCK_ACQUIRE 4'b 1111, // index[ 2] CACHEPOOL_PERIPHERAL_SPATZ_LOCK_RELEASE @@ -221,7 +228,8 @@ package cachepool_peripheral_reg_pkg; 4'b 0001, // index[16] CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS 4'b 0001, // index[17] CACHEPOOL_PERIPHERAL_L1D_PRIVATE 4'b 1111, // index[18] CACHEPOOL_PERIPHERAL_L1D_ADDR - 4'b 0001 // index[19] CACHEPOOL_PERIPHERAL_XBAR_OFFSET + 4'b 0001, // index[19] CACHEPOOL_PERIPHERAL_L1D_BYPASS + 4'b 0001 // index[20] CACHEPOOL_PERIPHERAL_XBAR_OFFSET }; endpackage diff --git a/hardware/cachepool_peripheral/cachepool_peripheral_reg_top.sv b/hardware/cachepool_peripheral/cachepool_peripheral_reg_top.sv index 94ba3f7e..28451432 100644 --- a/hardware/cachepool_peripheral/cachepool_peripheral_reg_top.sv +++ b/hardware/cachepool_peripheral/cachepool_peripheral_reg_top.sv @@ -122,6 +122,9 @@ module cachepool_peripheral_reg_top #( logic [31:0] l1d_addr_qs; logic [31:0] l1d_addr_wd; logic l1d_addr_we; + logic l1d_bypass_qs; + logic l1d_bypass_wd; + logic l1d_bypass_we; logic [4:0] xbar_offset_qs; logic [4:0] xbar_offset_wd; logic xbar_offset_we; @@ -625,6 +628,33 @@ module cachepool_peripheral_reg_top #( ); + // R[l1d_bypass]: V(False) + + prim_subreg #( + .DW (1), + .SWACCESS("RW"), + .RESVAL (1'h0) + ) u_l1d_bypass ( + .clk_i (clk_i ), + .rst_ni (rst_ni ), + + // from register interface + .we (l1d_bypass_we), + .wd (l1d_bypass_wd), + + // from internal hardware + .de (1'b0), + .d ('0 ), + + // to internal hardware + .qe (), + .q (reg2hw.l1d_bypass.q ), + + // to register interface (read) + .qs (l1d_bypass_qs) + ); + + // R[xbar_offset]: V(False) prim_subreg #( @@ -654,7 +684,7 @@ module cachepool_peripheral_reg_top #( - logic [19:0] addr_hit; + logic [20:0] addr_hit; always_comb begin addr_hit = '0; addr_hit[ 0] = (reg_addr == CACHEPOOL_PERIPHERAL_HW_BARRIER_OFFSET); @@ -676,7 +706,8 @@ module cachepool_peripheral_reg_top #( addr_hit[16] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_OFFSET); addr_hit[17] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_PRIVATE_OFFSET); addr_hit[18] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_ADDR_OFFSET); - addr_hit[19] = (reg_addr == CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET); + addr_hit[19] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_BYPASS_OFFSET); + addr_hit[20] = (reg_addr == CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET); end assign addrmiss = (reg_re || reg_we) ? ~|addr_hit : 1'b0 ; @@ -703,7 +734,8 @@ module cachepool_peripheral_reg_top #( (addr_hit[16] & (|(CACHEPOOL_PERIPHERAL_PERMIT[16] & ~reg_be))) | (addr_hit[17] & (|(CACHEPOOL_PERIPHERAL_PERMIT[17] & ~reg_be))) | (addr_hit[18] & (|(CACHEPOOL_PERIPHERAL_PERMIT[18] & ~reg_be))) | - (addr_hit[19] & (|(CACHEPOOL_PERIPHERAL_PERMIT[19] & ~reg_be))))); + (addr_hit[19] & (|(CACHEPOOL_PERIPHERAL_PERMIT[19] & ~reg_be))) | + (addr_hit[20] & (|(CACHEPOOL_PERIPHERAL_PERMIT[20] & ~reg_be))))); end assign hw_barrier_re = addr_hit[0] & reg_re & !reg_error; @@ -762,7 +794,10 @@ module cachepool_peripheral_reg_top #( assign l1d_addr_we = addr_hit[18] & reg_we & !reg_error; assign l1d_addr_wd = reg_wdata[31:0]; - assign xbar_offset_we = addr_hit[19] & reg_we & !reg_error; + assign l1d_bypass_we = addr_hit[19] & reg_we & !reg_error; + assign l1d_bypass_wd = reg_wdata[0]; + + assign xbar_offset_we = addr_hit[20] & reg_we & !reg_error; assign xbar_offset_wd = reg_wdata[4:0]; // Read data return @@ -847,6 +882,10 @@ module cachepool_peripheral_reg_top #( end addr_hit[19]: begin + reg_rdata_next[0] = l1d_bypass_qs; + end + + addr_hit[20]: begin reg_rdata_next[4:0] = xbar_offset_qs; end diff --git a/hardware/src/cachepool_cluster.sv b/hardware/src/cachepool_cluster.sv index 32b0fd4d..440bab78 100644 --- a/hardware/src/cachepool_cluster.sv +++ b/hardware/src/cachepool_cluster.sv @@ -160,11 +160,21 @@ module cachepool_cluster typedef logic [AxiDataWidth-1:0] data_cache_t; typedef logic [AxiDataWidth/8-1:0] strb_cache_t; typedef logic [WideIdWidthOut-1:0] id_cache_slv_t; - typedef logic [AxiUserWidth-1:0] user_cache_t; + // DRAM AXI user: refill user, plus the part index and xbar info of L1 bypass transactions + typedef logic [AxiUserWidth+L1PartIdxWidth+L1BypassInfoWidth-1:0] user_cache_t; // Post-chimney AXI types (used for axi_cut and DRAM output). `AXI_TYPEDEF_ALL(axi_slv_cache, addr_t, id_cache_slv_t, data_cache_t, strb_cache_t, user_cache_t) + // refill_user_t is carried in the DRAM AXI user field; it must not be truncated there. + `ASSERT_INIT(CheckAxiUserFitsRefillUser, AxiUserWidth >= $bits(refill_user_t)) + // The DRAM converters send single beats: a refill must cover a full cache line. + `ASSERT_INIT(CheckRefillIsFullLine, RefillDataWidth == L1LineWidth) + // L1 bypass range [L1D_ADDR, L1BypassEnd): fixed end inside DRAM, start set at runtime + // (checked in cachepool_tile). + `ASSERT_INIT(CheckL1BypassEndInDram, !L1BypassEn || + ((L1BypassEnd <= DramAddr + DramSize) && (L1BypassEnd <= TCDMStartAddr))) + // ---------------- // Wire Definitions // ---------------- @@ -913,23 +923,23 @@ module cachepool_cluster axi_slv_cache_req_t hbm0_dram_axi_req; axi_slv_cache_resp_t hbm0_dram_axi_rsp; - reqrsp_to_axi #( - .MaxTrans ( L2RefillMaxTrans ), - .ID ( 0 ), - .EnBurst ( 1 ), - .ShuffleId ( 1 ), - .AxiIdWidth ( WideIdWidthOut ), - .DataWidth ( AxiDataWidth ), - .UserWidth ( $bits(refill_user_t) ), - .AxiUserWidth ( AxiUserWidth ), - .reqrsp_req_t ( cache_trans_req_t ), - .reqrsp_rsp_t ( cache_trans_rsp_t ), - .axi_req_t ( axi_slv_cache_req_t ), - .axi_rsp_t ( axi_slv_cache_resp_t ) + cachepool_reqrsp_to_axi #( + .MaxTrans ( L2RefillMaxTrans ), + .ID ( 0 ), + .AxiIdWidth ( WideIdWidthOut ), + .DataWidth ( AxiDataWidth ), + .UserWidth ( $bits(refill_user_t) ), + .BaseUserWidth ( AxiUserWidth ), + .BypassDataWidth ( L1CoalDataWidth ), + .BypassPartWidth ( L1PartIdxWidth ), + .BypassInfoWidth ( L1BypassInfoWidth ), + .reqrsp_req_t ( cache_trans_req_t ), + .reqrsp_rsp_t ( cache_trans_rsp_t ), + .axi_req_t ( axi_slv_cache_req_t ), + .axi_rsp_t ( axi_slv_cache_resp_t ) ) i_hbm0_dram_reqrsp_to_axi ( .clk_i ( clk_i ), .rst_ni ( rst_ni ), - .user_i ( '0 ), .reqrsp_req_i ( hbm0_demux_req[0] ), .reqrsp_rsp_o ( hbm0_demux_rsp[0] ), .axi_req_o ( hbm0_dram_axi_req ), @@ -951,6 +961,10 @@ module cachepool_cluster hbm0_bootrom_req.q.addr = revertAddr(hbm0_demux_req[1].q.addr); end + // L1 bypass addresses are in DRAM; the BootROM port must never see one. + `ASSERT(NoL1BypassToBootrom, + hbm0_demux_req[1].q_valid |-> !hbm0_demux_req[1].q.user.bypass) + reqrsp_to_reg #( .AddrWidth ( AxiAddrWidth ), .DataWidth ( AxiDataWidth ), @@ -992,23 +1006,23 @@ module cachepool_cluster axi_slv_cache_req_t hbm_axi_req; axi_slv_cache_resp_t hbm_axi_rsp; - reqrsp_to_axi #( - .MaxTrans ( L2RefillMaxTrans ), - .ID ( 0 ), - .EnBurst ( 1 ), - .ShuffleId ( 1 ), - .AxiIdWidth ( WideIdWidthOut ), - .DataWidth ( AxiDataWidth ), - .UserWidth ( $bits(refill_user_t) ), - .AxiUserWidth ( AxiUserWidth ), - .reqrsp_req_t ( cache_trans_req_t ), - .reqrsp_rsp_t ( cache_trans_rsp_t ), - .axi_req_t ( axi_slv_cache_req_t ), - .axi_rsp_t ( axi_slv_cache_resp_t ) + cachepool_reqrsp_to_axi #( + .MaxTrans ( L2RefillMaxTrans ), + .ID ( 0 ), + .AxiIdWidth ( WideIdWidthOut ), + .DataWidth ( AxiDataWidth ), + .UserWidth ( $bits(refill_user_t) ), + .BaseUserWidth ( AxiUserWidth ), + .BypassDataWidth ( L1CoalDataWidth ), + .BypassPartWidth ( L1PartIdxWidth ), + .BypassInfoWidth ( L1BypassInfoWidth ), + .reqrsp_req_t ( cache_trans_req_t ), + .reqrsp_rsp_t ( cache_trans_rsp_t ), + .axi_req_t ( axi_slv_cache_req_t ), + .axi_rsp_t ( axi_slv_cache_resp_t ) ) i_hbm_reqrsp_to_axi ( .clk_i ( clk_i ), .rst_ni ( rst_ni ), - .user_i ( '0 ), .reqrsp_req_i ( hbm_reqrsp_req ), .reqrsp_rsp_o ( hbm_reqrsp_rsp ), .axi_req_o ( hbm_axi_req ), @@ -1205,23 +1219,23 @@ module cachepool_cluster axi_slv_cache_req_t hbm_axi_req; axi_slv_cache_resp_t hbm_axi_rsp; - reqrsp_to_axi #( - .MaxTrans ( L2RefillMaxTrans ), - .ID ( 0 ), - .EnBurst ( 1 ), - .ShuffleId ( 1 ), - .AxiIdWidth ( WideIdWidthOut ), - .DataWidth ( AxiDataWidth ), - .UserWidth ( $bits(refill_user_t) ), - .AxiUserWidth ( AxiUserWidth ), - .reqrsp_req_t ( cache_trans_req_t ), - .reqrsp_rsp_t ( cache_trans_rsp_t ), - .axi_req_t ( axi_slv_cache_req_t ), - .axi_rsp_t ( axi_slv_cache_resp_t ) + cachepool_reqrsp_to_axi #( + .MaxTrans ( L2RefillMaxTrans ), + .ID ( 0 ), + .AxiIdWidth ( WideIdWidthOut ), + .DataWidth ( AxiDataWidth ), + .UserWidth ( $bits(refill_user_t) ), + .BaseUserWidth ( AxiUserWidth ), + .BypassDataWidth ( L1CoalDataWidth ), + .BypassPartWidth ( L1PartIdxWidth ), + .BypassInfoWidth ( L1BypassInfoWidth ), + .reqrsp_req_t ( cache_trans_req_t ), + .reqrsp_rsp_t ( cache_trans_rsp_t ), + .axi_req_t ( axi_slv_cache_req_t ), + .axi_rsp_t ( axi_slv_cache_resp_t ) ) i_hbm_reqrsp_to_axi ( .clk_i ( clk_i ), .rst_ni ( rst_ni ), - .user_i ( '0 ), .reqrsp_req_i ( hbm_reqrsp_req ), .reqrsp_rsp_o ( hbm_reqrsp_rsp ), .axi_req_o ( hbm_axi_req ), @@ -1329,23 +1343,23 @@ module cachepool_cluster axi_slv_cache_req_t hbm_axi_req; axi_slv_cache_resp_t hbm_axi_rsp; - reqrsp_to_axi #( - .MaxTrans ( L2RefillMaxTrans ), - .ID ( 0 ), - .EnBurst ( 1 ), - .ShuffleId ( 1 ), - .AxiIdWidth ( WideIdWidthOut ), - .DataWidth ( AxiDataWidth ), - .UserWidth ( $bits(refill_user_t) ), - .AxiUserWidth ( AxiUserWidth ), - .reqrsp_req_t ( cache_trans_req_t ), - .reqrsp_rsp_t ( cache_trans_rsp_t ), - .axi_req_t ( axi_slv_cache_req_t ), - .axi_rsp_t ( axi_slv_cache_resp_t ) + cachepool_reqrsp_to_axi #( + .MaxTrans ( L2RefillMaxTrans ), + .ID ( 0 ), + .AxiIdWidth ( WideIdWidthOut ), + .DataWidth ( AxiDataWidth ), + .UserWidth ( $bits(refill_user_t) ), + .BaseUserWidth ( AxiUserWidth ), + .BypassDataWidth ( L1CoalDataWidth ), + .BypassPartWidth ( L1PartIdxWidth ), + .BypassInfoWidth ( L1BypassInfoWidth ), + .reqrsp_req_t ( cache_trans_req_t ), + .reqrsp_rsp_t ( cache_trans_rsp_t ), + .axi_req_t ( axi_slv_cache_req_t ), + .axi_rsp_t ( axi_slv_cache_resp_t ) ) i_hbm_reqrsp_to_axi ( .clk_i ( clk_i ), .rst_ni ( rst_ni ), - .user_i ( '0 ), .reqrsp_req_i ( hbm_reqrsp_req ), .reqrsp_rsp_o ( hbm_reqrsp_rsp ), .axi_req_o ( hbm_axi_req ), @@ -1446,23 +1460,23 @@ module cachepool_cluster axi_slv_cache_req_t hbm_axi_req; axi_slv_cache_resp_t hbm_axi_rsp; - reqrsp_to_axi #( - .MaxTrans ( L2RefillMaxTrans ), - .ID ( 0 ), - .EnBurst ( 1 ), - .ShuffleId ( 1 ), - .AxiIdWidth ( WideIdWidthOut ), - .DataWidth ( AxiDataWidth ), - .UserWidth ( $bits(refill_user_t) ), - .AxiUserWidth ( AxiUserWidth ), - .reqrsp_req_t ( cache_trans_req_t ), - .reqrsp_rsp_t ( cache_trans_rsp_t ), - .axi_req_t ( axi_slv_cache_req_t ), - .axi_rsp_t ( axi_slv_cache_resp_t ) + cachepool_reqrsp_to_axi #( + .MaxTrans ( L2RefillMaxTrans ), + .ID ( 0 ), + .AxiIdWidth ( WideIdWidthOut ), + .DataWidth ( AxiDataWidth ), + .UserWidth ( $bits(refill_user_t) ), + .BaseUserWidth ( AxiUserWidth ), + .BypassDataWidth ( L1CoalDataWidth ), + .BypassPartWidth ( L1PartIdxWidth ), + .BypassInfoWidth ( L1BypassInfoWidth ), + .reqrsp_req_t ( cache_trans_req_t ), + .reqrsp_rsp_t ( cache_trans_rsp_t ), + .axi_req_t ( axi_slv_cache_req_t ), + .axi_rsp_t ( axi_slv_cache_resp_t ) ) i_hbm_reqrsp_to_axi ( .clk_i ( clk_i ), .rst_ni ( rst_ni ), - .user_i ( '0 ), .reqrsp_req_i ( hbm_reqrsp_req ), .reqrsp_rsp_o ( hbm_reqrsp_rsp ), .axi_req_o ( hbm_axi_req ), diff --git a/hardware/src/cachepool_group.sv b/hardware/src/cachepool_group.sv index e95ccc81..3c0b40be 100644 --- a/hardware/src/cachepool_group.sv +++ b/hardware/src/cachepool_group.sv @@ -137,6 +137,8 @@ module cachepool_group input axi_addr_t cluster_base_addr_i, /// Partitioning address input axi_addr_t private_start_addr_i, + /// L1 bypass enable: the private range skips the L1 data cache + input logic l1d_bypass_i, /// L2 refill reqrsp port (single merged output: cache refill + iCache) output l2_req_t l2_req_o, input l2_rsp_t l2_rsp_i, @@ -406,10 +408,11 @@ module cachepool_group tile_req_chan [xbar_idx].user.tile_id = '0; tile_req_chan [xbar_idx].user.bank_id = BankIdICache; tile_req_chan [xbar_idx].user.l2_src_id = l2_group_id_i; - // axi_to_reqrsp puts the AXI ID in the user LSBs (burst + info). Move it into info - // and zero burst, so reqrsp_to_axi sees is_burst=0 and keeps ID=0 for iCache. + // axi_to_reqrsp puts the AXI ID in the user LSBs (uniq_id + info). Move it into info + // and clear uniq_id, so the DRAM converter keeps the fixed ID 0 for iCache. tile_req_chan [xbar_idx].user.info = cache_info_t'(cache_l2icache_req.q.user); - tile_req_chan [xbar_idx].user.burst = '0; + tile_req_chan [xbar_idx].user.uniq_id = 1'b0; + tile_req_chan [xbar_idx].user.bypass = 1'b0; end else begin tile_req_chan [xbar_idx] = '0; tile_req_valid [xbar_idx] = 1'b0; @@ -682,6 +685,7 @@ module cachepool_group .cluster_base_addr_i ( cluster_base_addr_i ), .tile_id_i ( tile_id ), .private_start_addr_i ( private_start_addr_i ), + .l1d_bypass_i ( l1d_bypass_i ), // Remote Access Ports .remote_req_o ( tile_remote_out_req [t] ), .remote_req_dst_o ( remote_out_sel_tile [t] ), @@ -775,6 +779,7 @@ module cachepool_group .cluster_base_addr_i ( cluster_base_addr_i ), .tile_id_i ( tile_id ), .private_start_addr_i ( private_start_addr_i ), + .l1d_bypass_i ( l1d_bypass_i ), // Remote Access Ports .remote_req_o ( tile_remote_out_req [t] ), .remote_req_dst_o ( remote_out_sel_tile [t] ), diff --git a/hardware/src/cachepool_group_cfg.sv b/hardware/src/cachepool_group_cfg.sv index b236630d..86c3b016 100644 --- a/hardware/src/cachepool_group_cfg.sv +++ b/hardware/src/cachepool_group_cfg.sv @@ -33,6 +33,7 @@ module cachepool_group_cfg output logic [L1PrivWidth-1:0] l1d_private_o, output axi_addr_t private_start_addr_o, output logic icache_prefetch_enable_o, + output logic l1d_bypass_o, output cache_insn_t l1d_insn_o, output logic l1d_insn_valid_o, input logic [NumTilesPerGroup-1:0] l1d_insn_ready_i @@ -84,6 +85,7 @@ module cachepool_group_cfg GroupCfgL1dPrivate: cfg_d.l1d_private = req_data[L1PrivWidth-1:0]; GroupCfgL1dAddr: cfg_d.private_addr = req_data; GroupCfgPrefetch: cfg_d.prefetch_en = req_data[0]; + GroupCfgL1dBypass: cfg_d.bypass_en = req_data[0]; GroupCfgL1dInsn: begin insn_d = req_insn; insn_valid_d = 1'b1; @@ -123,6 +125,7 @@ module cachepool_group_cfg assign l1d_private_o = cfg_q.l1d_private; assign private_start_addr_o = cfg_q.private_addr; assign icache_prefetch_enable_o = cfg_q.prefetch_en; + assign l1d_bypass_o = cfg_q.bypass_en; assign l1d_insn_o = insn_q; assign l1d_insn_valid_o = insn_valid_q; @@ -135,7 +138,7 @@ module cachepool_group_cfg `ASSERT(CfgAddrInRange, req_valid_i |-> (req_i.addr >= GroupCfgBase), clk_i, !rst_ni, "cachepool_group_cfg: request address below GroupCfgBase") `ASSERT(KnownCfgReg, req_valid_i |-> (req_reg inside {GroupCfgXbarOffset, GroupCfgL1dPrivate, - GroupCfgL1dAddr, GroupCfgPrefetch, GroupCfgL1dInsn}), clk_i, !rst_ni, + GroupCfgL1dAddr, GroupCfgPrefetch, GroupCfgL1dInsn, GroupCfgL1dBypass}), clk_i, !rst_ni, "cachepool_group_cfg: write to an unknown register") endmodule diff --git a/hardware/src/cachepool_group_noc_wrapper.sv b/hardware/src/cachepool_group_noc_wrapper.sv index 00bf276c..53024bf1 100644 --- a/hardware/src/cachepool_group_noc_wrapper.sv +++ b/hardware/src/cachepool_group_noc_wrapper.sv @@ -179,6 +179,7 @@ module cachepool_group_noc_wrapper xbar_offset_t dynamic_offset_q; logic [L1PrivWidth-1:0] l1d_private_q; axi_addr_t private_start_addr_q; + logic l1d_bypass_q; logic icache_prefetch_enable; cache_insn_t l1d_insn_q; logic l1d_insn_valid_q; @@ -841,6 +842,7 @@ module cachepool_group_noc_wrapper .dynamic_offset_o ( dynamic_offset_q ), .l1d_private_o ( l1d_private_q ), .private_start_addr_o ( private_start_addr_q ), + .l1d_bypass_o ( l1d_bypass_q ), .icache_prefetch_enable_o ( icache_prefetch_enable ), .l1d_insn_o ( l1d_insn_q ), .l1d_insn_valid_o ( l1d_insn_valid_q ), @@ -980,6 +982,7 @@ module cachepool_group_noc_wrapper .tile_base_id_i ( tile_base_id_i ), .cluster_base_addr_i ( cluster_base_addr_i ), .private_start_addr_i ( private_start_addr_q ), + .l1d_bypass_i ( l1d_bypass_q ), .l2_req_o ( l2_group_req ), .l2_rsp_i ( l2_group_rsp ), .l2_group_id_i ( l2_id_i ), diff --git a/hardware/src/cachepool_pkg.sv b/hardware/src/cachepool_pkg.sv index 4e63a262..a910e118 100644 --- a/hardware/src/cachepool_pkg.sv +++ b/hardware/src/cachepool_pkg.sv @@ -224,7 +224,9 @@ package cachepool_pkg; // per-config axi_user_width define: that breaks silently whenever NumEndpoints changes width. localparam int unsigned SpatzAxiUserWidth = `ifdef AXI_USER_WIDTH `AXI_USER_WIDTH `else 0 `endif + $clog2(NumTiles) - + $bits(floo_cachepool_noc_pkg::id_t); + + $bits(floo_cachepool_noc_pkg::id_t) + // refill_user_t.bypass + + 1; // ----------------------- // AXI ID field structure @@ -338,6 +340,12 @@ package cachepool_pkg; localparam int unsigned DramSize = 32'h4000_0000; localparam int unsigned DramPerChSize = DramSize / NumL2Channel; + // L1 bypass (built if L1BypassEn; enabled at runtime by L1D_BYPASS): the private range + // [L1D_ADDR, L1BypassEnd) goes around the L1 cache straight to DRAM. The fixed end keeps a + // stack overflowing the stack SPM cached. + localparam bit L1BypassEn = 1'b1; + localparam int unsigned L1BypassEnd = TCDMStartAddr; + // Currently set to 16 for now parameter int unsigned Interleave = `ifdef L2_INTERLEAVE `L2_INTERLEAVE `else 0 `endif; @@ -404,6 +412,7 @@ package cachepool_pkg; typedef logic [L1AddrWidth-1:0] narrow_addr_t; typedef logic [SPMAddrWidth-1:0] spm_addr_t; + // Burst port type of cachepool_cache_ctrl (refill_burst_o); not carried on the refill path. typedef struct packed { logic is_burst; burst_len_t burst_len; @@ -432,13 +441,16 @@ package cachepool_pkg; axi_addr_t private_addr; xbar_offset_t xbar_offset; logic prefetch_en; + // L1 bypass of the private range (see L1BypassEnd) + logic bypass_en; } l1d_cfg_t; localparam l1d_cfg_t L1dCfgRst = '{ l1d_private: L1dPrivateRst, private_addr: L1dPrivateAddrRst, xbar_offset: L1dXbarOffsetRst, - prefetch_en: ICachePrefetchRst + prefetch_en: ICachePrefetchRst, + bypass_en: 1'b0 }; // Registers of a group-level configuration register block, one 32-bit word each at @@ -448,7 +460,8 @@ package cachepool_pkg; GroupCfgL1dPrivate = 3'd1, GroupCfgL1dAddr = 3'd2, GroupCfgPrefetch = 3'd3, - GroupCfgL1dInsn = 3'd4 + GroupCfgL1dInsn = 3'd4, + GroupCfgL1dBypass = 3'd5 } group_cfg_reg_e; typedef struct packed { @@ -468,17 +481,43 @@ package cachepool_pkg; logic sc_fail; } tcdm_user_t; + // Width of the cache controller xbar info (coalescer_xbar_info_union_t in cachepool_cache_ctrl), + // which an L1 bypass transaction carries in the refill data and in the DRAM AXI user. + // Mirrors the folding setup of cachepool_tile; the controller checks it at elaboration. + localparam bit L1UseFolded = `ifdef L1D_USE_FOLDED `L1D_USE_FOLDED `else 1'b1 `endif; + localparam int unsigned L1FoldWayGroup = `ifdef L1D_FOLD_WAY_GROUP `L1D_FOLD_WAY_GROUP `else 0 `endif; + localparam bit L1SkewedFolded = L1UseFolded && (L1AssoPerCtrl > 1); + localparam int unsigned L1PartSplit = !L1SkewedFolded ? 1 : + (L1FoldWayGroup != 0) ? L1FoldWayGroup : + (L1AssoPerCtrl >= 4) ? 4 : 2; + localparam int unsigned L1CoalDataWidth = L1LineWidth / L1PartSplit; + localparam int unsigned L1CoalPorts = (NrTCDMPortsPerCore - 1) + * (L1SkewedFolded ? 1 : L1CoalFactor); + localparam int unsigned L1PartIdxWidth = (L1PartSplit > 1) ? $clog2(L1PartSplit) : 1; + // part_idx + id + bypass_coalescer + per port {hitmap, ofsts, infos} + localparam int unsigned L1BypassInfoWidth = L1PartIdxWidth + 2 + + L1CoalPorts * (1 + $clog2(L1CoalDataWidth / $bits(narrow_data_t)) + + $bits(tcdm_user_t)); + typedef struct packed { + // L1 bypass transaction: refill data carries the cache controller xbar payload + logic bypass; logic [BankIDWidth-1:0] bank_id; logic [TileIDWidth-1:0] tile_id; // L2 mesh source endpoint, used by the HBM chimney to route the response (unlike the - // group-local tile_id). Must stay above info/burst: the iCache EnUserIdPassthrough path in + // group-local tile_id). Must stay above info/uniq_id: the iCache EnUserIdPassthrough path in // cachepool_group.sv truncates this struct to cache_info_t and needs them at the LSBs. floo_cachepool_noc_pkg::id_t l2_src_id; cache_info_t info; - burst_req_t burst; + // 1: send with a fresh AXI ID (response may return out of order); 0: fixed ID, in order + logic uniq_id; } refill_user_t; + // DRAM AXI user: refill_user_t (zero-extended to SpatzAxiUserWidth), plus the part index and + // xbar info of an L1 bypass transaction, so they return with the DRAM response. + localparam int unsigned DramAxiUserWidth = SpatzAxiUserWidth + L1PartIdxWidth + L1BypassInfoWidth; + typedef logic [DramAxiUserWidth-1:0] dram_axi_user_t; + /////////////////// // GROUP TYPES // /////////////////// @@ -599,12 +638,14 @@ package cachepool_pkg; logic write; refill_data_t wdata; refill_strb_t wstrb; + logic bypass; } cache_refill_req_chan_t; typedef struct packed { logic write; refill_data_t data; cache_info_t info; + logic bypass; } cache_refill_rsp_chan_t; ////////////////// @@ -676,9 +717,9 @@ package cachepool_pkg; // AXI typedef bundles `AXI_TYPEDEF_ALL(spatz_axi_narrow, axi_addr_t, axi_narrow_id_t, axi_narrow_data_t, axi_narrow_strb_t, axi_user_t) `AXI_TYPEDEF_ALL(spatz_axi_in, axi_addr_t, axi_id_in_t, axi_narrow_data_t, axi_narrow_strb_t, axi_user_t) - `AXI_TYPEDEF_ALL(spatz_axi_out, axi_addr_t, axi_id_out_t, axi_wide_data_t, axi_wide_strb_t, axi_user_t) + `AXI_TYPEDEF_ALL(spatz_axi_out, axi_addr_t, axi_id_out_t, axi_wide_data_t, axi_wide_strb_t, dram_axi_user_t) // Wrapper-level external output type: ID from SpatzAxiIdOutWidth to WrapperAxiIdOutWidth. - `AXI_TYPEDEF_ALL(spatz_axi_wrapper_out, axi_addr_t, axi_id_wrapper_out_t, axi_wide_data_t, axi_wide_strb_t, axi_user_t) + `AXI_TYPEDEF_ALL(spatz_axi_wrapper_out, axi_addr_t, axi_id_wrapper_out_t, axi_wide_data_t, axi_wide_strb_t, dram_axi_user_t) // Wrapper-level external narrow output type: ID compressed from SpatzAxiUartIdWidth to WrapperAxiNarrowIdOutWidth. `AXI_TYPEDEF_ALL(spatz_axi_wrapper_narrow_out, axi_addr_t, axi_id_wrapper_narrow_out_t, axi_narrow_data_t, axi_narrow_strb_t, axi_user_t) diff --git a/hardware/src/cachepool_reqrsp_to_axi.sv b/hardware/src/cachepool_reqrsp_to_axi.sv new file mode 100644 index 00000000..3c3c33e0 --- /dev/null +++ b/hardware/src/cachepool_reqrsp_to_axi.sv @@ -0,0 +1,236 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 + +// Author: Diyou Shen + +`include "common_cells/registers.svh" +`include "common_cells/assertions.svh" + +// REQRSP to AXI for the off-chip (DRAM) refill path, from reqrsp_to_axi: single beats, no atomics, +// reads and writes never in flight together. user.uniq_id selects a fresh AXI ID (else the fixed +// ID); the reqrsp user rides in the AXI user, which the memory must echo back. + +// L1 bypass (user.bypass): data in data[BypassDataWidth-1:0], xbar info above it. AXI user is +// {info, part = addr[part bits], reqrsp user}; write data/strb move to the part, and the read +// part returns in slot 0 with the info above it. +module cachepool_reqrsp_to_axi #( + /// Number of same transactions which can be in-flight simultaneously, greater than 1. + parameter int unsigned MaxTrans = 32, + /// Fixed AXI ID, used when user.uniq_id is 0. + parameter int unsigned ID = 0, + parameter int unsigned AxiIdWidth = 8, + parameter int unsigned DataWidth = 512, + /// Width of the reqrsp user, and of its zero-extended slot in the AXI user. + parameter int unsigned UserWidth = 8, + parameter int unsigned BaseUserWidth = UserWidth, + /// L1 bypass data slot, part index and info widths. + parameter int unsigned BypassDataWidth = 128, + parameter int unsigned BypassPartWidth = 2, + parameter int unsigned BypassInfoWidth = 1, + parameter type reqrsp_req_t = logic, + parameter type reqrsp_rsp_t = logic, + parameter type axi_req_t = logic, + parameter type axi_rsp_t = logic, + // Dependent parameter, do not override. + localparam int unsigned AxiUserWidth = BaseUserWidth + BypassPartWidth + BypassInfoWidth +) ( + input logic clk_i, + input logic rst_ni, + input reqrsp_req_t reqrsp_req_i, + output reqrsp_rsp_t reqrsp_rsp_o, + output axi_req_t axi_req_o, + input axi_rsp_t axi_rsp_i +); + + localparam int unsigned CounterWidth = cf_math_pkg::idx_width(MaxTrans); + localparam int unsigned BypassStrbWidth = BypassDataWidth / 8; + localparam bit BypassSupported = (BypassDataWidth + BypassInfoWidth <= DataWidth); + + typedef logic [CounterWidth-1:0] cnt_t; + typedef logic [AxiUserWidth-1:0] axi_user_t; + typedef logic [DataWidth-1:0] data_t; + typedef logic [DataWidth/8-1:0] strb_t; + + // ---------------- + // Ordering control + // ---------------- + // A new transaction stalls while the other type is in flight or its counter would overflow. + cnt_t read_cnt_d, read_cnt_q; + cnt_t write_cnt_d, write_cnt_q; + logic [AxiIdWidth-1:0] id_cnt_d, id_cnt_q; + + logic is_write, q_valid, q_ready; + logic q_valid_read, q_ready_read, q_valid_write, q_ready_write; + logic stall_read, dec_read_cnt, read_cnt_not_zero; + logic stall_write, dec_write_cnt, write_cnt_not_zero; + logic r_valid, r_ready, b_valid, b_ready; + + assign is_write = reqrsp_req_i.q.write; + assign r_valid = axi_rsp_i.r_valid; + assign b_valid = axi_rsp_i.b_valid; + assign dec_read_cnt = r_valid & r_ready & axi_rsp_i.r.last; + assign dec_write_cnt = b_valid & b_ready; + + assign read_cnt_not_zero = (read_cnt_q > 1) | ((read_cnt_q == 1) & ~dec_read_cnt); + assign write_cnt_not_zero = (write_cnt_q > 1) | ((write_cnt_q == 1) & ~dec_write_cnt); + assign stall_write = is_write & ((write_cnt_q == MaxTrans-1) | read_cnt_not_zero); + assign stall_read = !is_write & ((read_cnt_q == MaxTrans-1) | write_cnt_not_zero); + + assign q_valid = reqrsp_req_i.q_valid & ~(stall_write | stall_read); + assign q_valid_read = q_valid & ~is_write; + assign q_valid_write = q_valid & is_write; + assign q_ready = (q_valid_read & q_ready_read) | (q_valid_write & q_ready_write); + + `FF(read_cnt_q, read_cnt_d, '0) + `FF(write_cnt_q, write_cnt_d, '0) + // Start above the fixed ID to avoid a collision with it + `FF(id_cnt_q, id_cnt_d, AxiIdWidth'($unsigned(ID) + 1)) + + always_comb begin + read_cnt_d = read_cnt_q; + write_cnt_d = write_cnt_q; + id_cnt_d = id_cnt_q; + if (q_valid && q_ready) begin + if (is_write) write_cnt_d = write_cnt_d + 1; + else read_cnt_d = read_cnt_d + 1; + id_cnt_d = (&id_cnt_q) ? AxiIdWidth'($unsigned(ID) + 1) : id_cnt_q + 1; + end + if (dec_read_cnt) read_cnt_d = read_cnt_d - 1; + if (dec_write_cnt) write_cnt_d = write_cnt_d - 1; + end + + // ---------------- + // Request payload + // ---------------- + logic q_bypass; + logic [BypassPartWidth-1:0] q_part; + logic [AxiIdWidth-1:0] q_id; + axi_user_t q_axi_user; + data_t q_wdata; + strb_t q_wstrb; + + assign q_bypass = reqrsp_req_i.q.user.bypass; + assign q_part = reqrsp_req_i.q.addr[$clog2(BypassStrbWidth) +: BypassPartWidth]; + assign q_id = reqrsp_req_i.q.user.uniq_id ? id_cnt_q : AxiIdWidth'($unsigned(ID)); + + always_comb begin + q_axi_user = '0; + q_axi_user[UserWidth-1:0] = reqrsp_req_i.q.user; + q_wdata = reqrsp_req_i.q.data; + q_wstrb = reqrsp_req_i.q.strb; + if (BypassSupported && q_bypass) begin + q_axi_user[BaseUserWidth +: BypassPartWidth] = q_part; + q_axi_user[BaseUserWidth + BypassPartWidth +: BypassInfoWidth] = + reqrsp_req_i.q.data[BypassDataWidth +: BypassInfoWidth]; + q_wdata = '0; + q_wstrb = '0; + q_wdata[q_part * BypassDataWidth +: BypassDataWidth] = + reqrsp_req_i.q.data[BypassDataWidth-1:0]; + q_wstrb[q_part * BypassStrbWidth +: BypassStrbWidth] = + reqrsp_req_i.q.strb[BypassStrbWidth-1:0]; + end + end + + // ------------- + // Read Channel + // ------------- + assign axi_req_o.ar.id = q_id; + assign axi_req_o.ar.addr = reqrsp_req_i.q.addr; + assign axi_req_o.ar.len = '0; + assign axi_req_o.ar.size = {1'b0, reqrsp_req_i.q.size}; + assign axi_req_o.ar.burst = axi_pkg::BURST_INCR; + assign axi_req_o.ar.lock = '0; + assign axi_req_o.ar.cache = axi_pkg::CACHE_MODIFIABLE; + assign axi_req_o.ar.prot = '0; + assign axi_req_o.ar.qos = '0; + assign axi_req_o.ar.region = '0; + assign axi_req_o.ar.user = q_axi_user; + assign axi_req_o.ar_valid = q_valid_read; + assign q_ready_read = axi_rsp_i.ar_ready; + + // ------------- + // Write Channel + // ------------- + assign axi_req_o.aw.id = q_id; + assign axi_req_o.aw.addr = reqrsp_req_i.q.addr; + assign axi_req_o.aw.len = '0; + assign axi_req_o.aw.size = {1'b0, reqrsp_req_i.q.size}; + assign axi_req_o.aw.burst = axi_pkg::BURST_INCR; + assign axi_req_o.aw.lock = '0; + assign axi_req_o.aw.cache = axi_pkg::CACHE_MODIFIABLE; + assign axi_req_o.aw.prot = '0; + assign axi_req_o.aw.qos = '0; + assign axi_req_o.aw.region = '0; + assign axi_req_o.aw.atop = '0; + assign axi_req_o.aw.user = q_axi_user; + assign axi_req_o.w.data = q_wdata; + assign axi_req_o.w.strb = q_wstrb; + assign axi_req_o.w.last = 1'b1; + assign axi_req_o.w.user = q_axi_user; + + // AW and W handshake independently + stream_fork #( + .N_OUP (2) + ) i_stream_fork ( + .clk_i, + .rst_ni, + .valid_i (q_valid_write ), + .ready_o (q_ready_write ), + .valid_o ({axi_req_o.aw_valid, axi_req_o.w_valid} ), + .ready_i ({axi_rsp_i.aw_ready, axi_rsp_i.w_ready} ) + ); + + assign reqrsp_rsp_o.q_ready = q_ready; + + // ----------- + // Return Path + // ----------- + // Reads and writes are never in flight together, so R and B are never valid together. + assign r_ready = reqrsp_req_i.p_ready & r_valid; + assign b_ready = reqrsp_req_i.p_ready & b_valid; + assign axi_req_o.r_ready = r_ready; + assign axi_req_o.b_ready = b_ready; + + axi_user_t p_axi_user; + logic p_bypass; + logic [BypassPartWidth-1:0] p_part; + + assign p_axi_user = r_valid ? axi_rsp_i.r.user : axi_rsp_i.b.user; + assign p_part = p_axi_user[BaseUserWidth +: BypassPartWidth]; + + always_comb begin + reqrsp_rsp_o.p_valid = r_valid | b_valid; + reqrsp_rsp_o.p.user = p_axi_user[UserWidth-1:0]; + reqrsp_rsp_o.p.write = b_valid; + reqrsp_rsp_o.p.error = (r_valid & axi_rsp_i.r.resp[1]) | (b_valid & axi_rsp_i.b.resp[1]); + reqrsp_rsp_o.p.data = r_valid ? axi_rsp_i.r.data : '0; + p_bypass = reqrsp_rsp_o.p.user.bypass; + if (BypassSupported && p_bypass) begin + reqrsp_rsp_o.p.data = '0; + if (r_valid) begin + reqrsp_rsp_o.p.data[BypassDataWidth-1:0] = + axi_rsp_i.r.data[p_part * BypassDataWidth +: BypassDataWidth]; + end + reqrsp_rsp_o.p.data[BypassDataWidth +: BypassInfoWidth] = + p_axi_user[BaseUserWidth + BypassPartWidth +: BypassInfoWidth]; + end + end + + // ---------- + // Assertions + // ---------- + `ASSERT_INIT(MaxTransGreaterThanOne, MaxTrans > 1) + `ASSERT_INIT(CheckAxiUserWidth, $bits(axi_req_o.ar.user) == AxiUserWidth) + `ASSERT_INIT(CheckBaseUserWidth, BaseUserWidth >= UserWidth) + `ASSERT(WriteCntOverflow, (write_cnt_q == MaxTrans - 1) |=> !(write_cnt_q == 0)) + `ASSERT(ReadCntOverflow, (read_cnt_q == MaxTrans - 1) |=> !(read_cnt_q == 0)) + `ASSERT(WriteCntUnderflow, (write_cnt_q == 0) |=> !(write_cnt_q == MaxTrans - 1)) + `ASSERT(ReadCntUnderflow, (read_cnt_q == 0) |=> !(read_cnt_q == MaxTrans - 1)) + `ASSERT(WriteStable, q_valid_write && !q_ready_write |=> q_valid_write) + `ASSERT(NoReadWriteRspOverlap, !(r_valid && b_valid)) + `ASSERT(SingleBeatRead, r_valid |-> axi_rsp_i.r.last) + `ASSERT(NoUnsupportedBypass, + reqrsp_req_i.q_valid |-> (BypassSupported || !reqrsp_req_i.q.user.bypass)) + +endmodule : cachepool_reqrsp_to_axi diff --git a/hardware/src/cachepool_tile.sv b/hardware/src/cachepool_tile.sv index a673148e..eeff5b1a 100644 --- a/hardware/src/cachepool_tile.sv +++ b/hardware/src/cachepool_tile.sv @@ -149,6 +149,8 @@ module cachepool_tile input remote_tile_sel_t tile_id_i, /// Partitioning address input axi_addr_t private_start_addr_i, + /// L1 bypass enable: the private range skips the L1 data cache + input logic l1d_bypass_i, /// Cache Refill ports output cache_trans_req_t [NumL1CtrlTile-1:0] cache_refill_req_o, input cache_trans_rsp_t [NumL1CtrlTile-1:0] cache_refill_rsp_i, @@ -955,12 +957,7 @@ module cachepool_tile end `endif - // CL-offset mask: bits below dynamic_offset_q, verbatim in both directions. - logic [SpatzAxiAddrWidth-1:0] bitmask_lo; - assign bitmask_lo = (SpatzAxiAddrWidth'(1) << dynamic_offset_q) - 1; - cache_refill_req_chan_t [NumL1CtrlTile-1 : 0] cache_refill_req; - burst_req_t [NumL1CtrlTile-1 : 0] cache_refill_burst; logic [NumL1CtrlTile-1 : 0] cache_refill_req_valid, cache_refill_req_ready; cache_refill_rsp_chan_t [NumL1CtrlTile-1 : 0] cache_refill_rsp; logic [NumL1CtrlTile-1 : 0] cache_refill_rsp_valid, cache_refill_rsp_ready; @@ -1052,6 +1049,18 @@ module cachepool_tile assign l1d_insn_ready_o = flush_pending_q && (cache_flush_d == '0); for (genvar cb = 0; cb < NumL1CtrlTile; cb++) begin: gen_l1_cache_ctrl + // Address rotation applied by tcdm_cache_interco (routing bits moved to the MSBs). + // N = refill_bits_to_rotate, per bank like gen_scramble; num_private_cache is runtime. + logic [RefillRotWidth-1:0] refill_bits_to_rotate; + + // Mirrors gen_scramble in tcdm_cache_interco: banks below the boundary are + // private and rotate BankSel only, the rest also rotate TileID. cb is a + // genvar constant, so this covers all-private and all-shared as well. + assign refill_bits_to_rotate = (cb < int'(num_private_cache)) + ? (group_fold ? RefillRotWidth'(RefillCacheBankBits + RefillLocalTileBits) + : RefillRotWidth'(RefillCacheBankBits)) + : RefillRotWidth'(RefillCacheBankBits + RefillTileBits); + cachepool_cache_ctrl #( // Core .NumPorts (NrTCDMPortsPerCore ), @@ -1070,6 +1079,11 @@ module cachepool_tile .BankFactor (L1BankFactor ), // .LogDebug (0 ), .RefillDataWidth (RefillDataWidth ), + .L1BypassEn (L1BypassEn ), + .L1BypassEnd (L1BypassEnd ), + .AddrRotWidth (RefillRotWidth ), + .L1BypassInfoWidth(L1BypassInfoWidth ), + .AddrRotOfstWidth ($bits(dynamic_offset_q)), // Type .core_meta_t (tcdm_user_t ), .impl_in_t (impl_in_t ), @@ -1086,6 +1100,12 @@ module cachepool_tile .cache_sync_insn_i (ctrl_sync_insn[cb] ), // SPM size: computed differently from other modules (times 2); assumes a full cache .bank_depth_for_SPM_i ('0 ), + // L1 bypass: [private_start_addr_i, L1BypassEnd) when enabled + .l1_bypass_en_i (l1d_bypass_i ), + .l1_bypass_start_i (private_start_addr_i ), + // Interco address rotation of this controller + .addr_rot_bits_i (refill_bits_to_rotate ), + .addr_rot_ofst_i (dynamic_offset_q ), // Request .core_req_valid_i (cache_req_valid[cb] ), .core_req_ready_o (cache_req_ready[cb] ), @@ -1102,7 +1122,8 @@ module cachepool_tile .core_resp_meta_o (cache_rsp_meta [cb] ), // TCDM Refill .refill_req_o (cache_refill_req [cb] ), - .refill_burst_o (cache_refill_burst [cb] ), + // Refills are single beats (RefillDataWidth == L1LineWidth): no burst info needed + .refill_burst_o ( ), .refill_req_valid_o (cache_refill_req_valid[cb] ), .refill_req_ready_i (cache_refill_req_ready[cb] ), .refill_rsp_i (cache_refill_rsp [cb] ), @@ -1125,19 +1146,19 @@ module cachepool_tile .tcdm_data_bank_gnt_i (l1_data_bank_gnt [cb] ) ); - // Inverse rotation for the refill address: the controller stores the address rotated by - // tcdm_cache_interco (routing bits at the MSB), so restore it before issuing to the NoC. - // N = refill_bits_to_rotate, per bank like gen_scramble; num_private_cache is runtime. - - logic [RefillRotWidth-1:0] refill_bits_to_rotate; - - // Mirrors gen_scramble in tcdm_cache_interco: banks below the boundary are - // private and rotate BankSel only, the rest also rotate TileID. cb is a - // genvar constant, so this covers all-private and all-shared as well. - assign refill_bits_to_rotate = (cb < int'(num_private_cache)) - ? (group_fold ? RefillRotWidth'(RefillCacheBankBits + RefillLocalTileBits) - : RefillRotWidth'(RefillCacheBankBits)) - : RefillRotWidth'(RefillCacheBankBits + RefillTileBits); + // The controller stores the rotated address, so restore it before issuing to the NoC. + logic [SpatzAxiAddrWidth-1:0] cache_refill_addr_unrot; + + cachepool_addr_unrotate #( + .AddrWidth (SpatzAxiAddrWidth ), + .RotWidth (RefillRotWidth ), + .OfstWidth ($bits(dynamic_offset_q) ) + ) i_refill_addr_unrotate ( + .addr_i (cache_refill_req[cb].addr), + .rot_bits_i (refill_bits_to_rotate ), + .ofst_i (dynamic_offset_q ), + .addr_o (cache_refill_addr_unrot ) + ); always_comb begin : bank_addr_scramble cache_refill_req_o[cb].q = '{ @@ -1151,9 +1172,11 @@ module cachepool_tile }; cache_refill_req_o[cb].q.user = '{ + bypass : cache_refill_req[cb].bypass, bank_id : cb, info : cache_refill_req[cb].info, - burst : cache_refill_burst[cb], + // Refills match responses by user, so any AXI ID will do + uniq_id : 1'b1, default : '0 }; cache_refill_req_o[cb].q_valid = cache_refill_req_valid[cb]; @@ -1163,30 +1186,14 @@ module cachepool_tile data : cache_refill_rsp_i[cb].p.data, write : cache_refill_rsp_i[cb].p.write, info : cache_refill_rsp_i[cb].p.user.info, + bypass: cache_refill_rsp_i[cb].p.user.bypass, default :'0 }; cache_refill_rsp_valid[cb] = cache_refill_rsp_i[cb].p_valid; cache_refill_req_ready[cb] = cache_refill_rsp_i[cb].q_ready; // Inverse rotation: recover original address from the rotated form. - begin - logic [SpatzAxiAddrWidth-1:0] addr_rot, lower, rot_field, upper; - addr_rot = cache_refill_req[cb].addr; - - lower = addr_rot & bitmask_lo; - - rot_field = addr_rot >> (SpatzAxiAddrWidth - refill_bits_to_rotate); - - upper = (addr_rot >> dynamic_offset_q) - & ((SpatzAxiAddrWidth'(1) << (SpatzAxiAddrWidth - - dynamic_offset_q - - refill_bits_to_rotate)) - 1); - - cache_refill_req_o[cb].q.addr = lower - | (rot_field << dynamic_offset_q) - | (upper << (dynamic_offset_q - + refill_bits_to_rotate)); - end + cache_refill_req_o[cb].q.addr = cache_refill_addr_unrot; end @@ -1766,6 +1773,9 @@ module cachepool_tile ((L1AssoPerCtrl % EffectiveFoldWayGroup) == 0)); `ASSERT_INIT(CheckLineSplit, (NumWordPerLine % PartSplit) == 0); // Check that the cluster base address aligns to the TCDMSize. + // While the L1 bypass is on, its range starts inside DRAM (only DRAM converters handle it) + `ASSERT(L1BypassStartInDram, l1d_bypass_i |-> + ((private_start_addr_i >= DramAddr) && (private_start_addr_i < DramAddr + DramSize))) `ASSERT(ClusterBaseAddrAlign, ((TCDMSize - 1) & cluster_base_addr_i) == 0) diff --git a/hardware/tb/tb_cachepool.sv b/hardware/tb/tb_cachepool.sv index 6b5e352a..2d5265ec 100644 --- a/hardware/tb/tb_cachepool.sv +++ b/hardware/tb/tb_cachepool.sv @@ -398,7 +398,7 @@ module tb_cachepool; .AxiAddrWidth ( SpatzAxiAddrWidth ), .AxiDataWidth ( SpatzAxiDataWidth ), .AxiIdWidth ( WrapperAxiIdOutWidth ), - .AxiUserWidth ( SpatzAxiUserWidth ), + .AxiUserWidth ( DramAxiUserWidth ), .axi_req_t ( spatz_axi_wrapper_out_req_t ), .axi_resp_t ( spatz_axi_wrapper_out_resp_t ), .axi_ar_t ( spatz_axi_wrapper_out_ar_chan_t ), diff --git a/software/snRuntime/include/cachepool_peripheral.h b/software/snRuntime/include/cachepool_peripheral.h index e4f31d74..4297d29b 100644 --- a/software/snRuntime/include/cachepool_peripheral.h +++ b/software/snRuntime/include/cachepool_peripheral.h @@ -125,8 +125,13 @@ extern "C" { // no #define CACHEPOOL_PERIPHERAL_L1D_ADDR_REG_OFFSET 0x48 +// L1D bypass: requests from L1D_ADDR up (below the stack) skip the L1 data +// cache. +#define CACHEPOOL_PERIPHERAL_L1D_BYPASS_REG_OFFSET 0x4c +#define CACHEPOOL_PERIPHERAL_L1D_BYPASS_EN_BIT 0 + // Cache xbar offset setting. Applied when the value changes, no commit -#define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_REG_OFFSET 0x4c +#define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_REG_OFFSET 0x50 #define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET_MASK 0x1f #define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET_OFFSET 0 #define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET_FIELD \ diff --git a/software/snRuntime/include/l1cache.h b/software/snRuntime/include/l1cache.h index 3b0a87f5..cd2ef5d0 100644 --- a/software/snRuntime/include/l1cache.h +++ b/software/snRuntime/include/l1cache.h @@ -32,4 +32,8 @@ void l1d_part (uint32_t size); void l1d_part_folded (uint32_t size, uint32_t fold); void l1d_addr (uint32_t addr); +// 1: the private range [L1D_ADDR, stack) bypasses the L1 data cache and goes +// straight to DRAM. Does not flush: flush first if that range may be cached. +void l1d_bypass (uint32_t en); + void set_eoc (uint32_t eoc_value); diff --git a/software/snRuntime/link/common.ld.in b/software/snRuntime/link/common.ld.in index 4a987e11..72445039 100644 --- a/software/snRuntime/link/common.ld.in +++ b/software/snRuntime/link/common.ld.in @@ -9,6 +9,8 @@ MEMORY { DRAM (rwxa) : ORIGIN = @MEM_DRAM_ORIGIN@, LENGTH = @MEM_DRAM_SIZE@ UNCACHED_REGION (rwxa) : ORIGIN = @UNCACHED_REGION_ORIGIN@, LENGTH = @UNCACHED_REGION_SIZE@ + /* L1 bypass range (cachepool_pkg L1BypassBase/End); own region, so its own load segment */ + L1_BYPASS (rwa) : ORIGIN = 0xB0000000, LENGTH = 0x0FFFF800 } SECTIONS @@ -145,4 +147,10 @@ SECTIONS KEEP(*(.pdcp_src)) _epdcp_src = .; /* optional: mark end of pdcp_src */ } > UNCACHED_REGION + + /* L1 bypass range [0xB000_0000, 0xBFFF_F800): bypasses the L1 data cache (cachepool_pkg) */ + .l1_bypass : + { + KEEP(*(.l1_bypass)) + } > L1_BYPASS } diff --git a/software/snRuntime/src/l1cache.c b/software/snRuntime/src/l1cache.c index 6d51cff3..eb50b08d 100644 --- a/software/snRuntime/src/l1cache.c +++ b/software/snRuntime/src/l1cache.c @@ -186,6 +186,22 @@ void l1d_addr (uint32_t addr) { snrt_cluster_hw_barrier(); } +// Enable/disable the L1 bypass of the private range [L1D_ADDR, stack). +// Applied as soon as the value changes, no commit needed. Does not flush. +void l1d_bypass (uint32_t en) { + asm volatile("fence" ::: "memory"); + snrt_cluster_hw_barrier(); + if (snrt_cluster_core_idx() == 0) { + volatile uint32_t *cfg_bypass = + (uint32_t *)(_snrt_team_current->root->cluster_mem.end + + CACHEPOOL_PERIPHERAL_L1D_BYPASS_REG_OFFSET); + *cfg_bypass = en & 0x1u; + // Wait until every group has applied the new setting + l1d_wait(); + } + snrt_cluster_hw_barrier(); +} + void set_eoc (uint32_t eoc_value) { volatile uint32_t *eoc_reg = (uint32_t *)(_snrt_team_current->root->cluster_mem.end + diff --git a/software/tests/CMakeLists.txt b/software/tests/CMakeLists.txt index 0c133516..59747f68 100644 --- a/software/tests/CMakeLists.txt +++ b/software/tests/CMakeLists.txt @@ -6,3 +6,4 @@ add_spatz_test_zeroParam(minimal-tile0-repro minimal-tile0-repro/main.c) add_spatz_test_zeroParam(load-store load-store/main.c) add_spatz_test_zeroParam(partial_barrier_benchmark partial_barrier_benchmark/main.c)# SOCMIPO add_spatz_test_zeroParam(bandwidth bandwidth/main.c) +add_spatz_test_zeroParam(l1-bypass l1-bypass/main.c) diff --git a/software/tests/l1-bypass/main.c b/software/tests/l1-bypass/main.c new file mode 100644 index 00000000..c41ba84d --- /dev/null +++ b/software/tests/l1-bypass/main.c @@ -0,0 +1,371 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Author: Diyou Shen + +// Directed test of the L1 bypass path. With L1D_BYPASS set, the private range +// [L1D_ADDR, stack) skips the L1 data cache and goes straight to DRAM; the test +// sets L1D_ADDR to 0xB000_0000. +// +// The buffer sits at the start of .l1_bypass and every word is preloaded with +// its own address, so a word returned from the wrong slot, part or line is +// caught by value. Each core works on its own slice: +// 1. scalar read of the preloaded pattern +// 2. vector unit-stride and strided read of the preloaded pattern +// 3. scalar word/halfword/byte writes, read-after-write +// 4. vector unit-stride and strided writes (strided: partial strobes per slot) +// 5. read of the neighbour core's slice (visibility through DRAM) +// 6. timing, two visits each: a cached region gets faster after its cold +// visit, the bypass region does not (DRAM refresh makes a third visit noisy) +// 7. bypass switched off: the same region now gets faster on its second visit +// All banks are tile-private, so every request is served by the local tile; +// private caches are not coherent across tiles, so part 5 needs the bypass. +// +// Strides of 12 B and unaligned e8 copies are left out: they hang on the cached +// path as well (pre-existing, independent of the bypass). + +#include +#include +#include +#include +#include + +#define BYP_BASE 0xB0000000u +// Cached region (.pdcp_src), below the private/bypass boundary +#define PRV_BASE 0xA0000000u +// Words per half: 16 KiB, so each half spans every DRAM channel (1 KiB interleave). +#define BYP_WORDS 4096u + +// Each word initialised to its own address (preprocessor expansion, no data file). +#define P1(b, i) ((b) + 4u * (i)) +#define P4(b, i) P1(b, i), P1(b, (i) + 1), P1(b, (i) + 2), P1(b, (i) + 3) +#define P16(b, i) P4(b, i), P4(b, (i) + 4), P4(b, (i) + 8), P4(b, (i) + 12) +#define P64(b, i) P16(b, i), P16(b, (i) + 16), P16(b, (i) + 32), P16(b, (i) + 48) +#define P256(b, i) P64(b, i), P64(b, (i) + 64), P64(b, (i) + 128), P64(b, (i) + 192) +#define P1024(b, i) P256(b, i), P256(b, (i) + 256), P256(b, (i) + 512), P256(b, (i) + 768) +#define P4096(b, i) P1024(b, i), P1024(b, (i) + 1024), P1024(b, (i) + 2048), P1024(b, (i) + 3072) +#define P16384(b, i) P4096(b, i), P4096(b, (i) + 4096), P4096(b, (i) + 8192), P4096(b, (i) + 12288) +#define A1(i) P1(BYP_BASE, i) + +// Part 6 timing regions: 16 lines per core, small enough to never evict. +#define T_CORE_WORDS 256u +#define T_WORDS (64u * T_CORE_WORDS) + +// Read-only half, read/write half, then the bypass timing region. +static uint32_t byp_buf[2 * BYP_WORDS + T_WORDS] __attribute__((section(".l1_bypass"), used)) = { + P4096(BYP_BASE, 0), P4096(BYP_BASE, BYP_WORDS), P16384(BYP_BASE, 2 * BYP_WORDS) +}; + +// Cached timing region (below the private/bypass boundary). +static uint32_t prv_buf[T_WORDS] __attribute__((section(".pdcp_src"), used)) = { + P16384(PRV_BASE, 0) +}; + +// Cached scratch, one slice per core. +#define MAX_CORES 64u +#define MAX_SLICE_WORDS (BYP_WORDS / 16u) +static uint32_t vbuf[MAX_CORES][MAX_SLICE_WORDS] __attribute__((section(".data"))); +static int error_cnt[MAX_CORES] __attribute__((section(".data"))); + +// ----------------------------------------------------------------------------- +// Helpers +// ----------------------------------------------------------------------------- + +static inline void vec_copy(uint32_t *dst, const uint32_t *src, uint32_t count) { + uint32_t avl = count; + uint32_t vl; + do { + asm volatile("vsetvli %0, %1, e32, m8, ta, ma" : "=r"(vl) : "r"(avl)); + asm volatile("vle32.v v0, (%0)" : : "r"(src) : "memory"); + asm volatile("vse32.v v0, (%0)" : : "r"(dst) : "memory"); + src += vl; + dst += vl; + avl -= vl; + } while (avl > 0); +} + +// dst[i] = src[i * stride_words], unit-stride store. +static inline void vec_gather_strided(uint32_t *dst, const uint32_t *src, + uint32_t stride_words, uint32_t count) { + uint32_t avl = count; + uint32_t vl; + const uint32_t stride_b = stride_words * sizeof(uint32_t); + do { + asm volatile("vsetvli %0, %1, e32, m8, ta, ma" : "=r"(vl) : "r"(avl)); + asm volatile("vlse32.v v0, (%0), %1" : : "r"(src), "r"(stride_b) : "memory"); + asm volatile("vse32.v v0, (%0)" : : "r"(dst) : "memory"); + src += vl * stride_words; + dst += vl; + avl -= vl; + } while (avl > 0); +} + +// dst[i * stride_words] = src[i], strided store. +static inline void vec_scatter_strided(uint32_t *dst, const uint32_t *src, + uint32_t stride_words, uint32_t count) { + uint32_t avl = count; + uint32_t vl; + const uint32_t stride_b = stride_words * sizeof(uint32_t); + do { + asm volatile("vsetvli %0, %1, e32, m8, ta, ma" : "=r"(vl) : "r"(avl)); + asm volatile("vle32.v v0, (%0)" : : "r"(src) : "memory"); + asm volatile("vsse32.v v0, (%0), %1" : : "r"(dst), "r"(stride_b) : "memory"); + src += vl; + dst += vl * stride_words; + avl -= vl; + } while (avl > 0); +} + +// Scalar stores and vector loads use different ports and are not ordered with +// each other: drain the scalar stores before Spatz reads the same buffer. +static inline void scalar_to_vector_fence(void) { asm volatile("fence" ::: "memory"); } + +// Stream count words through the vector unit (loads only). +static inline void vec_load(const uint32_t *src, uint32_t count) { + uint32_t avl = count; + uint32_t vl; + do { + asm volatile("vsetvli %0, %1, e32, m8, ta, ma" : "=r"(vl) : "r"(avl)); + asm volatile("vle32.v v0, (%0)" : : "r"(src) : "memory"); + src += vl; + avl -= vl; + } while (avl > 0); +} + +// One timed visit of every core's slice; cycles measured on core 0. +static uint32_t timed_visit(const uint32_t *slice_ptr, uint32_t count, uint32_t cid) { + snrt_cluster_host0_barrier(SNRT_HOST_BARRIER_SLOT); + uint32_t t0 = benchmark_get_cycle(); + vec_load(slice_ptr, count); + snrt_fence_spatz(); + snrt_cluster_host0_barrier(SNRT_HOST_BARRIER_SLOT); + return (cid == 0) ? benchmark_get_cycle() - t0 : 0; +} + +static inline uint32_t pat_a(uint32_t cid, uint32_t i) { return 0xA0000000u | (cid << 16) | i; } +static inline uint32_t pat_b(uint32_t cid, uint32_t i) { return 0x5B000000u | (cid << 16) | i; } +static inline uint32_t pat_c(uint32_t cid, uint32_t i) { return 0xC3000000u | (cid << 16) | i; } + +static int report(const char *name, uint32_t cid, uint32_t num_cores) { + snrt_cluster_host0_barrier(SNRT_HOST_BARRIER_SLOT); + int fails = 0; + if (cid == 0) { + int total = 0; + for (uint32_t c = 0; c < num_cores; c++) { + total += error_cnt[c]; + } + printf("%s: %s (%d errors)\n", name, total ? "FAIL" : "PASS", total); + fails = (total != 0); + } + snrt_cluster_host0_barrier(SNRT_HOST_BARRIER_SLOT); + return fails; +} + +// ----------------------------------------------------------------------------- +// main +// ----------------------------------------------------------------------------- +int main() { + // One hart per Spatz runs the test (dual-scalar CCs share a Spatz) + const uint32_t num_cores = snrt_cluster_vpu_num(); + const uint32_t cid = snrt_cluster_vpu_idx(); + const int is_primary = snrt_cluster_is_primary(); + int num_fails = 0; + + // All banks tile-private, private range from BYP_BASE up, bypass on. Every hart + // calls the l1d_* helpers (they use the cluster barrier). l1d_part() goes first: + // a commit re-issues the last L1D instruction, which l1d_part() leaves at flush. + l1d_part(snrt_cluster_vpu_per_tile()); + l1d_addr(BYP_BASE); + l1d_bypass(1); + + if (!is_primary) { + // Matches the primaries' l1d_bypass(0) before part 7 + l1d_bypass(0); + snrt_cluster_hw_barrier(); + return 0; + } + spatz_lock_acquire(); + + uint32_t slice = BYP_WORDS / num_cores; + if (slice > MAX_SLICE_WORDS) { + slice = MAX_SLICE_WORDS; + } + + if (cid == 0) { + printf("*** L1 bypass test ***\n"); + printf("Cores:%u buffer:0x%x slice:%u words\n", num_cores, (uint32_t)byp_buf, slice); + if ((uint32_t)byp_buf != BYP_BASE || (uint32_t)prv_buf != PRV_BASE || num_cores > MAX_CORES) { + printf("FATAL: buffers not at 0x%x / 0x%x or too many cores\n", BYP_BASE, PRV_BASE); + num_fails++; + } + } + snrt_cluster_host0_barrier(SNRT_HOST_BARRIER_SLOT); + + const uint32_t ro_base = cid * slice; + const uint32_t rw_base = BYP_WORDS + cid * slice; + volatile uint32_t *ro = &byp_buf[ro_base]; + volatile uint32_t *rw = &byp_buf[rw_base]; + uint32_t *vb = vbuf[cid]; + int err; + + // 1. Scalar read of the preloaded pattern + err = 0; + for (uint32_t i = 0; i < slice; i++) { + err += (ro[i] != A1(ro_base + i)); + } + error_cnt[cid] = err; + num_fails += report("1 scalar read", cid, num_cores); + + // 2a. Vector unit-stride read + vec_copy(vb, (const uint32_t *)ro, slice); + snrt_fence_spatz(); + err = 0; + for (uint32_t i = 0; i < slice; i++) { + err += (vb[i] != A1(ro_base + i)); + } + error_cnt[cid] = err; + num_fails += report("2a vector read", cid, num_cores); + + // 2b. Vector strided read (every second word, crosses parts and lines) + const uint32_t n_str = slice / 2; + vec_gather_strided(vb, (const uint32_t *)ro, 2, n_str); + snrt_fence_spatz(); + err = 0; + for (uint32_t i = 0; i < n_str; i++) { + err += (vb[i] != A1(ro_base + 2 * i)); + } + error_cnt[cid] = err; + num_fails += report("2b vector strided read", cid, num_cores); + + // 3. Scalar writes: word, then halfword and byte overwrites, each read back + // right away (read-after-write on the same address). + err = 0; + for (uint32_t i = 0; i < slice; i++) { + uint32_t exp = pat_a(cid, i); + rw[i] = exp; + err += (rw[i] != exp); + if (i % 2 == 0) { + volatile uint16_t *h = (volatile uint16_t *)&rw[i] + 1; + *h = (uint16_t)(0xBEEF ^ i); + exp = (exp & 0x0000FFFFu) | ((uint32_t)(uint16_t)(0xBEEF ^ i) << 16); + err += (rw[i] != exp); + } + if (i % 3 == 0) { + volatile uint8_t *b = (volatile uint8_t *)&rw[i] + 1; + *b = (uint8_t)(0x5A ^ i); + exp = (exp & 0xFFFF00FFu) | ((uint32_t)(uint8_t)(0x5A ^ i) << 8); + err += (rw[i] != exp); + } + } + error_cnt[cid] = err; + num_fails += report("3 scalar write/RAW", cid, num_cores); + + // 4a. Vector unit-stride write of pattern A, scalar read back + for (uint32_t i = 0; i < slice; i++) { + vb[i] = pat_a(cid, i); + } + scalar_to_vector_fence(); + vec_copy((uint32_t *)rw, vb, slice); + snrt_fence_spatz(); + err = 0; + for (uint32_t i = 0; i < slice; i++) { + err += (rw[i] != pat_a(cid, i)); + } + error_cnt[cid] = err; + num_fails += report("4a vector write", cid, num_cores); + + // 4b. Vector strided write of pattern B into every second word + const uint32_t n_half = slice / 2; + for (uint32_t i = 0; i < n_half; i++) { + vb[i] = pat_b(cid, i); + } + scalar_to_vector_fence(); + vec_scatter_strided((uint32_t *)rw, vb, 2, n_half); + snrt_fence_spatz(); + err = 0; + for (uint32_t i = 0; i < slice; i++) { + uint32_t exp = (i % 2 == 0 && i / 2 < n_half) ? pat_b(cid, i / 2) : pat_a(cid, i); + err += (rw[i] != exp); + } + error_cnt[cid] = err; + num_fails += report("4b vector strided write", cid, num_cores); + + // 5. Every core writes pattern C, then checks its neighbour's slice + for (uint32_t i = 0; i < slice; i++) { + vb[i] = pat_c(cid, i); + } + scalar_to_vector_fence(); + vec_copy((uint32_t *)rw, vb, slice); + snrt_fence_spatz(); + snrt_cluster_host0_barrier(SNRT_HOST_BARRIER_SLOT); + const uint32_t nb = (cid + 1) % num_cores; + volatile uint32_t *nrw = &byp_buf[BYP_WORDS + nb * slice]; + err = 0; + for (uint32_t i = 0; i < slice; i++) { + err += (nrw[i] != pat_c(nb, i)); + } + error_cnt[cid] = err; + num_fails += report("5 neighbour read", cid, num_cores); + + // 6. Timing: two visits of a cached region (B, cold first) and of the bypass + // region (A). B must speed up after the cold visit, A must not. + const uint32_t t_base = cid * T_CORE_WORDS; + const uint32_t *b_ptr = &prv_buf[t_base]; + const uint32_t *a_ptr = &byp_buf[2 * BYP_WORDS + t_base]; + uint32_t tb[2], ta[2]; + for (uint32_t t = 0; t < 2; t++) tb[t] = timed_visit(b_ptr, T_CORE_WORDS, cid); + for (uint32_t t = 0; t < 2; t++) ta[t] = timed_visit(a_ptr, T_CORE_WORDS, cid); + err = 0; + for (uint32_t i = 0; i < T_CORE_WORDS; i++) { + err += (b_ptr[i] != P1(PRV_BASE, t_base + i)); + } + error_cnt[cid] = err; + num_fails += report("6 cached region data", cid, num_cores); + if (cid == 0) { + printf("6 cycles cached B: %u %u | bypass A: %u %u\n", tb[0], tb[1], ta[0], ta[1]); + // cached: second visit < 0.7x cold; bypass: second visit within +-20% of the first + const int b_ok = (10 * tb[1] < 7 * tb[0]); + const int a_ok = (10 * ta[1] > 8 * ta[0]) && (10 * ta[1] < 12 * ta[0]); + printf("6 timing: cached speeds up %s, bypass stays flat %s\n", + b_ok ? "PASS" : "FAIL", a_ok ? "PASS" : "FAIL"); + num_fails += !(b_ok && a_ok); + } + + // 7. Bypass off: the same bypass-range region is cacheable again (cold, then hit). + // It was never cached, so no flush is needed. + l1d_bypass(0); + uint32_t tc[2]; + for (uint32_t t = 0; t < 2; t++) tc[t] = timed_visit(a_ptr, T_CORE_WORDS, cid); + err = 0; + for (uint32_t i = 0; i < T_CORE_WORDS; i++) { + err += (a_ptr[i] != P1(BYP_BASE, 2 * BYP_WORDS + t_base + i)); + } + error_cnt[cid] = err; + num_fails += report("7 bypass off, data", cid, num_cores); + if (cid == 0) { + const int c_ok = (10 * tc[1] < 7 * tc[0]); + printf("7 cycles same region, bypass off: %u %u -> %s\n", tc[0], tc[1], + c_ok ? "PASS" : "FAIL"); + num_fails += !c_ok; + } + + if (cid == 0) { + printf("*** L1 bypass test %s ***\n", num_fails ? "FAILED" : "PASSED"); + } + spatz_lock_release(); + snrt_cluster_hw_barrier(); + return num_fails ? -1 : 0; +} From 4c4f9ac1e957f00b097281af934e8b1c9ffc0359 Mon Sep 17 00:00:00 2001 From: Diyou Shen Date: Thu, 22 Oct 2026 09:43:10 +0200 Subject: [PATCH 3/5] [SRC] Add some probes and assertion checks for bypass logic --- .gitlab-ci.yml | 2 ++ hardware/src/cachepool_tile.sv | 8 +++++++ hardware/tb/cachepool_monitor.sv | 40 ++++++++++++++++++++++++++++---- 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e54f77b5..7a7132af 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -173,6 +173,7 @@ test: - rlc-multi_producer_single_consumer_double_linked_list_M1_N1350_K10 - rlc-multi_producer_single_consumer_double_linked_list_M1_N1350_K4_P2_C2_am_G1024_tbchk - rlc-multi_producer_single_consumer_double_linked_list_P1_C3_ul_ci + - l1-bypass script: # The vsim script writes a .rtlbinary marker here; monitor logs need the # core/noc/others subfolders. Build artifacts exclude sim/bin/logs/, so @@ -207,6 +208,7 @@ test-dual: - gemv_M512_N128_K32 - fmatmul-32b_M64_N64_K64 - fft-32b_M1024_N16 + - l1-bypass script: - mkdir -p sim/bin/logs sim/bin/logs/core sim/bin/logs/noc sim/bin/logs/others - chmod +x sim/bin/cachepool_cluster.vsim diff --git a/hardware/src/cachepool_tile.sv b/hardware/src/cachepool_tile.sv index eeff5b1a..5f420f10 100644 --- a/hardware/src/cachepool_tile.sv +++ b/hardware/src/cachepool_tile.sv @@ -1196,6 +1196,14 @@ module cachepool_tile cache_refill_req_o[cb].q.addr = cache_refill_addr_unrot; end + // Guard: while the L1 bypass is on, the cache never refills a line of the bypass range + // (writebacks may: lines dirty from before the bypass was switched on, no flush in hw). + `ASSERT(L1BypassNoCacheRefill, + cache_refill_req_valid[cb] && !cache_refill_req[cb].write && + !cache_refill_req[cb].bypass && L1BypassEn && l1d_bypass_i |-> + !((cache_refill_addr_unrot >= private_start_addr_i) && + (cache_refill_addr_unrot < L1BypassEnd))) + for (genvar j = 0; j < NumTagBankPerCtrl; j++) begin tc_sram_impl #( diff --git a/hardware/tb/cachepool_monitor.sv b/hardware/tb/cachepool_monitor.sv index 9e504f4e..3a916175 100644 --- a/hardware/tb/cachepool_monitor.sv +++ b/hardware/tb/cachepool_monitor.sv @@ -209,6 +209,8 @@ module cachepool_monitor // a request waited for q_ready, read responses received, and the time // integral of reads in flight. avg round trip = in-flight integral / reads, // i.e. the refill latency seen at the controller boundary (Little's law). + // L1 bypass transactions (user.bypass) share the port; they are counted + // separately (reads and writes, both get a response) and excluded above. // --------------------------------------------------------------------- for (genvar gy = 0; gy < NumGroupsY; gy++) begin : gen_refill_gy for (genvar gx = 0; gx < NumGroupsX; gx++) begin : gen_refill_gx @@ -226,16 +228,37 @@ module cachepool_monitor `FF(inflight_q, inflight_d, '0) `FF(area_q, area_d, '0) + // L1 bypass transactions on the same port + cnt_t [NumL1CtrlTile-1:0] byp_rd_d, byp_rd_q; + cnt_t [NumL1CtrlTile-1:0] byp_wr_d, byp_wr_q; + cnt_t [NumL1CtrlTile-1:0] byp_rsp_d, byp_rsp_q; + cnt_t [NumL1CtrlTile-1:0] byp_inflight_d, byp_inflight_q; + logic [NumL1CtrlTile-1:0][63:0] byp_area_d, byp_area_q; + `FF(byp_rd_q, byp_rd_d, '0) + `FF(byp_wr_q, byp_wr_d, '0) + `FF(byp_rsp_q, byp_rsp_d, '0) + `FF(byp_inflight_q, byp_inflight_d, '0) + `FF(byp_area_q, byp_area_d, '0) + for (genvar cb = 0; cb < NumL1CtrlTile; cb++) begin : gen_refill_ctrl + logic req_any, req_byp, rsp_any, rsp_byp; logic req_hs, req_wait, rsp_hs; - assign req_hs = `TILE_PATH(gy, gx, t).cache_refill_req_o[cb].q_valid - & `TILE_PATH(gy, gx, t).cache_refill_rsp_i[cb].q_ready + logic byp_rd_hs, byp_wr_hs, byp_rsp_hs; + assign req_any = `TILE_PATH(gy, gx, t).cache_refill_req_o[cb].q_valid + & `TILE_PATH(gy, gx, t).cache_refill_rsp_i[cb].q_ready; + assign req_byp = `TILE_PATH(gy, gx, t).cache_refill_req_o[cb].q.user.bypass; + assign rsp_any = `TILE_PATH(gy, gx, t).cache_refill_rsp_i[cb].p_valid + & `TILE_PATH(gy, gx, t).cache_refill_req_o[cb].p_ready; + assign rsp_byp = `TILE_PATH(gy, gx, t).cache_refill_rsp_i[cb].p.user.bypass; + assign req_hs = req_any & ~req_byp & ~`TILE_PATH(gy, gx, t).cache_refill_req_o[cb].q.write; assign req_wait = `TILE_PATH(gy, gx, t).cache_refill_req_o[cb].q_valid & ~`TILE_PATH(gy, gx, t).cache_refill_rsp_i[cb].q_ready; - assign rsp_hs = `TILE_PATH(gy, gx, t).cache_refill_rsp_i[cb].p_valid - & `TILE_PATH(gy, gx, t).cache_refill_req_o[cb].p_ready + assign rsp_hs = rsp_any & ~rsp_byp & ~`TILE_PATH(gy, gx, t).cache_refill_rsp_i[cb].p.write; + assign byp_rd_hs = req_any & req_byp & ~`TILE_PATH(gy, gx, t).cache_refill_req_o[cb].q.write; + assign byp_wr_hs = req_any & req_byp & `TILE_PATH(gy, gx, t).cache_refill_req_o[cb].q.write; + assign byp_rsp_hs = rsp_any & rsp_byp; always_comb begin // In-flight count spans sessions; only the per-session totals reset @@ -244,6 +267,11 @@ module cachepool_monitor req_stall_d[cb] = session_edge ? '0 : req_stall_q[cb] + cnt_t'(req_wait); rd_rsp_d[cb] = session_edge ? '0 : rd_rsp_q[cb] + cnt_t'(rsp_hs); area_d[cb] = session_edge ? '0 : area_q[cb] + 64'(inflight_q[cb]); + byp_inflight_d[cb] = byp_inflight_q[cb] + cnt_t'(byp_rd_hs | byp_wr_hs) - cnt_t'(byp_rsp_hs); + byp_rd_d[cb] = session_edge ? '0 : byp_rd_q[cb] + cnt_t'(byp_rd_hs); + byp_wr_d[cb] = session_edge ? '0 : byp_wr_q[cb] + cnt_t'(byp_wr_hs); + byp_rsp_d[cb] = session_edge ? '0 : byp_rsp_q[cb] + cnt_t'(byp_rsp_hs); + byp_area_d[cb] = session_edge ? '0 : byp_area_q[cb] + 64'(byp_inflight_q[cb]); end end @@ -263,8 +291,12 @@ module cachepool_monitor automatic real span = cycle_q - session_start_cycle_q; automatic real rt = rd_rsp_q[cb] == 0 ? 0 : real'(area_q[cb]) / real'(rd_rsp_q[cb]); automatic real avg = span == 0 ? 0 : real'(area_q[cb]) / span; + automatic real brt = byp_rsp_q[cb] == 0 ? 0 : real'(byp_area_q[cb]) / real'(byp_rsp_q[cb]); + automatic real bavg = span == 0 ? 0 : real'(byp_area_q[cb]) / span; $fwrite(refill_f, " Ctrl %0d: reads=%0d rsp=%0d req stall cyc=%0d avg in flight=%0.2f avg round trip=%0.2f\n", cb, rd_req_q[cb], rd_rsp_q[cb], req_stall_q[cb], avg, rt); + $fwrite(refill_f, " Ctrl %0d bypass: reads=%0d writes=%0d rsp=%0d avg in flight=%0.2f avg round trip=%0.2f\n", + cb, byp_rd_q[cb], byp_wr_q[cb], byp_rsp_q[cb], bavg, brt); end $fwrite(refill_f, "\n"); endfunction From 598ddb87e59c9daf8038689708512cda1680d64a Mon Sep 17 00:00:00 2001 From: Diyou Shen Date: Wed, 28 Oct 2026 14:15:45 +0100 Subject: [PATCH 4/5] [SRC] Small fixes to bypass path after rebasing --- hardware/src/cachepool_pkg.sv | 10 ++++--- hardware/src/cachepool_reqrsp_to_axi.sv | 22 +++++++++++--- hardware/src/cachepool_tile.sv | 2 +- software/tests/l1-bypass/main.c | 39 ++++++++++++++++++++++--- 4 files changed, 60 insertions(+), 13 deletions(-) diff --git a/hardware/src/cachepool_pkg.sv b/hardware/src/cachepool_pkg.sv index a910e118..a826b0d0 100644 --- a/hardware/src/cachepool_pkg.sv +++ b/hardware/src/cachepool_pkg.sv @@ -340,10 +340,9 @@ package cachepool_pkg; localparam int unsigned DramSize = 32'h4000_0000; localparam int unsigned DramPerChSize = DramSize / NumL2Channel; - // L1 bypass (built if L1BypassEn; enabled at runtime by L1D_BYPASS): the private range - // [L1D_ADDR, L1BypassEnd) goes around the L1 cache straight to DRAM. The fixed end keeps a - // stack overflowing the stack SPM cached. - localparam bit L1BypassEn = 1'b1; + // L1 bypass (enabled at runtime by L1D_BYPASS): the private range [L1D_ADDR, L1BypassEnd) + // goes around the L1 cache straight to DRAM. The fixed end keeps a stack overflowing the + // stack SPM cached. Built only where it fits a refill beat (L1BypassEn, below). localparam int unsigned L1BypassEnd = TCDMStartAddr; // Currently set to 16 for now @@ -498,6 +497,9 @@ package cachepool_pkg; localparam int unsigned L1BypassInfoWidth = L1PartIdxWidth + 2 + L1CoalPorts * (1 + $clog2(L1CoalDataWidth / $bits(narrow_data_t)) + $bits(tcdm_user_t)); + // The bypass hardware exists only if data and xbar info fit one refill beat (folded banks); + // otherwise L1D_BYPASS has no effect and the range stays cached. + localparam bit L1BypassEn = (L1CoalDataWidth + L1BypassInfoWidth <= RefillDataWidth); typedef struct packed { // L1 bypass transaction: refill data carries the cache controller xbar payload diff --git a/hardware/src/cachepool_reqrsp_to_axi.sv b/hardware/src/cachepool_reqrsp_to_axi.sv index 3c3c33e0..bcf3f7e1 100644 --- a/hardware/src/cachepool_reqrsp_to_axi.sv +++ b/hardware/src/cachepool_reqrsp_to_axi.sv @@ -46,6 +46,9 @@ module cachepool_reqrsp_to_axi #( localparam int unsigned CounterWidth = cf_math_pkg::idx_width(MaxTrans); localparam int unsigned BypassStrbWidth = BypassDataWidth / 8; localparam bit BypassSupported = (BypassDataWidth + BypassInfoWidth <= DataWidth); + // Info position in the data word; folded to 0 where unsupported so the slices stay legal + localparam int unsigned BypassInfoLsb = BypassSupported ? BypassDataWidth : 0; + localparam int unsigned BeatOfstWidth = $clog2(DataWidth/8); typedef logic [CounterWidth-1:0] cnt_t; typedef logic [AxiUserWidth-1:0] axi_user_t; @@ -106,6 +109,7 @@ module cachepool_reqrsp_to_axi #( logic q_bypass; logic [BypassPartWidth-1:0] q_part; logic [AxiIdWidth-1:0] q_id; + logic [$bits(reqrsp_req_i.q.addr)-1:0] q_addr; axi_user_t q_axi_user; data_t q_wdata; strb_t q_wstrb; @@ -113,6 +117,11 @@ module cachepool_reqrsp_to_axi #( assign q_bypass = reqrsp_req_i.q.user.bypass; assign q_part = reqrsp_req_i.q.addr[$clog2(BypassStrbWidth) +: BypassPartWidth]; assign q_id = reqrsp_req_i.q.user.uniq_id ? id_cnt_q : AxiIdWidth'($unsigned(ID)); + // A bypass address points into the beat (word offset); AXI wants it beat aligned, the + // part index above keeps the original offset for data placement and routing. + assign q_addr = (BypassSupported && q_bypass) + ? (reqrsp_req_i.q.addr & ({$bits(q_addr){1'b1}} << BeatOfstWidth)) + : reqrsp_req_i.q.addr; always_comb begin q_axi_user = '0; @@ -122,7 +131,7 @@ module cachepool_reqrsp_to_axi #( if (BypassSupported && q_bypass) begin q_axi_user[BaseUserWidth +: BypassPartWidth] = q_part; q_axi_user[BaseUserWidth + BypassPartWidth +: BypassInfoWidth] = - reqrsp_req_i.q.data[BypassDataWidth +: BypassInfoWidth]; + reqrsp_req_i.q.data[BypassInfoLsb +: BypassInfoWidth]; q_wdata = '0; q_wstrb = '0; q_wdata[q_part * BypassDataWidth +: BypassDataWidth] = @@ -136,7 +145,7 @@ module cachepool_reqrsp_to_axi #( // Read Channel // ------------- assign axi_req_o.ar.id = q_id; - assign axi_req_o.ar.addr = reqrsp_req_i.q.addr; + assign axi_req_o.ar.addr = q_addr; assign axi_req_o.ar.len = '0; assign axi_req_o.ar.size = {1'b0, reqrsp_req_i.q.size}; assign axi_req_o.ar.burst = axi_pkg::BURST_INCR; @@ -153,7 +162,7 @@ module cachepool_reqrsp_to_axi #( // Write Channel // ------------- assign axi_req_o.aw.id = q_id; - assign axi_req_o.aw.addr = reqrsp_req_i.q.addr; + assign axi_req_o.aw.addr = q_addr; assign axi_req_o.aw.len = '0; assign axi_req_o.aw.size = {1'b0, reqrsp_req_i.q.size}; assign axi_req_o.aw.burst = axi_pkg::BURST_INCR; @@ -212,7 +221,7 @@ module cachepool_reqrsp_to_axi #( reqrsp_rsp_o.p.data[BypassDataWidth-1:0] = axi_rsp_i.r.data[p_part * BypassDataWidth +: BypassDataWidth]; end - reqrsp_rsp_o.p.data[BypassDataWidth +: BypassInfoWidth] = + reqrsp_rsp_o.p.data[BypassInfoLsb +: BypassInfoWidth] = p_axi_user[BaseUserWidth + BypassPartWidth +: BypassInfoWidth]; end end @@ -230,6 +239,11 @@ module cachepool_reqrsp_to_axi #( `ASSERT(WriteStable, q_valid_write && !q_ready_write |=> q_valid_write) `ASSERT(NoReadWriteRspOverlap, !(r_valid && b_valid)) `ASSERT(SingleBeatRead, r_valid |-> axi_rsp_i.r.last) + // AXI: the start address of a transfer is aligned to its size (reported, not fatal) + `ASSERT(ArAddrAligned, axi_req_o.ar_valid |-> + ((axi_req_o.ar.addr & ((1 << axi_req_o.ar.size) - 1)) == '0)) + `ASSERT(AwAddrAligned, axi_req_o.aw_valid |-> + ((axi_req_o.aw.addr & ((1 << axi_req_o.aw.size) - 1)) == '0)) `ASSERT(NoUnsupportedBypass, reqrsp_req_i.q_valid |-> (BypassSupported || !reqrsp_req_i.q.user.bypass)) diff --git a/hardware/src/cachepool_tile.sv b/hardware/src/cachepool_tile.sv index 5f420f10..009c1b46 100644 --- a/hardware/src/cachepool_tile.sv +++ b/hardware/src/cachepool_tile.sv @@ -1782,7 +1782,7 @@ module cachepool_tile `ASSERT_INIT(CheckLineSplit, (NumWordPerLine % PartSplit) == 0); // Check that the cluster base address aligns to the TCDMSize. // While the L1 bypass is on, its range starts inside DRAM (only DRAM converters handle it) - `ASSERT(L1BypassStartInDram, l1d_bypass_i |-> + `ASSERT(L1BypassStartInDram, L1BypassEn && l1d_bypass_i |-> ((private_start_addr_i >= DramAddr) && (private_start_addr_i < DramAddr + DramSize))) `ASSERT(ClusterBaseAddrAlign, ((TCDMSize - 1) & cluster_base_addr_i) == 0) diff --git a/software/tests/l1-bypass/main.c b/software/tests/l1-bypass/main.c index c41ba84d..9c0f3baf 100644 --- a/software/tests/l1-bypass/main.c +++ b/software/tests/l1-bypass/main.c @@ -23,6 +23,7 @@ // The buffer sits at the start of .l1_bypass and every word is preloaded with // its own address, so a word returned from the wrong slot, part or line is // caught by value. Each core works on its own slice: +// 0. switching the bypass keeps dirty cached data (no implicit invalidate) // 1. scalar read of the preloaded pattern // 2. vector unit-stride and strided read of the preloaded pattern // 3. scalar word/halfword/byte writes, read-after-write @@ -80,6 +81,16 @@ static uint32_t prv_buf[T_WORDS] __attribute__((section(".pdcp_src"), used)) = { static uint32_t vbuf[MAX_CORES][MAX_SLICE_WORDS] __attribute__((section(".data"))); static int error_cnt[MAX_CORES] __attribute__((section(".data"))); +// Part 0: a cached value that is dirty when the bypass is switched +#define CANARY_INIT 0xDEADBEEFu +#define CANARY_NEW 0x1234ABCDu +static volatile uint32_t canary __attribute__((section(".data"))) = CANARY_INIT; + +// Self-check of the reporting: 1 makes the last core report one error in part 1 +#ifndef LBYP_INJECT_ERR +#define LBYP_INJECT_ERR 0 +#endif + // ----------------------------------------------------------------------------- // Helpers // ----------------------------------------------------------------------------- @@ -184,9 +195,26 @@ int main() { const int is_primary = snrt_cluster_is_primary(); int num_fails = 0; + // Buffers are sized for MAX_CORES: reject larger clusters on every hart, before + // any collective call, so nothing runs out of bounds or waits in a barrier. + if (num_cores > MAX_CORES) { + if (snrt_cluster_core_idx() == 0) { + printf("FATAL: l1-bypass supports at most %u VPUs (got %u)\n", MAX_CORES, num_cores); + } + snrt_cluster_hw_barrier(); + return -1; + } + + // 0. Dirty cached data survives switching the bypass on and off + if (snrt_cluster_core_idx() == 0) { + canary = CANARY_NEW; + } + l1d_bypass(1); + l1d_bypass(0); + const int canary_ok = (canary == CANARY_NEW); + // All banks tile-private, private range from BYP_BASE up, bypass on. Every hart - // calls the l1d_* helpers (they use the cluster barrier). l1d_part() goes first: - // a commit re-issues the last L1D instruction, which l1d_part() leaves at flush. + // calls the l1d_* helpers (they use the cluster barrier). l1d_part(snrt_cluster_vpu_per_tile()); l1d_addr(BYP_BASE); l1d_bypass(1); @@ -206,9 +234,11 @@ int main() { if (cid == 0) { printf("*** L1 bypass test ***\n"); + printf("0 bypass switch keeps dirty data: %s\n", canary_ok ? "PASS" : "FAIL"); + num_fails += !canary_ok; printf("Cores:%u buffer:0x%x slice:%u words\n", num_cores, (uint32_t)byp_buf, slice); - if ((uint32_t)byp_buf != BYP_BASE || (uint32_t)prv_buf != PRV_BASE || num_cores > MAX_CORES) { - printf("FATAL: buffers not at 0x%x / 0x%x or too many cores\n", BYP_BASE, PRV_BASE); + if ((uint32_t)byp_buf != BYP_BASE || (uint32_t)prv_buf != PRV_BASE) { + printf("FATAL: buffers not at 0x%x / 0x%x\n", BYP_BASE, PRV_BASE); num_fails++; } } @@ -226,6 +256,7 @@ int main() { for (uint32_t i = 0; i < slice; i++) { err += (ro[i] != A1(ro_base + i)); } + err += (LBYP_INJECT_ERR && cid == num_cores - 1); error_cnt[cid] = err; num_fails += report("1 scalar read", cid, num_cores); From f51143a0917ff3f789a2c00cffab10b8f9ec9857 Mon Sep 17 00:00:00 2001 From: Diyou Shen Date: Thu, 29 Oct 2026 21:44:44 +0100 Subject: [PATCH 5/5] [Deps] Fix a bug in insitu-cache triggered by the bypass data path --- Bender.lock | 2 +- software/snRuntime/link/common.ld.in | 5 +++-- software/tests/l1-bypass/main.c | 11 ++++++----- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Bender.lock b/Bender.lock index 8a6045c1..ec62e7af 100644 --- a/Bender.lock +++ b/Bender.lock @@ -109,7 +109,7 @@ packages: - obi - register_interface insitu-cache: - revision: 9a4fe83884fd94baf87bba6ea4e8ad75d4f4893f + revision: a407d39c832aa514551d1d153d93b60bb5d3a6e5 version: null source: Git: https://github.com/pulp-platform/Insitu-Cache.git diff --git a/software/snRuntime/link/common.ld.in b/software/snRuntime/link/common.ld.in index 72445039..6038d34c 100644 --- a/software/snRuntime/link/common.ld.in +++ b/software/snRuntime/link/common.ld.in @@ -9,7 +9,7 @@ MEMORY { DRAM (rwxa) : ORIGIN = @MEM_DRAM_ORIGIN@, LENGTH = @MEM_DRAM_SIZE@ UNCACHED_REGION (rwxa) : ORIGIN = @UNCACHED_REGION_ORIGIN@, LENGTH = @UNCACHED_REGION_SIZE@ - /* L1 bypass range (cachepool_pkg L1BypassBase/End); own region, so its own load segment */ + /* Data for the L1 bypass range, up to L1BypassEnd (cachepool_pkg); own region, so its own load segment */ L1_BYPASS (rwa) : ORIGIN = 0xB0000000, LENGTH = 0x0FFFF800 } @@ -148,7 +148,8 @@ SECTIONS _epdcp_src = .; /* optional: mark end of pdcp_src */ } > UNCACHED_REGION - /* L1 bypass range [0xB000_0000, 0xBFFF_F800): bypasses the L1 data cache (cachepool_pkg) */ + /* Preloaded data in [0xB000_0000, 0xBFFF_F800). It bypasses the L1 data cache only when + L1D_BYPASS is set and L1D_ADDR is at or below it (bypass range [L1D_ADDR, L1BypassEnd)) */ .l1_bypass : { KEEP(*(.l1_bypass)) diff --git a/software/tests/l1-bypass/main.c b/software/tests/l1-bypass/main.c index 9c0f3baf..7bbe4805 100644 --- a/software/tests/l1-bypass/main.c +++ b/software/tests/l1-bypass/main.c @@ -32,8 +32,9 @@ // 6. timing, two visits each: a cached region gets faster after its cold // visit, the bypass region does not (DRAM refresh makes a third visit noisy) // 7. bypass switched off: the same region now gets faster on its second visit -// All banks are tile-private, so every request is served by the local tile; -// private caches are not coherent across tiles, so part 5 needs the bypass. +// Half of each tile's banks are private: the private/bypass range is served by +// the local tile, everything below L1D_ADDR (counters, scratch, the cached +// timing region) is shared and coherent across tiles. // // Strides of 12 B and unaligned e8 copies are left out: they hang on the cached // path as well (pre-existing, independent of the bypass). @@ -213,9 +214,9 @@ int main() { l1d_bypass(0); const int canary_ok = (canary == CANARY_NEW); - // All banks tile-private, private range from BYP_BASE up, bypass on. Every hart - // calls the l1d_* helpers (they use the cluster barrier). - l1d_part(snrt_cluster_vpu_per_tile()); + // Half the banks tile-private, private range from BYP_BASE up, bypass on. Every + // hart calls the l1d_* helpers (they use the cluster barrier). + l1d_part(snrt_cluster_vpu_per_tile() / 2); l1d_addr(BYP_BASE); l1d_bypass(1);