[Placement] Resolved Inconsistency with Placing at Root Tile Locs

Found an inconsistency with how the placement of cluster blocks into
large tiles was handled in VPR. During placement, clusters are only
placed at the rool tile location of the grid; however, specific parts of
the later stages require that clusters are placed at all tile locations
of large tiles. This is inconsistent and makes it confusing to know
where clusters should be placed.

Enforced that clusters must be placed at the root tile location and
never placed in other tile locations in the large tiles. This required
fixing an oversite in the flat router which was relying on the fact that
clusters will not be placed at the root tile location.

Author: AlexandreSinger

vpr/src/base/vpr_api.cpp

@@ -16,6 +16,7 @@
 #include <cmath>
 
 #include "cluster_util.h"
+#include "verify_placement.h"
 #include "vpr_context.h"
 #include "vtr_assert.h"
 #include "vtr_math.h"
@@ -801,10 +802,6 @@ bool vpr_place_flow(const Netlist<>& net_list, t_vpr_setup& vpr_setup, const t_a
             vpr_load_placement(vpr_setup, arch);
         }
 
-        // FIXME: This synchronization is not consistent with the rest of
-        //        placement. This requires it to happen after the placement is
-        //        verified. See issue #2801
-        sync_grid_to_blocks();
         post_place_sync();
     }
 
@@ -877,6 +874,18 @@ void vpr_load_placement(t_vpr_setup& vpr_setup, const t_arch& arch) {
     place_ctx.placement_id = read_place(filename_opts.NetFile.c_str(), filename_opts.PlaceFile.c_str(),
                                         blk_loc_registry,
                                         filename_opts.verify_file_digests, device_ctx.grid);
+
+    // Verify that the placement invariants are met after reading the placement
+    // from a file.
+    unsigned num_errors = verify_placement(g_vpr_ctx);
+    if (num_errors == 0) {
+        VTR_LOG("Completed placement consistency check successfully.\n");
+    } else {
+        VPR_ERROR(VPR_ERROR_PLACE,
+                  "Completed placement consistency check, %d errors found.\n"
+                  "Aborting program.\n",
+                  num_errors);
+    }
 }
 
 RouteStatus vpr_route_flow(const Netlist<>& net_list,

vpr/src/pack/sync_netlists_to_routing_flat.cpp

@@ -70,19 +70,24 @@ inline ClusterBlockId get_cluster_block_from_rr_node(RRNodeId inode){
     auto& place_ctx = g_vpr_ctx.placement();
     auto& rr_graph = device_ctx.rr_graph;
 
-    auto physical_tile = device_ctx.grid.get_physical_type({
-        rr_graph.node_xlow(inode),
-        rr_graph.node_ylow(inode),
-        rr_graph.node_layer(inode)
-    });
+    t_physical_tile_loc node_phy_tile_loc(rr_graph.node_xlow(inode),
+                                          rr_graph.node_ylow(inode),
+                                          rr_graph.node_layer(inode));
+    auto physical_tile = device_ctx.grid.get_physical_type(node_phy_tile_loc);
 
     int source_pin = rr_graph.node_pin_num(inode);
 
     auto [_, subtile] = get_sub_tile_from_pin_physical_num(physical_tile, source_pin);
 
+    // The placer will only place Clusters at the root tile location. Need to
+    // offset to the root tile location to get the cluster block at this
+    // tile.
+    int width_offset = device_ctx.grid.get_width_offset(node_phy_tile_loc);
+    int height_offset = device_ctx.grid.get_height_offset(node_phy_tile_loc);
+
     ClusterBlockId clb = place_ctx.grid_blocks().block_at_location({
-        rr_graph.node_xlow(inode),
-        rr_graph.node_ylow(inode),
+        rr_graph.node_xlow(inode) - width_offset,
+        rr_graph.node_ylow(inode) - height_offset,
         subtile,
         rr_graph.node_layer(inode)
     });
@@ -185,7 +190,6 @@ static void sync_pb_routes_to_routing(void){
     auto& device_ctx = g_vpr_ctx.device();
     auto& atom_ctx = g_vpr_ctx.atom();
     auto& cluster_ctx = g_vpr_ctx.mutable_clustering();
-    auto& place_ctx = g_vpr_ctx.placement();
     auto& route_ctx = g_vpr_ctx.routing();
     auto& rr_graph = device_ctx.rr_graph;
 
@@ -220,6 +224,7 @@ static void sync_pb_routes_to_routing(void){
 
         /* Restore the connections */
         for(auto [source_inode, sink_inode]: conns_to_restore){
+            ClusterBlockId clb = get_cluster_block_from_rr_node(source_inode);
             auto physical_tile = device_ctx.grid.get_physical_type({
                 rr_graph.node_xlow(source_inode),
                 rr_graph.node_ylow(source_inode),
@@ -228,15 +233,6 @@ static void sync_pb_routes_to_routing(void){
             int source_pin = rr_graph.node_pin_num(source_inode);
             int sink_pin = rr_graph.node_pin_num(sink_inode);
 
-            auto [_, subtile] = get_sub_tile_from_pin_physical_num(physical_tile, source_pin);
-
-            ClusterBlockId clb = place_ctx.grid_blocks().block_at_location({
-                rr_graph.node_xlow(source_inode),
-                rr_graph.node_ylow(source_inode),
-                subtile,
-                rr_graph.node_layer(source_inode)
-            });
-
             /* Look up pb graph pins from pb type if pin is not on tile, look up from block otherwise */
             const t_pb_graph_pin* source_pb_graph_pin, *sink_pb_graph_pin;
             if(is_pin_on_tile(physical_tile, sink_pin)){
@@ -361,14 +357,7 @@ static void sync_clustered_netlist_to_routing(void){
 
             int pin_index = rr_graph.node_pin_num(rt_node.inode);
 
-            auto [_, subtile] = get_sub_tile_from_pin_physical_num(physical_tile, pin_index);
-
-            ClusterBlockId clb = place_ctx.grid_blocks().block_at_location({
-                rr_graph.node_xlow(rt_node.inode),
-                rr_graph.node_ylow(rt_node.inode),
-                subtile,
-                rr_graph.node_layer(rt_node.inode)
-            });
+            ClusterBlockId clb = get_cluster_block_from_rr_node(rt_node.inode);
 
             if(!is_pin_on_tile(physical_tile, pin_index))
                 continue;

vpr/src/place/verify_placement.cpp

@@ -33,6 +33,7 @@
  *  - The grid blocks matches the block locations.
  *  - Blocks are not in invalid grid locations.
  *  - All clusters are placed.
+ *  - Clusters are only placed at the root tile location of large tiles.
  *
  *  @param blk_loc_registry     A registry containing the block locations and
  *                              the blocks at each grid location.
@@ -57,6 +58,35 @@ static unsigned check_block_placement_consistency(const BlkLocRegistry& blk_loc_
             for (int j = 0; j < (int)device_grid.height(); j++) {
                 const t_physical_tile_loc tile_loc(i, j, layer_num);
                 const auto& type = device_grid.get_physical_type(tile_loc);
+
+                // If this is not a root tile block, ensure that its usage is 0
+                // and that it has no valid clusters placed at this location.
+                // TODO: Eventually it should be made impossible to place blocks
+                //       at these locations.
+                if (device_grid.get_width_offset(tile_loc) != 0 ||
+                    device_grid.get_height_offset(tile_loc) != 0) {
+                    // Usage must be 0
+                    if (grid_blocks.get_usage(tile_loc) != 0) {
+                        VTR_LOG_ERROR(
+                            "%d blocks were placed at non-root tile location "
+                            "(%d, %d, %d), but no blocks should be placed here.\n",
+                            grid_blocks.get_usage(tile_loc), i, j, layer_num);
+                        num_errors++;
+                    }
+                    // Check that all clusters at this tile location are invalid.
+                    for (int k = 0; k < type->capacity; k++) {
+                        ClusterBlockId bnum = grid_blocks.block_at_location({i, j, k, layer_num});
+                        if (bnum.is_valid()) {
+                            VTR_LOG_ERROR(
+                                "Block %zu was placed at non-root tile location "
+                                "(%d, %d, %d), but no blocks should be placed "
+                                "here.\n",
+                                size_t(bnum), i, j, layer_num);
+                            num_errors++;
+                        }
+                    }
+                }
+
                 if (grid_blocks.get_usage(tile_loc) > type->capacity) {
                     VTR_LOG_ERROR(
                         "%d blocks were placed at grid location (%d,%d,%d), but location capacity is %d.\n",

vpr/src/util/vpr_utils.cpp

@@ -88,82 +88,6 @@ void print_tabs(FILE* fpout, int num_tab) {
     }
 }
 
-/* Points the place_ctx.grid_blocks structure back to the blocks list */
-void sync_grid_to_blocks() {
-    auto& place_ctx = g_vpr_ctx.mutable_placement();
-    auto& device_ctx = g_vpr_ctx.device();
-    auto& device_grid = device_ctx.grid;
-
-    const int num_layers = device_ctx.grid.get_num_layers();
-
-    auto& grid_blocks = place_ctx.mutable_grid_blocks();
-    auto& block_locs = place_ctx.block_locs();
-
-    /* Reset usage and allocate blocks list if needed */
-    grid_blocks = GridBlock(device_grid.width(), device_grid.height(), device_ctx.grid.get_num_layers());
-
-
-    for (int layer_num = 0; layer_num < num_layers; layer_num++) {
-        for (int x = 0; x < (int)device_grid.width(); ++x) {
-            for (int y = 0; y < (int)device_grid.height(); ++y) {
-                const t_physical_tile_type_ptr type = device_ctx.grid.get_physical_type({x, y, layer_num});
-                grid_blocks.initialized_grid_block_at_location({x, y, layer_num}, type->capacity);
-            }
-        }
-    }
-
-    /* Go through each block */
-    auto& cluster_ctx = g_vpr_ctx.clustering();
-    for (ClusterBlockId blk_id : cluster_ctx.clb_nlist.blocks()) {
-        const auto& blk_loc = block_locs[blk_id].loc;
-        int blk_x = block_locs[blk_id].loc.x;
-        int blk_y = block_locs[blk_id].loc.y;
-        int blk_z = block_locs[blk_id].loc.sub_tile;
-        int blk_layer = block_locs[blk_id].loc.layer;
-
-        auto type = physical_tile_type(blk_loc);
-
-        /* Check range of block coords */
-        if (blk_x < 0 || blk_y < 0
-            || (blk_x + type->width - 1) > int(device_ctx.grid.width() - 1)
-            || (blk_y + type->height - 1) > int(device_ctx.grid.height() - 1)
-            || blk_z < 0 || blk_z > (type->capacity)) {
-            VPR_FATAL_ERROR(VPR_ERROR_PLACE, "Block %zu is at invalid location (%d, %d, %d).\n",
-                            size_t(blk_id), blk_x, blk_y, blk_z);
-        }
-
-        /* Check types match */
-        if (type != device_ctx.grid.get_physical_type({blk_x, blk_y, blk_layer})) {
-            VPR_FATAL_ERROR(VPR_ERROR_PLACE, "A block is in a grid location (%d x %d) layer (%d) with a conflicting types '%s' and '%s' .\n",
-                            blk_x, blk_y, blk_layer,
-                            type->name.c_str(),
-                            device_ctx.grid.get_physical_type({blk_x, blk_y, blk_layer})->name.c_str());
-        }
-
-        /* Check already in use */
-        if (grid_blocks.block_at_location(blk_loc)) {
-            VPR_FATAL_ERROR(VPR_ERROR_PLACE, "Location (%d, %d, %d, %d) is used more than once.\n",
-                            blk_x, blk_y, blk_z, blk_layer);
-        }
-
-        if (device_ctx.grid.get_width_offset({blk_x, blk_y, blk_layer}) != 0 || device_ctx.grid.get_height_offset({blk_x, blk_y, blk_layer}) != 0) {
-            VPR_FATAL_ERROR(VPR_ERROR_PLACE, "Large block not aligned in placement for cluster_ctx.blocks %lu at (%d, %d, %d, %d).",
-                            size_t(blk_id), blk_x, blk_y, blk_z, blk_layer);
-        }
-
-        /* Set the block */
-        for (int width = 0; width < type->width; ++width) {
-            for (int height = 0; height < type->height; ++height) {
-                grid_blocks.set_block_at_location({blk_x + width, blk_y + height, blk_z, blk_layer}, blk_id);
-                grid_blocks.increment_usage({blk_x + width, blk_y + height, blk_layer});
-
-                VTR_ASSERT(device_ctx.grid.get_width_offset({blk_x + width, blk_y + height, blk_layer}) == width);
-                VTR_ASSERT(device_ctx.grid.get_height_offset({blk_x + width, blk_y + height, blk_layer}) == height);
-            }
-        }
-    }
-}
-
 std::string rr_node_arch_name(RRNodeId inode, bool is_flat) {
     auto& device_ctx = g_vpr_ctx.device();
     const auto& rr_graph = device_ctx.rr_graph;
@@ -2134,4 +2058,4 @@ int PortPinToBlockPinConverter::get_blk_pin_from_port_pin(int blk_type_index, in
     // Return the port and port_pin for the pin.
     int blk_pin = blk_pin_from_port_pin_[blk_type_index][sub_tile][port][port_pin];
     return blk_pin;
-}
+}

vpr/src/util/vpr_utils.h

@@ -126,8 +126,6 @@ inline ParentNetId get_cluster_net_parent_id(const AtomLookup& atom_look_up, Clu
     return par_net_id;
 }
 
-void sync_grid_to_blocks();
-
 //Returns a user-friendly architectural identifier for the specified RR node
 std::string rr_node_arch_name(RRNodeId inode, bool is_flat);