diff --git a/benchmarks/linear_programming/cuopt/run_cpufj.cu b/benchmarks/linear_programming/cuopt/run_cpufj.cu new file mode 100644 index 0000000000..67dafa847b --- /dev/null +++ b/benchmarks/linear_programming/cuopt/run_cpufj.cu @@ -0,0 +1,444 @@ +/* clang-format off */ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +/* clang-format on */ + +#include "miplib2017_bks.hpp" + +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using i_t = int; +using f_t = double; +namespace mip = cuopt::mathematical_optimization::mip; + +using clk = std::chrono::high_resolution_clock; +double since(clk::time_point t0) +{ + return std::chrono::duration_cast>(clk::now() - t0).count(); +} + +struct climber_result_t { + bool crossed{false}; + double t_first{-1.0}; + f_t best_objective{std::numeric_limits::infinity()}; + i_t iterations{0}; + double seconds{0.0}; +}; + +void pin_to_core(int core) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(core, &set); + pthread_setaffinity_np(pthread_self(), sizeof(set), &set); +} + +// The CPUs this process is actually permitted to run on. A cgroup mask can be non-contiguous, so +// indexing hardware_concurrency() directly would collide several climbers onto one core. +std::vector allowed_cpus() +{ + std::vector allowed; + cpu_set_t set; + CPU_ZERO(&set); + if (sched_getaffinity(0, sizeof(set), &set) == 0) { + for (int cpu = 0; cpu < CPU_SETSIZE; ++cpu) { + if (CPU_ISSET(cpu, &set)) allowed.push_back(cpu); + } + } + if (allowed.empty()) allowed.push_back(0); + return allowed; +} + +void run_climber(mip::fj_cpu_climber_t* climber, + f_t time_limit, + int core, + climber_result_t& result) +{ + pin_to_core(core); + const auto t0 = clk::now(); + + climber->improvement_callback = [&result, t0](f_t objective, const std::vector&, double) { + if (!result.crossed) { + result.crossed = true; + result.t_first = since(t0); + } + result.best_objective = objective; + }; + + mip::cpufj_solve(climber, time_limit); + + result.seconds = since(t0); + result.iterations = climber->iterations; +} + +} // namespace + +int main(int argc, char** argv) +{ + if (argc < 2) { + std::fprintf(stderr, "usage: %s [time_limit_s=60] [climbers=16] [seed=12345]\n", + argv[0]); + return 2; + } + const std::string path = argv[1]; + const f_t time_limit = argc > 2 ? std::atof(argv[2]) : 60.0; + const int n_climbers = argc > 3 ? std::atoi(argv[3]) : 16; + const unsigned base_seed = argc > 4 ? (unsigned)std::atoll(argv[4]) : 12345u; + + // Console sink so the engine's end-of-solve incumbent audit is visible, as solve_MIP does it. + cuopt::init_logger_t log_guard("", true); + + raft::handle_t handle; + + const auto mps_data_model = cuopt::mathematical_optimization::io::read_mps(path, false); + const auto op_problem = + cuopt::mathematical_optimization::mps_data_model_to_optimization_problem( + &handle, mps_data_model); + mip::problem_t problem(op_problem); + + // Anonymise the instance before anything under evolution can see it. + // + // problem_t exposes var_names, row_names and objective_name as public members, and + // the FJ code receives problem_t&. For a fixed benchmark set those strings are an + // exact fingerprint -- row_names[0] alone identifies most MIPLIB instances -- so a + // candidate could branch on identity and return a memorised objective. Reading the + // MODEL is intended and useful: coefficients, bounds, variable types, sparsity and + // row structure are all untouched here, so recognising set-packing rows, knapsack + // substructure or GUB constraints still works exactly as before. Only the labels go. + // + // Each string is cleared in place rather than the vectors being emptied, so size() + // and indexing stay valid and any code that walks names by variable index still + // works -- it just gets empty strings. + // + // This file is outside target_code and is sha256-gated by evaluate.py's FROZEN_FILES, + // so a candidate cannot restore the names. Do not move this below the solve. + for (auto& name : problem.var_names) name.clear(); + for (auto& name : problem.row_names) name.clear(); + problem.objective_name.clear(); + + std::printf("instance: %s n_vars=%d n_cstrs=%d nnz=%d\n", + path.c_str(), + problem.n_variables, + problem.n_constraints, + problem.nnz); + + // Taken from the host-side parse, so it is independent of everything under target_code. + { + const auto& col_indices = mps_data_model.get_constraint_matrix_indices(); + const auto& row_lb = mps_data_model.get_constraint_lower_bounds(); + const auto& row_ub = mps_data_model.get_constraint_upper_bounds(); + const int64_t nnz = (int64_t)col_indices.size(); + + const i_t n_cols = mps_data_model.get_n_variables(); + std::vector degree(n_cols, 0); + for (i_t index : col_indices) { + if (index >= 0 && index < n_cols) ++degree[index]; + } + std::sort(degree.begin(), degree.end()); + + const i_t max_degree = degree.empty() ? 0 : degree.back(); + auto quantile = [&](double q) { + return degree.empty() + ? 0 + : degree[std::min(degree.size() - 1, (size_t)(q * degree.size()))]; + }; + int64_t top10 = 0; + for (size_t k = 0; k < 10 && k < degree.size(); ++k) + top10 += degree[degree.size() - 1 - k]; + const double mean_degree = n_cols > 0 ? (double)nnz / n_cols : 0.0; + std::printf("census cols: n=%d degree max=%d p99=%d p90=%d median=%d mean=%.1f" + " widest=%.1f%% top10=%.1f%% of nnz hub=%.0fx mean\n", + n_cols, + max_degree, + quantile(0.99), + quantile(0.90), + quantile(0.50), + mean_degree, + nnz > 0 ? 100.0 * max_degree / nnz : 0.0, + nnz > 0 ? 100.0 * top10 / nnz : 0.0, + mean_degree > 0 ? max_degree / mean_degree : 0.0); + + const i_t n_rows = (i_t)std::min(row_lb.size(), row_ub.size()); + i_t lb_only = 0, ub_only = 0, equality = 0, ranged = 0, free_rows = 0; + for (i_t r = 0; r < n_rows; ++r) { + const bool has_lb = std::isfinite((double)row_lb[r]); + const bool has_ub = std::isfinite((double)row_ub[r]); + if (has_lb && has_ub) { + ++(row_lb[r] == row_ub[r] ? equality : ranged); + } else if (has_lb) { + ++lb_only; + } else if (has_ub) { + ++ub_only; + } else { + ++free_rows; + } + } + std::printf("census rows: n=%d lb_only=%d ub_only=%d equality=%d ranged=%d free=%d" + " one_sided=%.1f%%\n", + n_rows, + lb_only, + ub_only, + equality, + ranged, + free_rows, + n_rows > 0 ? 100.0 * (lb_only + ub_only) / n_rows : 0.0); + } + + // FROZEN -- defines t=0 for the benchmark. Everything above it (the MPS parse, + // problem construction under problem/, and the name anonymisation) is outside + // target_code; everything below it is editable. A marker any later would leave + // editable code ahead of the clock, which is somewhere to do unmeasured work; any + // earlier would charge the budget for a parse and a CUDA context no candidate can + // influence. + CUOPT_LOG_INFO("CPUFJ solve window start"); + + // Shared by every climber. Built by build_start_assignment, which is editable -- + // this driver is not. + mip::solution_t solution(problem); + mip::build_start_assignment(problem, solution, &handle); + + std::vector> preemption_flags(n_climbers); + std::vector>> climbers(n_climbers); + // Composition and per-climber parameters come from build_climber_portfolio, which + // is editable. The log prefix is assigned here and not there, so every climber + // stays identifiable in the log whatever the portfolio does. + mip::build_climber_portfolio(problem, solution, preemption_flags, climbers, base_seed); + for (int k = 0; k < n_climbers; ++k) { + climbers[k]->log_prefix = "[climber " + std::to_string(k) + "] "; + } + + const std::vector cpus = allowed_cpus(); + std::printf("running %d climbers x %.0fs, base seed %u, %zu allowed CPUs (%d..%d)\n", + n_climbers, (double)time_limit, base_seed, cpus.size(), cpus.front(), cpus.back()); + + std::vector results(n_climbers); + std::vector threads; + threads.reserve(n_climbers); + const auto wall0 = clk::now(); + for (int k = 0; k < n_climbers; ++k) { + threads.emplace_back( + run_climber, climbers[k].get(), time_limit, cpus[k % cpus.size()], std::ref(results[k])); + } + for (auto& t : threads) { + t.join(); + } + const double wall = since(wall0); + + int crossed = 0; + double sum_iters = 0; + f_t best_overall = std::numeric_limits::infinity(); + std::printf("\n climber | crossed | t_first(s) | obj | iters | iters/s\n"); + std::printf("---------+---------+------------+--------------+----------+---------\n"); + for (int k = 0; k < n_climbers; ++k) { + const auto& r = results[k]; + sum_iters += r.iterations; + if (r.crossed) { + ++crossed; + best_overall = std::min(best_overall, r.best_objective); + } + std::printf(" %7d | %7s | %10s | %12.6g | %8d | %8.0f\n", + k, + r.crossed ? "YES" : "no", + r.crossed ? std::to_string(r.t_first).c_str() : "-", + r.crossed ? (double)r.best_objective : 0.0, + r.iterations, + r.seconds > 0 ? r.iterations / r.seconds : 0.0); + } + // Runs after the measured window closes, so its cost is off the clock. + // Solver space is always a minimisation, so beating the best known is always a smaller value. + const auto bks_user = cuopt_bench::lookup_miplib_bks(path); + const double bks = bks_user ? (double)problem.get_solver_obj_from_user_obj((f_t)*bks_user) : 0.0; + const double bks_slack = std::max(1e-6, std::fabs(bks) * 1e-9); + + int audited = 0, invalid = 0; + std::printf("\n climber | viol rows worst/tol | bnd viol worst/tol | int viol worst/tol |" + " obj drift rel | vs bks\n"); + std::printf("---------+----------------------+---------------------+---------------------+" + "----------------------+----------\n"); + for (int k = 0; k < n_climbers; ++k) { + auto& c = *climbers[k]; + if (c.feasible_found != results[k].crossed) { + std::printf(" %7d | feasible_found=%d disagrees with a reported incumbent=%d\n", + k, + (int)c.feasible_found, + (int)results[k].crossed); + ++invalid; + continue; + } + if (!c.feasible_found) continue; + ++audited; + + const double int_tol = c.view.pb.tolerances.integrality_tolerance; + + i_t rows_over = 0; + double worst_row_ratio = 0.0; + for (i_t r = 0; r < c.view.pb.n_constraints; ++r) { + __float128 activity = 0; + for (i_t j = c.h_offsets[r]; j < c.h_offsets[r + 1]; ++j) { + const i_t var = c.h_variables[j]; + const double coefficient = c.h_coefficients[j]; + const double value = c.h_best_assignment[var]; + activity += (__float128)coefficient * (__float128)value; + } + + const f_t lb = c.h_cstr_lb[r]; + const f_t ub = c.h_cstr_ub[r]; + const __float128 below = (__float128)lb - activity; + const __float128 above = activity - (__float128)ub; + const double excess = (double)std::max(std::max(below, above), (__float128)0); + if (excess <= 0.0) continue; + + const double tol = c.view.get_corrected_tolerance(r, lb, ub); + const double ratio = tol > 0 ? excess / tol : std::numeric_limits::infinity(); + if (ratio > 1.0) ++rows_over; + worst_row_ratio = std::max(worst_row_ratio, ratio); + } + + i_t bounds_over = 0; + i_t integers_over = 0; + double worst_bound_ratio = 0.0; + double worst_integer_ratio = 0.0; + __float128 objective = 0; + for (i_t v = 0; v < c.view.pb.n_variables; ++v) { + auto bounds = c.h_var_bounds[v].get(); + const double x = (double)c.h_best_assignment[v]; + const double out = std::max( + std::max((double)cuopt::get_lower(bounds) - x, x - (double)cuopt::get_upper(bounds)), 0.0); + if (out > int_tol) ++bounds_over; + worst_bound_ratio = std::max(worst_bound_ratio, int_tol > 0 ? out / int_tol : 0.0); + + if (c.view.pb.is_integer_var(v)) { + const double residual = std::fabs(x - std::round(x)); + if (residual > int_tol) ++integers_over; + worst_integer_ratio = std::max(worst_integer_ratio, int_tol > 0 ? residual / int_tol : 0.0); + } + const double coefficient = c.h_obj_coeffs[v]; + objective += (__float128)coefficient * (__float128)x; + } + + // Differenced before narrowing; the drift is smaller than a double ulp of the sum. + const __float128 difference = objective - (__float128)results[k].best_objective; + const double drift = (double)(difference < 0 ? -difference : difference); + const double exact = (double)objective; + const double scale = std::max(std::fabs(exact), 1.0); + const bool below_bks = bks_user && exact < bks - bks_slack; + const bool bad = rows_over > 0 || bounds_over > 0 || integers_over > 0 || below_bks; + if (bad) ++invalid; + std::printf(" %7d | %9d %10.3g | %8d %10.3g | %8d %10.3g | %12.3g %6.1e | %9.3g%s%s\n", + k, + rows_over, + worst_row_ratio, + bounds_over, + worst_bound_ratio, + integers_over, + worst_integer_ratio, + drift, + drift / scale, + bks_user ? exact - bks : 0.0, + below_bks ? " BELOW BKS" : "", + bad ? " INVALID" : ""); + } + std::printf("AUDIT: %d/%d reporting climbers checked, %d invalid, bks %s\n", + audited, + crossed, + invalid, + bks_user ? std::to_string(*bks_user).c_str() + : (cuopt_bench::is_known_infeasible(path) ? "known infeasible" : "unknown")); + + std::printf("\n climber | moves | apply nnz | nnz/move | bitmap elems | ratio |" + " bump/apply | bump/weight | mtm inval | cache hit%%\n"); + std::printf("---------+-----------+------------+----------+--------------+-------+" + "------------+-------------+-----------+-----------\n"); + for (int k = 0; k < n_climbers; ++k) { + const auto& c = *climbers[k]; + const int64_t bitmap = 2 * c.n_moves_applied * (int64_t)c.view.pb.n_variables; + const int64_t probes = c.hit_count + c.miss_count; + std::printf(" %7d | %9lld | %10lld | %8.1f | %12lld | %5.0f | %10lld | %11lld | %9lld |" + " %9.2f\n", + k, + (long long)c.n_moves_applied, + (long long)c.apply_move_nnz, + c.n_moves_applied > 0 ? (double)c.apply_move_nnz / c.n_moves_applied : 0.0, + (long long)bitmap, + c.apply_move_nnz > 0 ? (double)bitmap / c.apply_move_nnz : 0.0, + (long long)c.n_version_bumps_apply, + (long long)c.n_version_bumps_weights, + (long long)c.n_mtm_cache_invalidations, + probes > 0 ? 100.0 * c.hit_count / probes : 0.0); + } + + std::printf("\n climber | mtm calls | row entries | ent/call | capped ent | capped/call |" + " score calls | score nnz | nnz/score | nnz budget\n"); + std::printf("---------+-----------+-------------+----------+-------------+-------------+" + "-------------+-----------+-----------+-----------\n"); + for (int k = 0; k < n_climbers; ++k) { + const auto& c = *climbers[k]; + std::printf(" %7d | %9lld | %11lld | %8.0f | %11lld | %11.0f | %11lld | %9lld | %9.1f |" + " %10d\n", + k, + (long long)c.n_mtm_calls, + (long long)c.mtm_row_entries, + c.n_mtm_calls > 0 ? (double)c.mtm_row_entries / c.n_mtm_calls : 0.0, + (long long)c.mtm_entries_capped, + c.n_mtm_calls > 0 ? (double)c.mtm_entries_capped / c.n_mtm_calls : 0.0, + (long long)c.n_compute_score_calls, + (long long)c.compute_score_nnz, + c.n_compute_score_calls > 0 + ? (double)c.compute_score_nnz / c.n_compute_score_calls + : 0.0, + c.nnz_samples); + } + + std::printf("\n climber | refresh period | lhs total | periodic | bigval | perturb | restart |" + " epi vars | epi projections\n"); + std::printf("---------+----------------+-----------+----------+--------+---------+---------+" + "----------+----------------\n"); + for (int k = 0; k < n_climbers; ++k) { + const auto& c = *climbers[k]; + std::printf(" %7d | %14d | %9lld | %8lld | %6lld | %7lld | %7lld | %8zu | %15lld\n", + k, + c.lhs_refresh_period_used, + (long long)c.n_lhs_recompute_total, + (long long)c.n_lhs_recompute_periodic, + (long long)c.n_lhs_recompute_bigval, + (long long)c.n_lhs_recompute_perturb, + (long long)c.n_lhs_recompute_restart, + c.epigraph_vars.size(), + (long long)c.n_epigraph_projections); + } + + std::printf("\nSUMMARY: %d/%d crossed (%.0f%%) wall=%.1fs total_iters=%.0f agg_iters/s=%.0f\n", + crossed, + n_climbers, + 100.0 * crossed / n_climbers, + wall, + sum_iters, + wall > 0 ? sum_iters / wall : 0.0); + if (crossed > 0) { std::printf("BEST OBJECTIVE: %.10g\n", (double)best_overall); } + return 0; +} diff --git a/benchmarks/linear_programming/cuopt/run_mip.cpp b/benchmarks/linear_programming/cuopt/run_mip.cpp index 98cd9a56d2..6a9a3303fd 100644 --- a/benchmarks/linear_programming/cuopt/run_mip.cpp +++ b/benchmarks/linear_programming/cuopt/run_mip.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -136,6 +137,52 @@ std::vector> read_solution_from_dir(const std::string file_p return initial_solutions; } +struct incumbent_record_t { + double objective; + double work_timestamp; + double wall_time; +}; + +class incumbent_tracker_t : public cuopt::internals::get_solution_callback_t { + public: + explicit incumbent_tracker_t(std::chrono::high_resolution_clock::time_point start_time) + : start_time_(start_time) + { + } + + void get_solution(void* /*data*/, + void* cost, + void* /*solution_bound*/, + void* /*user_data*/) override + { + const auto now = std::chrono::high_resolution_clock::now(); + records_.push_back({*static_cast(cost), + 0.0, + std::chrono::duration(now - start_time_).count()}); + } + + void write_csv(const std::string& path) const + { + std::ofstream file(path); + if (!file.is_open()) { + std::cerr << "Error opening incumbent file " << path << std::endl; + return; + } + file << "index,objective,work_timestamp,wall_time_s\n"; + for (size_t i = 0; i < records_.size(); ++i) { + file << i << "," << std::setprecision(15) << records_[i].objective << "," + << records_[i].work_timestamp << "," << std::setprecision(6) << records_[i].wall_time + << "\n"; + } + } + + size_t size() const { return records_.size(); } + + private: + std::chrono::high_resolution_clock::time_point start_time_; + std::vector records_; +}; + int run_single_file(std::string file_path, int device, int batch_id, @@ -151,6 +198,8 @@ int run_single_file(std::string file_path, double work_limit, bool deterministic) { + (void)cudaFree(0); + const raft::handle_t handle_{}; cuopt::mathematical_optimization::mip_solver_settings_t settings; std::string base_filename = file_path.substr(file_path.find_last_of("/\\") + 1); @@ -218,6 +267,8 @@ int run_single_file(std::string file_path, cuopt::mathematical_optimization::benchmark_info_t benchmark_info; settings.benchmark_info_ptr = &benchmark_info; auto start_run_solver = std::chrono::high_resolution_clock::now(); + incumbent_tracker_t incumbent_tracker(start_run_solver); + settings.set_mip_callback(&incumbent_tracker); auto solution = cuopt::mathematical_optimization::solve_mip(&handle_, mps_data_model, settings); CUOPT_LOG_INFO( "first obj: %f last improvement of best feasible: %f last improvement after recombination: %f", @@ -291,6 +342,13 @@ int run_single_file(std::string file_path, << "\n"; write_to_output_file(out_dir, base_filename, device, n_gpus, batch_id, ss.str()); CUOPT_LOG_INFO("Results written to the file %s", base_filename.c_str()); + if (out_dir != "") { + std::string csv_path = + out_dir + "/" + base_filename.substr(0, base_filename.find(".mps")) + "_incumbents.csv"; + incumbent_tracker.write_csv(csv_path); + CUOPT_LOG_INFO( + "Incumbent trace (%zu entries) written to %s", incumbent_tracker.size(), csv_path.c_str()); + } return sol_found; } diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index b375cc4c56..72d03b3d2d 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -49,6 +49,7 @@ rapids_cmake_build_type(Release) option(CMAKE_CUDA_LINEINFO "Enable the -lineinfo option for nvcc useful for cuda-memcheck / profiler" ON) option(BUILD_TESTS "Configure CMake to build tests" ON) option(BUILD_LP_ONLY "Build only linear programming components, exclude routing and MIP-specific files" OFF) +option(BUILD_MIP_BENCHMARKS "Build MIP benchmarks" OFF) option(SKIP_C_PYTHON_ADAPTERS "Skip building C and Python adapter files (cython_solve.cu and cuopt_c.cpp)" OFF) option(SKIP_ROUTING_BUILD "Skip building routing components" OFF) option(SKIP_GRPC_BUILD "Skip building gRPC and protobuf components" OFF) @@ -307,6 +308,25 @@ set(BUILD_SHARED_LIBS OFF) FetchContent_MakeAvailable(pslp) set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_SAVED}) +FetchContent_Declare( + highway + GIT_REPOSITORY "https://github.com/google/highway.git" + GIT_TAG "1.4.0" + GIT_PROGRESS TRUE + EXCLUDE_FROM_ALL + SYSTEM +) + +set(HWY_ENABLE_CONTRIB OFF CACHE BOOL "" FORCE) +set(HWY_ENABLE_EXAMPLES OFF CACHE BOOL "" FORCE) +set(HWY_ENABLE_TESTS OFF CACHE BOOL "" FORCE) +set(HWY_ENABLE_INSTALL OFF CACHE BOOL "" FORCE) + +set(BUILD_SHARED_LIBS_SAVED ${BUILD_SHARED_LIBS}) +set(BUILD_SHARED_LIBS OFF) +FetchContent_MakeAvailable(highway) +set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_SAVED}) + # dejavu - header-only graph automorphism library for MIP symmetry detection # https://github.com/markusa4/dejavu (header-only, skip its CMakeLists.txt) @@ -666,6 +686,7 @@ target_include_directories(cuopt_objs PRIVATE target_include_directories(cuopt_objs SYSTEM PRIVATE "${pslp_SOURCE_DIR}/include" "${dejavu_SOURCE_DIR}" + "${highway_SOURCE_DIR}" ) target_include_directories(cuopt_objs @@ -691,6 +712,9 @@ target_include_directories(cuopt_objs target_link_libraries(cuopt_objs PRIVATE $) add_dependencies(cuopt_objs PSLP) +target_link_libraries(cuopt_objs PRIVATE $) +add_dependencies(cuopt_objs hwy) + # Link KaMinPar by file to avoid export dependency tracking (mirrors PSLP above). # KaMinPar is a from-source static library fully embedded into libcuopt.so; it is never # installed (INSTALL_KAMINPAR OFF) and consumers of cuopt::cuopt never use it, so it must @@ -777,6 +801,9 @@ target_link_libraries(cuopt_objs # - generate tests -------------------------------------------------------------------------------- if (BUILD_TESTS) include(CTest) +endif () + +if (BUILD_TESTS OR (BUILD_MIP_BENCHMARKS AND NOT BUILD_LP_ONLY)) add_library(cuopt_static STATIC $) target_link_libraries(cuopt_static PUBLIC @@ -813,10 +840,15 @@ if (BUILD_TESTS) ) target_link_libraries(cuopt_static PRIVATE $) add_dependencies(cuopt_static PSLP) + target_link_libraries(cuopt_static PRIVATE $) + add_dependencies(cuopt_static hwy) target_link_libraries(cuopt_static PRIVATE $) if (TARGET KaMinPar) add_dependencies(cuopt_static KaMinPar) endif () +endif () + +if (BUILD_TESTS) add_subdirectory(tests) endif (BUILD_TESTS) @@ -857,6 +889,8 @@ target_link_libraries(cuopt ) target_link_libraries(cuopt PRIVATE $) add_dependencies(cuopt PSLP) +target_link_libraries(cuopt PRIVATE $) +add_dependencies(cuopt hwy) target_link_libraries(cuopt PRIVATE $) if (TARGET KaMinPar) add_dependencies(cuopt KaMinPar) @@ -1031,7 +1065,6 @@ if (NOT BUILD_LP_ONLY) endif () -option(BUILD_MIP_BENCHMARKS "Build MIP benchmarks" OFF) if (BUILD_MIP_BENCHMARKS AND NOT BUILD_LP_ONLY) add_executable(solve_MIP ../benchmarks/linear_programming/cuopt/run_mip.cpp) target_include_directories(solve_MIP @@ -1065,6 +1098,31 @@ if (BUILD_MIP_BENCHMARKS AND NOT BUILD_LP_ONLY) "${CMAKE_CURRENT_SOURCE_DIR}/src" ) + # CPU FJ standalone portfolio benchmark + add_executable(solve_CPUFJ ../benchmarks/linear_programming/cuopt/run_cpufj.cu) + set_target_properties(solve_CPUFJ PROPERTIES CXX_SCAN_FOR_MODULES OFF) + target_compile_options(solve_CPUFJ + PRIVATE "$<$:${CUOPT_CXX_FLAGS}>" + "$<$:${CUOPT_CUDA_FLAGS}>" + "$<$:-fopenmp>" + ) + target_link_libraries(solve_CPUFJ + PUBLIC + cuopt_static + OpenMP::OpenMP_CXX + OpenMP::OpenMP_CUDA + ) + target_include_directories(solve_CPUFJ + PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src" + "${papilo_SOURCE_DIR}/src" + "${papilo_BINARY_DIR}" + ) + target_include_directories(solve_CPUFJ SYSTEM PRIVATE + "${pslp_SOURCE_DIR}/include" + "${dejavu_SOURCE_DIR}" + ) + endif () option(BUILD_LP_BENCHMARKS "Build LP benchmarks" OFF) diff --git a/cpp/include/cuopt/mathematical_optimization/cpu_optimization_problem.hpp b/cpp/include/cuopt/mathematical_optimization/cpu_optimization_problem.hpp index 28aa91a82f..f0673a4f66 100644 --- a/cpp/include/cuopt/mathematical_optimization/cpu_optimization_problem.hpp +++ b/cpp/include/cuopt/mathematical_optimization/cpu_optimization_problem.hpp @@ -123,6 +123,12 @@ class cpu_optimization_problem_t : public optimization_problem_interface_t& get_variable_names() const override; const std::vector& get_row_names() const override; const std::vector& get_quadratic_objective_offsets() const override; @@ -208,6 +214,7 @@ class cpu_optimization_problem_t : public optimization_problem_interface_t std::string get_objective_name() const override; std::string get_problem_name() const override; problem_category_t get_problem_category() const override; + /** + * @brief Whether any variable type is SEMI_CONTINUOUS. + * + * Cached in set_variable_types(); used to skip SC reformulation host probes. + */ + bool has_semi_continuous_variables() const noexcept; const std::vector& get_variable_names() const override; const std::vector& get_row_names() const override; const std::vector& get_quadratic_objective_offsets() const override; @@ -391,6 +397,7 @@ class optimization_problem_t : public optimization_problem_interface_t rmm::cuda_stream_view stream_view_; problem_category_t problem_category_ = problem_category_t::LP; + bool has_semi_continuous_variables_{false}; bool maximize_{false}; i_t n_vars_{0}; i_t n_constraints_{0}; diff --git a/cpp/src/branch_and_bound/branch_and_bound.cpp b/cpp/src/branch_and_bound/branch_and_bound.cpp index 29174148b7..18b45afc19 100644 --- a/cpp/src/branch_and_bound/branch_and_bound.cpp +++ b/cpp/src/branch_and_bound/branch_and_bound.cpp @@ -2430,6 +2430,7 @@ void branch_and_bound_t::solve_submip(diving_worker_t* worke f_t work_limit = 1.0; submip_fj_cpu_worker.create_worker(submip_bnb.original_lp_, submip_bnb.var_types_, + submip_bnb.original_problem_.num_cols, initial_guess, submip_bnb.settings_, std::format("{} [CPU FJ]", log_prefix), @@ -2891,6 +2892,7 @@ void branch_and_bound_t::recursive_submip(diving_worker_t* w f_t work_limit = 1.0; submip_fj_cpu_worker.create_worker(worker->leaf_problem, var_types, + original_problem_.num_cols, worker->leaf_solution.x, settings_, std::format("{} [CPU FJ]", log_prefix), @@ -2963,12 +2965,31 @@ void branch_and_bound_t::launch_root_heuristics( f_t work_limit = std::numeric_limits::infinity(); f_t time_limit = settings_.time_limit - toc(exploration_stats_.start_time); + // Odd passes start from the incumbent, even ones from the relaxation. The size guard covers a + // concurrent pass having grown the LP past the crush the incumbent was last taken through. + std::vector fj_seed; + if (cut_pass % 2 == 1) { + mutex_upper_.lock(); + if (incumbent_.has_incumbent && incumbent_.x.size() == (size_t)lp.num_cols) { + fj_seed = incumbent_.x; + } + mutex_upper_.unlock(); + } + if (fj_seed.empty()) { fj_seed = sol; } + current_heuristic->fj_cpu_worker_.improvement_callback = [this](f_t obj, const std::vector& assignment, double work_units) { set_solution_from_cpu_fj(obj, assignment, work_units); }; - current_heuristic->fj_cpu_worker_.create_worker( - lp, var_types_, sol, settings_, "[RootCut CPUFJ] "); + current_heuristic->fj_cpu_worker_.create_worker(lp, + var_types_, + original_problem_.num_cols, + fj_seed, + settings_, + "[RootCut CPUFJ " + std::to_string(cut_pass) + + "] ", + /*seed=*/-1, + /*lane=*/cut_pass); ++(*worker_count); #pragma omp task priority(CUOPT_DEFAULT_TASK_PRIORITY) \ @@ -3525,6 +3546,29 @@ mip_status_t branch_and_bound_t::solve(mip_solution_t& solut lp_status_t root_status = lp_status_t::UNSET; solving_root_relaxation_ = true; + // Started here so the lanes run through the root LP and every cut pass. No relaxation exists + // yet, so they seed from the anchor. + root_heuristics_t root_heuristics(settings_.num_threads - 1); + const i_t n_root_fj_lanes = + std::clamp(settings_.num_threads / 4, 0, CUOPT_MIP_ROOT_CPUFJ_MAX_LANES); + const f_t root_fj_time_limit = settings_.time_limit - toc(exploration_stats_.start_time); + if (!settings_.deterministic && n_root_fj_lanes > 0 && root_fj_time_limit > 0) { + root_heuristics.start_persistent_lanes( + original_lp_, + var_types_, + original_problem_.num_cols, + {}, + settings_, + n_root_fj_lanes, + root_fj_time_limit, + (int64_t)settings_.random_seed, + [this](f_t obj, const std::vector& assignment, double work_units) { + cuopt_assert(assignment.size() == (size_t)original_problem_.num_cols, + "root CPU FJ lanes must report a slack-free assignment"); + set_solution_from_cpu_fj(obj, assignment, work_units); + }); + } + f_t root_relax_start_time = tic(); if (!enable_concurrent_lp_root_solve()) { @@ -3682,8 +3726,6 @@ mip_status_t branch_and_bound_t::solve(mip_solution_t& solut compute_user_objective(original_lp_, root_relax_objective); } - root_heuristics_t root_heuristics(settings_.num_threads - 1); - f_t cut_generation_start_time = tic(); i_t cut_pool_size = 0; for (i_t cut_pass = 0; cut_pass < settings_.max_cut_passes; cut_pass++) { diff --git a/cpp/src/mip_heuristics/CMakeLists.txt b/cpp/src/mip_heuristics/CMakeLists.txt index f54619cfe3..a35cdd7e4f 100644 --- a/cpp/src/mip_heuristics/CMakeLists.txt +++ b/cpp/src/mip_heuristics/CMakeLists.txt @@ -45,6 +45,8 @@ set(MIP_NON_LP_FILES ${CMAKE_CURRENT_SOURCE_DIR}/feasibility_jump/feasibility_jump.cu ${CMAKE_CURRENT_SOURCE_DIR}/feasibility_jump/feasibility_jump_kernels.cu ${CMAKE_CURRENT_SOURCE_DIR}/feasibility_jump/fj_cpu.cu + ${CMAKE_CURRENT_SOURCE_DIR}/feasibility_jump/fj_cpu_binary.cu + ${CMAKE_CURRENT_SOURCE_DIR}/feasibility_jump/fj_cpu_binary_kernels.cpp ${CMAKE_CURRENT_SOURCE_DIR}/feasibility_jump/early_cpufj.cu ${CMAKE_CURRENT_SOURCE_DIR}/feasibility_jump/early_gpufj.cu) diff --git a/cpp/src/mip_heuristics/diversity/diversity_manager.cu b/cpp/src/mip_heuristics/diversity/diversity_manager.cu index ec82c4b423..e69022c42d 100644 --- a/cpp/src/mip_heuristics/diversity/diversity_manager.cu +++ b/cpp/src/mip_heuristics/diversity/diversity_manager.cu @@ -8,6 +8,7 @@ #include "cuda_profiler_api.h" #include "diversity_manager.cuh" +#include #include #include @@ -22,6 +23,9 @@ #include #include +#include + +#include #include #include #include @@ -314,6 +318,13 @@ bool diversity_manager_t::run_presolve(f_t time_limit, timer_t global_ if (run_probing_cache && !global_timer.check_time_limit() && !presolve_timer.check_time_limit()) { log_presolve_budget("PROBING", probing_features, probing_budget); + // The early CPUFJ lanes hold their threads for the whole of presolve, and probing's default + // task count assumes the whole team. Its pools are sized per task, so this bounds host memory + // as well as concurrency. + const i_t held_by_cpufj = + context.early_cpufj_ptr != nullptr ? (i_t)context.early_cpufj_ptr->lane_count() : 0; + ls.constraint_prop.bounds_update.settings.num_tasks = + std::max(1, omp_get_num_threads() - 1 - held_by_cpufj); f_t time_for_probing_cache = std::min(time_limit, (f_t)global_timer.remaining_time()); timer_t probing_timer{time_for_probing_cache}; [[maybe_unused]] const auto probing_t0 = std::chrono::steady_clock::now(); diff --git a/cpp/src/mip_heuristics/diversity/population.cu b/cpp/src/mip_heuristics/diversity/population.cu index e6fffa97b2..033119915f 100644 --- a/cpp/src/mip_heuristics/diversity/population.cu +++ b/cpp/src/mip_heuristics/diversity/population.cu @@ -264,41 +264,6 @@ bool population_t::is_better_than_best_feasible(solution_t& return obj_better && sol.get_feasible(); } -template -void population_t::invoke_get_solution_callback( - solution_t& sol, internals::get_solution_callback_t* callback) -{ - f_t user_objective = sol.get_user_objective(); - f_t user_bound = context.stats.get_solution_bound(); - solution_t temp_sol(sol); - problem_ptr->post_process_assignment(temp_sol.assignment); - if (problem_ptr->has_papilo_presolve_data()) { - problem_ptr->papilo_uncrush_assignment(temp_sol.assignment); - } - - std::vector user_objective_vec(1); - std::vector user_bound_vec(1); - std::vector user_assignment_vec(temp_sol.assignment.size()); - user_objective_vec[0] = user_objective; - user_bound_vec[0] = user_bound; - raft::copy(user_assignment_vec.data(), - temp_sol.assignment.data(), - temp_sol.assignment.size(), - temp_sol.handle_ptr->get_stream()); - temp_sol.handle_ptr->sync_stream(); - if (mip_solver_settings_accessor::has_semi_continuous_callback_translation( - context.settings)) { - mip::strip_semi_continuous_auxiliaries_from_assignment( - user_assignment_vec, - mip_solver_settings_accessor::get_semi_continuous_original_num_variables( - context.settings)); - } - callback->get_solution(user_assignment_vec.data(), - user_objective_vec.data(), - user_bound_vec.data(), - callback->get_user_data()); -} - template void population_t::run_solution_callbacks(solution_t& sol) { @@ -309,15 +274,14 @@ void population_t::run_solution_callbacks(solution_t& sol) context.settings.benchmark_info_ptr->last_improvement_of_best_feasible = timer.elapsed_time(); } CUOPT_LOG_DEBUG("Population: Found new best solution %g", sol.get_user_objective()); - if (problem_ptr->branch_and_bound_callback != nullptr) { - problem_ptr->branch_and_bound_callback(sol.get_host_assignment(), - heuristics_origin_t::HEURISTICS); - } - for (auto callback : user_callbacks) { - if (callback->get_type() == internals::base_solution_callback_type::GET_SOLUTION) { - auto get_sol_callback = static_cast(callback); - invoke_get_solution_callback(sol, get_sol_callback); + if (problem_ptr->branch_and_bound_callback != nullptr || + context.solution_publication.enabled()) { + auto host_assignment = sol.get_host_assignment(); + if (problem_ptr->branch_and_bound_callback != nullptr) { + problem_ptr->branch_and_bound_callback(host_assignment, heuristics_origin_t::HEURISTICS); } + context.solution_publication.publish_if_better( + problem_ptr, host_assignment, sol.get_objective()); } // Save the best objective here even if callback handling later exits early. // This prevents older solutions from being reported as "new best" in subsequent callbacks. diff --git a/cpp/src/mip_heuristics/diversity/population.cuh b/cpp/src/mip_heuristics/diversity/population.cuh index 593b1ddf1e..5a9db26928 100644 --- a/cpp/src/mip_heuristics/diversity/population.cuh +++ b/cpp/src/mip_heuristics/diversity/population.cuh @@ -160,9 +160,6 @@ class population_t { void diversity_step(i_t max_iterations_without_improvement); - void invoke_get_solution_callback(solution_t& sol, - internals::get_solution_callback_t* callback); - // does some consistency tests bool test_invariant(); diff --git a/cpp/src/mip_heuristics/early_heuristic.cuh b/cpp/src/mip_heuristics/early_heuristic.cuh index 6654470732..84d5f86f7c 100644 --- a/cpp/src/mip_heuristics/early_heuristic.cuh +++ b/cpp/src/mip_heuristics/early_heuristic.cuh @@ -7,18 +7,13 @@ #pragma once -#include -#include - #include - -#include - -#include +#include #include #include #include +#include #include namespace cuopt::mathematical_optimization::mip { @@ -34,25 +29,13 @@ template class early_heuristic_t { public: early_heuristic_t(const optimization_problem_t& op_problem, - const typename mip_solver_settings_t::tolerances_t& tolerances, early_incumbent_callback_t incumbent_callback) - : incumbent_callback_(std::move(incumbent_callback)) + : objective_scaling_factor_(op_problem.get_sense() ? -op_problem.get_objective_scaling_factor() + : op_problem.get_objective_scaling_factor()), + objective_offset_(op_problem.get_sense() ? -op_problem.get_objective_offset() + : op_problem.get_objective_offset()), + incumbent_callback_(std::move(incumbent_callback)) { - RAFT_CUDA_TRY(cudaGetDevice(&device_id_)); - - // Build and preprocess on the original handle, then copy onto our own handle - // so the derived solver can run on a dedicated stream (prevents graph capture conflicts). - problem_t temp_problem(op_problem, tolerances, false); - temp_problem.preprocess_problem(); - temp_problem.handle_ptr->sync_stream(); - problem_ptr_ = std::make_unique>(temp_problem, &handle_); - - solution_ptr_ = std::make_unique>(*problem_ptr_); - thrust::fill(handle_.get_thrust_policy(), - solution_ptr_->assignment.begin(), - solution_ptr_->assignment.end(), - f_t{0}); - solution_ptr_->clamp_within_bounds(); } bool solution_found() const { return solution_found_; } @@ -60,12 +43,12 @@ class early_heuristic_t { // Return the best objective converted to user-space (sense-aware, offset-aware). f_t get_best_user_objective() const { - return problem_ptr_->get_user_obj_from_solver_obj(best_objective_); + return objective_scaling_factor_ * (best_objective_ + objective_offset_); } // Set the incumbent threshold. `obj` must be in THIS heuristic's solver-space - // (i.e. the space of problem_ptr_). Callers that hold a value from a different - // problem representation (e.g., the original pre-presolve problem) must convert - // it first, otherwise try_update_best will reject valid solutions. + // (i.e. the space of its input problem). Callers that hold a value from a + // different problem representation (e.g., the original pre-presolve problem) + // must convert it first, otherwise try_update_best will reject valid solutions. void set_best_objective(f_t obj) { best_objective_ = obj; } const std::vector& get_best_assignment() const { return best_assignment_; } @@ -73,40 +56,25 @@ class early_heuristic_t { ~early_heuristic_t() = default; // NOT thread-safe. solver_obj is in solver-space (always minimization). - // Uses a private CUDA stream to avoid racing with the FJ solver's stream. void try_update_best(f_t solver_obj, const std::vector& assignment) { if (solver_obj >= best_objective_) { return; } best_objective_ = solver_obj; - RAFT_CUDA_TRY(cudaSetDevice(device_id_)); - auto stream = handle_.get_stream(); - rmm::device_uvector d_assignment(assignment.size(), stream); - raft::copy(d_assignment.data(), assignment.data(), assignment.size(), stream); - problem_ptr_->post_process_assignment(d_assignment, true, stream); - auto user_assignment = cuopt::host_copy(d_assignment, stream); - - best_assignment_ = user_assignment; + best_assignment_ = ((Derived*)this)->to_user_assignment(assignment); solution_found_ = true; - f_t user_obj = problem_ptr_->get_user_obj_from_solver_obj(solver_obj); + f_t user_obj = get_best_user_objective(); // Log and callback are deferred to the shared incumbent_callback_ which enforces // global monotonicity across all early heuristic instances. if (incumbent_callback_) { - incumbent_callback_(solver_obj, user_obj, user_assignment, Derived::name()); + incumbent_callback_(solver_obj, user_obj, best_assignment_, Derived::name()); } } - int device_id_{0}; - - // handle_ must be declared before problem_ptr_/solution_ptr_ so it outlives them - // (C++ destroys members in reverse declaration order) - raft::handle_t handle_; - - std::unique_ptr> problem_ptr_; - std::unique_ptr> solution_ptr_; - bool solution_found_{false}; f_t best_objective_{std::numeric_limits::infinity()}; + f_t objective_scaling_factor_; + f_t objective_offset_; std::vector best_assignment_; early_incumbent_callback_t incumbent_callback_; diff --git a/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cu b/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cu index ba14e657d5..5f9a68ac99 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cu @@ -8,6 +8,10 @@ #include "early_cpufj.cuh" #include +#include + +#include +#include namespace cuopt::mathematical_optimization::mip { @@ -16,8 +20,9 @@ early_cpufj_t::early_cpufj_t( const optimization_problem_t& op_problem, const typename mip_solver_settings_t::tolerances_t& tolerances, early_incumbent_callback_t incumbent_callback) - : early_heuristic_t>( - op_problem, tolerances, std::move(incumbent_callback)) + : early_heuristic_t>(op_problem, std::move(incumbent_callback)), + problem_ptr_(&op_problem), + tolerances_(tolerances) { } @@ -28,43 +33,90 @@ early_cpufj_t::~early_cpufj_t() } template -void early_cpufj_t::start() +void early_cpufj_t::start(int n_lanes) { // 1: presolve, 1: early GPU FJ, 1: early CPU FJ - if (fj_cpu_ || omp_get_num_threads() < CUOPT_MIP_EARLY_CPUFJ_REQUIRED_THREAD_COUNT) { return; } + if (!climbers_.empty() || omp_get_num_threads() < CUOPT_MIP_EARLY_CPUFJ_REQUIRED_THREAD_COUNT) { + return; + } this->preemption_flag_.store(false); this->start_time_ = std::chrono::steady_clock::now(); - fj_cpu_ = init_fj_cpu_standalone(*this->problem_ptr_, *this->solution_ptr_, preemption_flag_); - - fj_cpu_->log_prefix = "[Early CPUFJ] "; - - fj_cpu_->improvement_callback = [this](f_t solver_obj, - const std::vector& assignment, - double) { this->try_update_best(solver_obj, assignment); }; - - CUOPT_LOG_DEBUG("Launching early CPUFJ task"); -#pragma omp task shared(fj_cpu_) priority(CUOPT_DEFAULT_TASK_PRIORITY) \ - depend(out : *fj_cpu_) default(none) - cpufj_solve(fj_cpu_.get()); + // Tasks are not preempted, so a lane posted beyond the team size would sit in the queue for the + // whole of presolve without running an iteration. + n_lanes = std::clamp(n_lanes, 1, omp_get_num_threads()); + const int64_t base_seed = cuopt::seed_generator::get_seed(); + climbers_.resize(n_lanes); + + auto report_incumbent = [this](f_t solver_obj, const std::vector& assignment, double) { + std::lock_guard guard(incumbent_mutex_); + this->try_update_best(solver_obj, assignment); + }; + + // Lane 0 builds the host problem representation and every other lane copies it. All of it + // finishes before the first task is posted, so no lane reads a template another lane is running + // on. seed_generator steps a non-atomic global, which is why the draws stay on this thread. + for (int k = 0; k < n_lanes; ++k) { + if (k == 0) { + climbers_[0] = + init_fj_cpu_from_optimization_problem(*this->problem_ptr_, tolerances_, preemption_flag_); + } else { + fj_settings_t settings; + settings.seed = (int)cuopt::seed_generator::get_seed(); + climbers_[k] = init_fj_cpu_clone(*climbers_[0], preemption_flag_, settings); + } + apply_lane_diversification(*climbers_[k], k, base_seed); + climbers_[k]->log_prefix = "[Early CPUFJ " + std::to_string(k) + "] "; + climbers_[k]->improvement_callback = report_incumbent; + } + + auto shared = std::make_shared>(); + for (int k = 0; k < n_lanes; ++k) + climbers_[k]->shared_incumbent = shared; + + CUOPT_LOG_DEBUG("Launching %d early CPUFJ tasks", n_lanes); + for (int k = 0; k < n_lanes; ++k) { + auto* climber = climbers_[k].get(); +#pragma omp task firstprivate(climber) priority(CUOPT_DEFAULT_TASK_PRIORITY) \ + depend(out : *climber) default(none) + cpufj_solve(climber); + } } template void early_cpufj_t::stop() { - if (!fj_cpu_) { return; } + if (climbers_.empty()) { return; } preemption_flag_.store(true); - fj_cpu_->halted = true; -#pragma omp taskwait depend(in : *fj_cpu_) // Wait for the early CPUFJ task to finish - - CUOPT_LOG_DEBUG("[Early CPUFJ] Stopped after %d iterations, solution_found=%d", - fj_cpu_ ? fj_cpu_->iterations : 0, + // Every lane is told to stop before any wait, otherwise the first wait blocks on a lane that has + // not been asked to exit yet. + for (auto& climber : climbers_) { + climber->halted = true; + } + for (size_t k = 0; k < climbers_.size(); ++k) { +#pragma omp taskwait depend(in : *climbers_[k]) // Wait for each early CPUFJ task to finish + } + + i_t total_iterations = 0; + for (const auto& climber : climbers_) { + total_iterations += climber->iterations; + } + + CUOPT_LOG_DEBUG("[Early CPUFJ] Stopped after %d iterations over %d climbers, solution_found=%d", + total_iterations, + (int)climbers_.size(), this->solution_found_); - fj_cpu_.reset(); + climbers_.clear(); +} + +template +std::vector early_cpufj_t::to_user_assignment(const std::vector& assignment) +{ + return assignment; } #if MIP_INSTANTIATE_FLOAT diff --git a/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cuh b/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cuh index e2bb2c07b2..3bae5ed63b 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/early_cpufj.cuh @@ -12,6 +12,8 @@ #include #include +#include +#include namespace cuopt::mathematical_optimization::mip { @@ -26,12 +28,25 @@ class early_cpufj_t : public early_heuristic_t static constexpr const char* name() { return "CPUFJ"; } - void start(); + // Lanes are OMP tasks that never yield, so n_lanes threads are unavailable to anything else + // until stop(). Callers sharing the team with other work size it accordingly. + void start(int n_lanes); void stop(); + int lane_count() const { return (int)climbers_.size(); } + private: - std::unique_ptr> fj_cpu_; + friend class early_heuristic_t>; + + std::vector to_user_assignment(const std::vector& assignment); + + const optimization_problem_t* problem_ptr_; + typename mip_solver_settings_t::tolerances_t tolerances_; + std::vector>> climbers_; std::atomic preemption_flag_{false}; + // try_update_best and the incumbent callback behind it are not thread-safe, and every lane + // reports into them from its own task. + std::mutex incumbent_mutex_; }; } // namespace cuopt::mathematical_optimization::mip diff --git a/cpp/src/mip_heuristics/feasibility_jump/early_gpufj.cu b/cpp/src/mip_heuristics/feasibility_jump/early_gpufj.cu index 463f074f59..c9d787a236 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/early_gpufj.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/early_gpufj.cu @@ -10,9 +10,15 @@ #include #include #include +#include #include #include +#include +#include + +#include +#include #include @@ -22,11 +28,26 @@ template early_gpufj_t::early_gpufj_t(const optimization_problem_t& op_problem, const mip_solver_settings_t& settings, early_incumbent_callback_t incumbent_callback) - : early_heuristic_t>( - op_problem, settings.get_tolerances(), std::move(incumbent_callback)) + : early_heuristic_t>(op_problem, std::move(incumbent_callback)) { - context_ptr_ = std::make_unique>( - &this->handle_, this->problem_ptr_.get(), settings); + RAFT_CUDA_TRY(cudaGetDevice(&device_id_)); + + // Build and preprocess on the original handle, then copy onto our own handle + // so the derived solver can run on a dedicated stream (prevents graph capture conflicts). + problem_t temp_problem(op_problem, settings.get_tolerances(), false); + temp_problem.preprocess_problem(); + temp_problem.handle_ptr->sync_stream(); + problem_ptr_ = std::make_unique>(temp_problem, &handle_); + + solution_ptr_ = std::make_unique>(*problem_ptr_); + thrust::fill(handle_.get_thrust_policy(), + solution_ptr_->assignment.begin(), + solution_ptr_->assignment.end(), + f_t{0}); + solution_ptr_->clamp_within_bounds(); + + context_ptr_ = + std::make_unique>(&handle_, problem_ptr_.get(), settings); } template @@ -81,6 +102,18 @@ void early_gpufj_t::stop() fj_ptr_.reset(); } +template +std::vector early_gpufj_t::to_user_assignment(const std::vector& assignment) +{ + // Uses a private CUDA stream to avoid racing with the FJ solver's stream. + RAFT_CUDA_TRY(cudaSetDevice(device_id_)); + auto stream = handle_.get_stream(); + rmm::device_uvector d_assignment(assignment.size(), stream); + raft::copy(d_assignment.data(), assignment.data(), assignment.size(), stream); + problem_ptr_->post_process_assignment(d_assignment, true, stream); + return cuopt::host_copy(d_assignment, stream); +} + #if MIP_INSTANTIATE_FLOAT template class early_gpufj_t; #endif diff --git a/cpp/src/mip_heuristics/feasibility_jump/early_gpufj.cuh b/cpp/src/mip_heuristics/feasibility_jump/early_gpufj.cuh index 99e8579d31..ed8d17206e 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/early_gpufj.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/early_gpufj.cuh @@ -8,8 +8,11 @@ #pragma once #include +#include +#include #include +#include namespace cuopt::mathematical_optimization::mip { @@ -34,6 +37,18 @@ class early_gpufj_t : public early_heuristic_t void stop(); private: + friend class early_heuristic_t>; + + std::vector to_user_assignment(const std::vector& assignment); + + int device_id_{0}; + + // handle_ must be declared before problem_ptr_/solution_ptr_ so it outlives them + // (C++ destroys members in reverse declaration order) + raft::handle_t handle_; + + std::unique_ptr> problem_ptr_; + std::unique_ptr> solution_ptr_; std::unique_ptr> context_ptr_; std::unique_ptr> fj_ptr_; }; diff --git a/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump.cuh b/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump.cuh index ac1da031e3..437dfa3a1b 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump.cuh @@ -529,12 +529,14 @@ class fj_t { HDI f_t lower_excess_score(i_t cstr, f_t lhs, f_t c_lb) const { - return raft::min(lhs - c_lb, (f_t)0); + const f_t excess = lhs - c_lb; + return excess < (f_t)0 ? excess : (f_t)0; } HDI f_t upper_excess_score(i_t cstr, f_t lhs, f_t c_ub) const { - return raft::min(c_ub - lhs, (f_t)0); + const f_t excess = c_ub - lhs; + return excess < (f_t)0 ? excess : (f_t)0; } // Computes the constraint's contribution to the feasibility score: @@ -564,7 +566,8 @@ class fj_t { { f_t cstr_tolerance = get_cstr_tolerance( c_lb, c_ub, pb.tolerances.absolute_tolerance, pb.tolerances.relative_tolerance); - return max((f_t)1e-12, cstr_tolerance - MACHINE_EPSILON); + const f_t corrected = cstr_tolerance - MACHINE_EPSILON; + return corrected > (f_t)1e-12 ? corrected : (f_t)1e-12; } HDI f_t get_corrected_tolerance(i_t cstr) const { diff --git a/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump_impl_common.cuh b/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump_impl_common.cuh index 046e138c5b..6535794e08 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump_impl_common.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump_impl_common.cuh @@ -196,7 +196,6 @@ HDI f_t get_breakthrough_move(typename fj_t::climber_data_t::view_t fj auto bounds = fj.pb.variable_bounds[var_idx]; f_t v_lb = get_lower(bounds); f_t v_ub = get_upper(bounds); - cuopt_assert(isfinite(v_lb) || isfinite(v_ub), "unexpected free variable"); cuopt_assert(v_lb <= v_ub, "invalid bounds"); cuopt_assert(fj.pb.check_variable_within_bounds(var_idx, fj.incumbent_assignment[var_idx]), "invalid incumbent assignment"); @@ -220,10 +219,12 @@ HDI f_t get_breakthrough_move(typename fj_t::climber_data_t::view_t fj new_val = old_val + delta_ij; } - // fallback + // A positive coefficient gives a negative delta, so only the lower bound can be the one broken, + // and a broken bound is finite. Free and half-free variables therefore land here finite too. if (!fj.pb.check_variable_within_bounds(var_idx, new_val)) { new_val = obj_coeff > 0 ? v_lb : v_ub; } + cuopt_assert(isfinite(new_val), "breakthrough move left the representable range"); return new_val; } diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu index b789159953..c375b16441 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cu @@ -9,14 +9,19 @@ #include #include +#include +#include +#include #include "feasibility_jump.cuh" #include "feasibility_jump_impl_common.cuh" #include "fj_cpu.cuh" +#include "fj_cpu_binary.cuh" #include "fj_cpu_worker.cuh" #include +#include #include #include @@ -30,9 +35,11 @@ #include #include #include +#include #include #include #include +#include #include #include @@ -63,6 +70,16 @@ void finalize_fj_cpu_host_initialization( i_t nnz, const typename mip_solver_settings_t::tolerances_t& tolerances); +template +static void finalize_fj_cpu_host_initialization_from_template( + fj_cpu_climber_t& fj_cpu, + const fj_cpu_climber_t& tmpl, + i_t n_variables, + i_t n_constraints, + i_t n_integer_vars, + i_t nnz, + const typename mip_solver_settings_t::tolerances_t& tolerances); + template thrust::tuple get_mtm_for_bound(const typename fj_t::climber_data_t::view_t& fj, i_t var_idx, @@ -87,22 +104,19 @@ thrust::tuple get_mtm_for_bound(const typename fj_t::climber } template -thrust::tuple get_mtm_for_constraint( - const typename fj_t::climber_data_t::view_t& fj, - i_t var_idx, - i_t cstr_idx, - f_t cstr_coeff, - f_t c_lb, - f_t c_ub, - const ArrayType& assignment, - const ArrayType& lhs_vector) +thrust::tuple get_mtm_for_constraint(i_t var_idx, + i_t cstr_idx, + f_t cstr_coeff, + f_t c_lb, + f_t c_ub, + const ArrayType& assignment, + const ArrayType& lhs_vector, + f_t cstr_tolerance) { f_t sign = -1; f_t delta_ij = 0; f_t slack = 0; - f_t cstr_tolerance = fj.get_corrected_tolerance(cstr_idx, c_lb, c_ub); - f_t old_val = assignment[var_idx]; // process each bound as two separate constraints @@ -133,7 +147,7 @@ thrust::tuple get_mtm_for_constraint( } template -std::pair feas_score_constraint(const typename fj_t::climber_data_t::view_t& fj, +std::pair feas_score_constraint(fj_cpu_climber_t& fj_cpu, f_t delta, i_t cstr_idx, f_t cstr_coeff, @@ -141,33 +155,39 @@ std::pair feas_score_constraint(const typename fj_t::climber f_t c_ub, f_t current_lhs, f_t left_weight, - f_t right_weight) + f_t right_weight, + f_t cstr_tolerance) { + const auto& fj = fj_cpu.view; cuopt_assert(isfinite(delta), "invalid delta"); - cuopt_assert(cstr_coeff != 0 && isfinite(cstr_coeff), "invalid coefficient"); + // A model may store explicit zeros, and a zero coefficient contributes nothing to the row. + cuopt_assert(isfinite(cstr_coeff), "invalid coefficient"); f_t base_feas = 0; f_t bonus_robust = 0; f_t bounds[2] = {c_lb, c_ub}; cuopt_assert(isfinite(c_lb) || isfinite(c_ub), "no range"); + + // Independent of bound_idx. + const f_t moved_lhs = current_lhs + cstr_coeff * delta; + const bool old_viol = fj.excess_score(cstr_idx, current_lhs, c_lb, c_ub) < -cstr_tolerance; + const bool new_viol = fj.excess_score(cstr_idx, moved_lhs, c_lb, c_ub) < -cstr_tolerance; + for (i_t bound_idx = 0; bound_idx < 2; ++bound_idx) { if (!isfinite(bounds[bound_idx])) continue; - // factor to correct the lhs/rhs to turn a lb <= lhs <= ub constraint into - // two virtual leq constraints "lhs <= ub" and "-lhs <= -lb" in order to match - // the convention of the paper - - // TODO: broadcast left/right weights to a csr_offset-indexed table? local minimums - // usually occur on a rarer basis (around 50 iteratiosn to 1 local minimum) - // likely unreasonable and overkill however + // factor to correct the lhs/rhs to turn a lb <= lhs <= ub constraint into two virtual leq + // constraints "lhs <= ub" and "-lhs <= -lb", to match the convention of the paper f_t cstr_weight = bound_idx == 0 ? left_weight : right_weight; f_t sign = bound_idx == 0 ? -1 : 1; f_t rhs = bounds[bound_idx] * sign; f_t old_lhs = current_lhs * sign; - f_t new_lhs = (current_lhs + cstr_coeff * delta) * sign; - f_t old_slack = rhs - old_lhs; - f_t new_slack = rhs - new_lhs; + f_t new_lhs = moved_lhs * sign; + [[maybe_unused]] + f_t old_slack = rhs - old_lhs; + [[maybe_unused]] + f_t new_slack = rhs - new_lhs; cuopt_assert(isfinite(cstr_weight), "invalid weight"); cuopt_assert(cstr_weight >= 0, "invalid weight"); @@ -175,12 +195,6 @@ std::pair feas_score_constraint(const typename fj_t::climber cuopt_assert(isfinite(new_lhs), ""); cuopt_assert(isfinite(old_slack) && isfinite(new_slack), ""); - f_t cstr_tolerance = fj.get_corrected_tolerance(cstr_idx, c_lb, c_ub); - - bool old_viol = fj.excess_score(cstr_idx, current_lhs, c_lb, c_ub) < -cstr_tolerance; - bool new_viol = - fj.excess_score(cstr_idx, current_lhs + cstr_coeff * delta, c_lb, c_ub) < -cstr_tolerance; - bool old_sat = old_lhs < rhs + cstr_tolerance; bool new_sat = new_lhs < rhs + cstr_tolerance; @@ -203,12 +217,12 @@ std::pair feas_score_constraint(const typename fj_t::climber // simple improvement else if (!old_sat && !new_sat && old_lhs > new_lhs) { cuopt_assert(old_viol && new_viol, ""); - base_feas += (i_t)(cstr_weight * fj.settings->parameters.excess_improvement_weight); + base_feas += (i_t)(cstr_weight * fj_cpu.settings.parameters.excess_improvement_weight); } // simple worsening else if (!old_sat && !new_sat && old_lhs < new_lhs) { cuopt_assert(old_viol && new_viol, ""); - base_feas -= (i_t)(cstr_weight * fj.settings->parameters.excess_improvement_weight); + base_feas -= (i_t)(cstr_weight * fj_cpu.settings.parameters.excess_improvement_weight); } // robustness score bonus if this would leave some strick slack @@ -276,43 +290,43 @@ static void print_timing_stats(fj_cpu_climber_t& fj_cpu) auto [apply_avg, apply_total] = compute_avg_and_total(fj_cpu.apply_move_times); auto [weights_avg, weights_total] = compute_avg_and_total(fj_cpu.update_weights_times); auto [compute_score_avg, compute_score_total] = compute_avg_and_total(fj_cpu.compute_score_times); - CUOPT_LOG_TRACE("=== Timing Statistics (Iteration %d) ===", fj_cpu.iterations); - CUOPT_LOG_TRACE("find_lift_move: avg=%.6f ms, total=%.6f ms, calls=%zu", + CUOPT_LOG_DEBUG("=== Timing Statistics (Iteration %d) ===", fj_cpu.iterations); + CUOPT_LOG_DEBUG("find_lift_move: avg=%.6f ms, total=%.6f ms, calls=%zu", lift_avg * 1000.0, lift_total * 1000.0, fj_cpu.find_lift_move_times.size()); - CUOPT_LOG_TRACE("find_mtm_move_viol: avg=%.6f ms, total=%.6f ms, calls=%zu", + CUOPT_LOG_DEBUG("find_mtm_move_viol: avg=%.6f ms, total=%.6f ms, calls=%zu", viol_avg * 1000.0, viol_total * 1000.0, fj_cpu.find_mtm_move_viol_times.size()); - CUOPT_LOG_TRACE("find_mtm_move_sat: avg=%.6f ms, total=%.6f ms, calls=%zu", + CUOPT_LOG_DEBUG("find_mtm_move_sat: avg=%.6f ms, total=%.6f ms, calls=%zu", sat_avg * 1000.0, sat_total * 1000.0, fj_cpu.find_mtm_move_sat_times.size()); - CUOPT_LOG_TRACE("apply_move: avg=%.6f ms, total=%.6f ms, calls=%zu", + CUOPT_LOG_DEBUG("apply_move: avg=%.6f ms, total=%.6f ms, calls=%zu", apply_avg * 1000.0, apply_total * 1000.0, fj_cpu.apply_move_times.size()); - CUOPT_LOG_TRACE("update_weights: avg=%.6f ms, total=%.6f ms, calls=%zu", + CUOPT_LOG_DEBUG("update_weights: avg=%.6f ms, total=%.6f ms, calls=%zu", weights_avg * 1000.0, weights_total * 1000.0, fj_cpu.update_weights_times.size()); - CUOPT_LOG_TRACE("compute_score: avg=%.6f ms, total=%.6f ms, calls=%zu", + CUOPT_LOG_DEBUG("compute_score: avg=%.6f ms, total=%.6f ms, calls=%zu", compute_score_avg * 1000.0, compute_score_total * 1000.0, fj_cpu.compute_score_times.size()); - CUOPT_LOG_TRACE("cache hit percentage: %.2f%%", + CUOPT_LOG_DEBUG("cache hit percentage: %.2f%%", (double)fj_cpu.hit_count / (fj_cpu.hit_count + fj_cpu.miss_count) * 100.0); - CUOPT_LOG_TRACE("bin candidate move hit percentage: %.2f%%", + CUOPT_LOG_DEBUG("bin candidate move hit percentage: %.2f%%", (double)fj_cpu.candidate_move_hits[0] / (fj_cpu.candidate_move_hits[0] + fj_cpu.candidate_move_misses[0]) * 100.0); - CUOPT_LOG_TRACE("int candidate move hit percentage: %.2f%%", + CUOPT_LOG_DEBUG("int candidate move hit percentage: %.2f%%", (double)fj_cpu.candidate_move_hits[1] / (fj_cpu.candidate_move_hits[1] + fj_cpu.candidate_move_misses[1]) * 100.0); - CUOPT_LOG_TRACE("cont candidate move hit percentage: %.2f%%", + CUOPT_LOG_DEBUG("cont candidate move hit percentage: %.2f%%", (double)fj_cpu.candidate_move_hits[2] / (fj_cpu.candidate_move_hits[2] + fj_cpu.candidate_move_misses[2]) * 100.0); - CUOPT_LOG_TRACE("========================================"); + CUOPT_LOG_DEBUG("========================================"); } template @@ -374,6 +388,11 @@ static void precompute_problem_features(fj_cpu_climber_t& fj_cpu) fj_cpu.problem_density = (double)total_nnz / ((double)n_vars * n_cstrs); } +// Greedy first-fit colouring of the variable co-occurrence graph, where each row is a clique. The +// adjacency is walked per variable and never stored: the clique expansion is far larger than nnz. +template +static void compute_variable_coloring(fj_cpu_climber_t& fj_cpu); + template static void log_regression_features(fj_cpu_climber_t& fj_cpu, double time_window_ms, @@ -397,9 +416,9 @@ static void log_regression_features(fj_cpu_climber_t& fj_cpu, double eval_intensity = (double)fj_cpu.nnz_processed_window / 1000.0; // Cache and locality metrics - i_t cache_hits_window = fj_cpu.hit_count - fj_cpu.hit_count_window_start; - i_t cache_misses_window = fj_cpu.miss_count - fj_cpu.miss_count_window_start; - i_t total_cache_accesses = cache_hits_window + cache_misses_window; + int64_t cache_hits_window = fj_cpu.hit_count - fj_cpu.hit_count_window_start; + int64_t cache_misses_window = fj_cpu.miss_count - fj_cpu.miss_count_window_start; + int64_t total_cache_accesses = cache_hits_window + cache_misses_window; double cache_hit_rate = total_cache_accesses > 0 ? (double)cache_hits_window / total_cache_accesses : 0.0; @@ -537,6 +556,207 @@ static inline std::pair range_for_constraint(fj_cpu_climber_t +static void compute_variable_coloring(fj_cpu_climber_t& fj_cpu) +{ + const i_t n_vars = fj_cpu.view.pb.n_variables; + const i_t n_cstrs = fj_cpu.view.pb.n_constraints; + + i_t max_row_length = 0; + double clique_edges = 0; + for (i_t row = 0; row < n_cstrs; ++row) { + const i_t length = fj_cpu.h_offsets[row + 1] - fj_cpu.h_offsets[row]; + max_row_length = std::max(max_row_length, length); + if (length > 1) clique_edges += (double)length * (length - 1) / 2.0; + } + if (n_vars <= 0 || max_row_length <= 0) return; + + const double class_size = (double)n_vars / max_row_length; + const double edges_per_nnz = clique_edges / std::max(1, (double)fj_cpu.view.pb.nnz); + if (class_size < fj_batch_min_class_size || edges_per_nnz > fj_batch_max_edges_per_nnz) { + CUOPT_LOG_DEBUG("CPUFJ move batching declined: class size %.2f, clique edges/nnz %.2f", + class_size, + edges_per_nnz); + return; + } + + const auto started = std::chrono::steady_clock::now(); + fj_cpu.h_var_color.assign(n_vars, -1); + fj_cpu.n_colors = 0; + std::vector neighbor_stamp(n_vars, -1); + std::vector color_stamp(n_vars, -1); + + for (i_t var = 0; var < n_vars; ++var) { + const auto [rev_begin, rev_end] = reverse_range_for_var(fj_cpu, var); + for (i_t p = rev_begin; p < rev_end; ++p) { + const auto [begin, end] = + range_for_constraint(fj_cpu, fj_cpu.h_reverse_constraints[p]); + for (i_t k = begin; k < end; ++k) { + const i_t other = fj_cpu.h_variables[k]; + if (other == var || neighbor_stamp[other] == var) continue; + neighbor_stamp[other] = var; + const i_t taken = fj_cpu.h_var_color[other]; + if (taken >= 0) color_stamp[taken] = var; + } + } + + i_t color = 0; + while (color < fj_cpu.n_colors && color_stamp[color] == var) ++color; + if (color == fj_cpu.n_colors) ++fj_cpu.n_colors; + fj_cpu.h_var_color[var] = color; + } + + fj_cpu.h_var_best_score.assign(n_vars, fj_staged_score_t::invalid()); + fj_cpu.h_var_best_delta.assign(n_vars, f_t{0}); + fj_cpu.h_var_best_stamp.assign(n_vars, 0); + fj_cpu.h_var_best_rowsum.assign(n_vars, 0); + fj_cpu.h_var_bucket_stamp.assign(n_vars, 0); + fj_cpu.batch_size_hist.assign(fj_batch_hist_bins, 0); + fj_cpu.h_color_candidates.assign(fj_cpu.n_colors, {}); + fj_cpu.h_color_epoch.assign(fj_cpu.n_colors, 0); + fj_cpu.var_best_epoch = 1; + + CUOPT_LOG_DEBUG("CPUFJ move batching: %d colours over %d variables in %.3f ms", + fj_cpu.n_colors, + n_vars, + std::chrono::duration(std::chrono::steady_clock::now() - + started) + .count()); +} + +// Sum of the versions of the rows a variable appears in. Versions only ever increase, so an +// unchanged sum means no incident row has been touched. +template +static inline int64_t incident_row_version_sum(fj_cpu_climber_t& fj_cpu, i_t var_idx) +{ + const auto [begin, end] = reverse_range_for_var(fj_cpu, var_idx); + int64_t sum = 0; + for (i_t p = begin; p < end; ++p) + sum += fj_cpu.h_cstr_version[fj_cpu.h_reverse_constraints[p]]; + return sum; +} + +// Records a candidate move for its variable. The table keeps a best per variable, independent of +// the argmax the caller is tracking, which is what lets a batch be assembled later. +template +static inline void record_var_best_move(fj_cpu_climber_t& fj_cpu, + i_t var_idx, + fj_staged_score_t score, + f_t delta) +{ + if (!fj_cpu.use_move_batching) return; + if (!(score > fj_staged_score_t::zero())) return; + + const bool current = fj_cpu.h_var_best_stamp[var_idx] == fj_cpu.var_best_epoch; + if (current && !(score > fj_cpu.h_var_best_score[var_idx])) return; + + fj_cpu.h_var_best_score[var_idx] = score; + fj_cpu.h_var_best_delta[var_idx] = delta; + fj_cpu.h_var_best_stamp[var_idx] = fj_cpu.var_best_epoch; + fj_cpu.h_var_best_rowsum[var_idx] = incident_row_version_sum(fj_cpu, var_idx); + + const i_t color = fj_cpu.h_var_color[var_idx]; + cuopt_assert(color >= 0 && color < fj_cpu.n_colors, "variable has no colour"); + if (fj_cpu.h_color_epoch[color] != fj_cpu.var_best_epoch) { + fj_cpu.h_color_candidates[color].clear(); + fj_cpu.h_color_epoch[color] = fj_cpu.var_best_epoch; + } + if (fj_cpu.h_var_bucket_stamp[var_idx] == fj_cpu.var_best_epoch) return; + fj_cpu.h_var_bucket_stamp[var_idx] = fj_cpu.var_best_epoch; + fj_cpu.h_color_candidates[color].push_back(var_idx); +} + +// Retires the whole table in constant time. Called wherever the weights or the assignment move far +// enough that every cached score is suspect. +template +static inline void retire_var_best_moves(fj_cpu_climber_t& fj_cpu) +{ + if (!fj_cpu.use_move_batching) return; + ++fj_cpu.var_best_epoch; +} + +// Companions per batch attempt, as min, median, max and mean. A median landing in the saturating +// last bin reads as that bin's index, and max_batch_size carries the true tail. +template +static void log_batch_distribution(const fj_cpu_climber_t& fj_cpu) +{ + if (fj_cpu.n_batch_attempts == 0) return; + + int32_t smallest = -1; + int32_t median = -1; + int64_t seen = 0; + for (size_t bin = 0; bin < fj_cpu.batch_size_hist.size(); ++bin) { + if (fj_cpu.batch_size_hist[bin] == 0) continue; + if (smallest < 0) smallest = (int32_t)bin; + seen += fj_cpu.batch_size_hist[bin]; + if (median < 0 && 2 * seen > fj_cpu.n_batch_attempts) median = (int32_t)bin; + } + + CUOPT_LOG_DEBUG( + "%sCPUFJ batch companions: min %d median %d max %lld mean %.3f over %lld attempts, %lld total, " + "%d colours, batching %s", + fj_cpu.log_prefix.c_str(), + smallest, + median, + (long long)fj_cpu.max_batch_size, + (double)fj_cpu.n_batched_moves / (double)fj_cpu.n_batch_attempts, + (long long)fj_cpu.n_batch_attempts, + (long long)fj_cpu.n_batched_moves, + fj_cpu.n_colors, + fj_cpu.use_move_batching ? "on" : "off"); +} + +// Companions for the chosen move: same colour, so they share no row with it or with each other and +// their recorded scores and deltas hold as the batch is applied. Excludes the chosen move itself. +template +static void collect_move_batch(fj_cpu_climber_t& fj_cpu, + fj_move_t chosen, + std::vector& batch) +{ + batch.clear(); + if (!fj_cpu.use_move_batching) return; + + const i_t color = fj_cpu.h_var_color[chosen.var_idx]; + cuopt_assert(color >= 0 && color < fj_cpu.n_colors, "chosen move has no colour"); + if (fj_cpu.h_color_epoch[color] != fj_cpu.var_best_epoch) return; + + for (i_t var_idx : fj_cpu.h_color_candidates[color]) { + if (var_idx == chosen.var_idx) continue; + if (fj_cpu.h_var_best_stamp[var_idx] != fj_cpu.var_best_epoch) continue; + if (!(fj_cpu.h_var_best_score[var_idx] > fj_staged_score_t::zero())) continue; + if (fj_cpu.h_var_best_rowsum[var_idx] != incident_row_version_sum(fj_cpu, var_idx)) + continue; + + batch.push_back({var_idx, fj_cpu.h_var_best_delta[var_idx]}); + // Invalidated so a second pass over the bucket cannot apply the move twice. + fj_cpu.h_var_best_stamp[var_idx] = 0; + } + + ++fj_cpu.n_batch_attempts; + fj_cpu.n_batched_moves += (int64_t)batch.size(); + ++fj_cpu.batch_size_hist[std::min(batch.size(), fj_cpu.batch_size_hist.size() - 1)]; + if ((int64_t)batch.size() > fj_cpu.max_batch_size) + fj_cpu.max_batch_size = (int64_t)batch.size(); + if (fj_cpu.n_batch_attempts == fj_batch_probe_attempts && + (double)fj_cpu.n_batched_moves < fj_batch_min_yield * (double)fj_batch_probe_attempts) { + fj_cpu.use_move_batching = false; + CUOPT_LOG_DEBUG("%sCPUFJ move batching off: %lld companions over %lld attempts", + fj_cpu.log_prefix.c_str(), + (long long)fj_cpu.n_batched_moves, + (long long)fj_cpu.n_batch_attempts); + } +} + template static inline bool check_variable_within_bounds(fj_cpu_climber_t& fj_cpu, i_t var_idx, @@ -548,6 +768,69 @@ static inline bool check_variable_within_bounds(fj_cpu_climber_t& fj_c return within_bounds; } +// Names the first variable whose assignment sits outside its own bounds, so the writer that left it +// there is identified by the call site. Scans, and is only reached through cuopt_func_call. +template +static void audit_assignment_bounds(fj_cpu_climber_t& fj_cpu, const char* site) +{ + for (i_t var = 0; var < fj_cpu.view.pb.n_variables; ++var) { + const f_t val = fj_cpu.h_assignment[var]; + auto bounds = fj_cpu.h_var_bounds[var].get(); + const bool inbox = fj_cpu.view.pb.check_variable_within_bounds(var, val); + const bool integral = + var_t::INTEGER != fj_cpu.h_var_types[var] || fj_cpu.view.pb.is_integer(val); + if (inbox && integral) continue; + + // stderr and flushed, so the abort below cannot swallow it. + std::fprintf(stderr, + "%sCPUFJ %s left var %d at %.17g outside [%.17g, %.17g], integer %d\n", + fj_cpu.log_prefix.c_str(), + site, + (int)var, + (double)val, + (double)get_lower(bounds), + (double)get_upper(bounds), + (int)(var_t::INTEGER == fj_cpu.h_var_types[var])); + std::fflush(stderr); + cuopt_assert(false, "assignment left the variable bounds"); + return; + } +} + +// Reports the first objective variable get_breakthrough_move would reject, reading the value both +// from the climber's vector and through the view span so a bad value is told from a stale span. +template +static void audit_breakthrough_inputs(fj_cpu_climber_t& fj_cpu) +{ + for (auto var_idx : fj_cpu.h_objective_vars) { + const f_t viewed = fj_cpu.view.incumbent_assignment[var_idx]; + if (fj_cpu.view.pb.check_variable_within_bounds(var_idx, viewed)) continue; + + const f_t direct = fj_cpu.h_assignment[var_idx]; + auto bounds = fj_cpu.h_var_bounds[var_idx].get(); + auto viewed_bnd = fj_cpu.view.pb.variable_bounds[var_idx]; + // stderr and flushed, so the abort below cannot swallow it. + std::fprintf(stderr, + "%sCPUFJ breakthrough input var %d: direct %.17g viewed %.17g nan %d, bounds " + "direct [%.17g, %.17g] viewed [%.17g, %.17g], obj %.17g, degree %d, integer %d\n", + fj_cpu.log_prefix.c_str(), + (int)var_idx, + (double)direct, + (double)viewed, + (int)(viewed != viewed), + (double)get_lower(bounds), + (double)get_upper(bounds), + (double)get_lower(viewed_bnd), + (double)get_upper(viewed_bnd), + (double)fj_cpu.h_obj_coeffs[var_idx], + (int)(fj_cpu.h_reverse_offsets[var_idx + 1] - fj_cpu.h_reverse_offsets[var_idx]), + (int)(var_t::INTEGER == fj_cpu.h_var_types[var_idx])); + std::fflush(stderr); + cuopt_assert(false, "breakthrough move input out of bounds"); + return; + } +} + template static inline bool is_integer_var(fj_cpu_climber_t& fj_cpu, i_t var_idx) { @@ -604,34 +887,60 @@ static inline std::pair compute_score(fj_cpu_climber_t(fj_cpu, var_idx); fj_cpu.nnz_processed_window += (offset_end - offset_begin); + const size_t nnz_read = (size_t)(offset_end - offset_begin); + ++fj_cpu.n_compute_score_calls; + fj_cpu.compute_score_nnz += (int64_t)nnz_read; + fj_cpu.h_reverse_constraints.byte_loads += nnz_read * sizeof(i_t); + fj_cpu.h_reverse_coefficients.byte_loads += nnz_read * sizeof(f_t); + fj_cpu.cached_cstr_bounds.byte_loads += nnz_read * sizeof(std::pair); + fj_cpu.h_lhs.byte_loads += nnz_read * sizeof(f_t); + fj_cpu.h_cstr_left_weights.byte_loads += nnz_read * sizeof(f_t); + fj_cpu.h_cstr_right_weights.byte_loads += nnz_read * sizeof(f_t); + fj_cpu.h_cstr_tolerance.byte_loads += nnz_read * sizeof(f_t); + + const i_t* const rev_cstr = fj_cpu.view.pb.reverse_constraints.data(); + const f_t* const rev_coeff = fj_cpu.view.pb.reverse_coefficients.data(); + const f_t* const row_lhs = fj_cpu.view.incumbent_lhs.data(); + const f_t* const weight_l = fj_cpu.view.cstr_left_weights.data(); + const f_t* const weight_r = fj_cpu.view.cstr_right_weights.data(); + const f_t* const row_tol = fj_cpu.h_cstr_tolerance.data(); + const std::pair* const cstr_bounds = fj_cpu.cached_cstr_bounds.data(); + for (i_t i = offset_begin; i < offset_end; i++) { - auto cstr_idx = fj_cpu.h_reverse_constraints[i]; - fj_cpu.unique_cstrs_accessed_window.insert(cstr_idx); - auto cstr_coeff = fj_cpu.h_reverse_coefficients[i]; - auto [c_lb, c_ub] = fj_cpu.cached_cstr_bounds[i].get(); + const i_t cstr_idx = rev_cstr[i]; + const f_t cstr_coeff = rev_coeff[i]; + const auto [c_lb, c_ub] = cstr_bounds[i]; + // An explicit zero moves no row, so the move cannot change this row's score. + if (cstr_coeff == f_t{0}) continue; cuopt_assert(c_lb <= c_ub, "invalid bounds"); - auto [cstr_base_feas, cstr_bonus_robust] = - feas_score_constraint(fj_cpu.view, - delta, - cstr_idx, - cstr_coeff, - c_lb, - c_ub, - fj_cpu.h_lhs[cstr_idx], - fj_cpu.h_cstr_left_weights[cstr_idx], - fj_cpu.h_cstr_right_weights[cstr_idx]); + auto [cstr_base_feas, cstr_bonus_robust] = feas_score_constraint(fj_cpu, + delta, + cstr_idx, + cstr_coeff, + c_lb, + c_ub, + row_lhs[cstr_idx], + weight_l[cstr_idx], + weight_r[cstr_idx], + row_tol[cstr_idx]); base_feas_sum += cstr_base_feas; bonus_robust_sum += cstr_bonus_robust; } f_t base_obj = 0; - if (obj_diff < 0) // improving move wrt objective - base_obj = fj_cpu.h_objective_weight; - else if (obj_diff > 0) - base_obj = -fj_cpu.h_objective_weight; + if (fj_cpu.h_objective_weight > 0 && obj_diff != 0) { + // Scaling base is only meaningful where there is feasibility impact to trade against. + f_t weighted = fj_cpu.h_objective_weight; + if (base_feas_sum != 0) { + cuopt_assert(fj_cpu.obj_magnitude > 0, "objective magnitude unit must be positive"); + weighted *= min((f_t)fj_obj_mult_max, + max((f_t)fj_obj_mult_min, fabs(obj_diff) / fj_cpu.obj_magnitude)); + } + base_obj = obj_diff < 0 ? weighted : -weighted; + } f_t bonus_breakthrough = 0; @@ -695,7 +1004,7 @@ static fj_staged_score_t two_opt_compute_pair_score( // The coefficients are already folded into lhs_delta, hence the unit coefficient auto [cstr_base_feas, cstr_bonus_robust] = - feas_score_constraint(fj_cpu.view, + feas_score_constraint(fj_cpu, lhs_delta, cstr_idx, 1, @@ -703,7 +1012,8 @@ static fj_staged_score_t two_opt_compute_pair_score( fj_cpu.h_cstr_ub[cstr_idx], fj_cpu.h_lhs[cstr_idx], fj_cpu.h_cstr_left_weights[cstr_idx], - fj_cpu.h_cstr_right_weights[cstr_idx]); + fj_cpu.h_cstr_right_weights[cstr_idx], + fj_cpu.h_cstr_tolerance[cstr_idx]); base_feas_sum += cstr_base_feas; bonus_robust_sum += cstr_bonus_robust; } @@ -918,7 +1228,7 @@ static void smooth_weights(fj_cpu_climber_t& fj_cpu) CPUFJ_NVTX_RANGE("CPUFJ::smooth_weights"); for (i_t cstr_idx = 0; cstr_idx < fj_cpu.view.pb.n_constraints; cstr_idx++) { // consider only satisfied constraints - if (fj_cpu.violated_constraints.count(cstr_idx)) continue; + if (fj_cpu.violated_constraints.contains(cstr_idx)) continue; f_t weight_l = max((f_t)0, fj_cpu.h_cstr_left_weights[cstr_idx] - 1); f_t weight_r = max((f_t)0, fj_cpu.h_cstr_right_weights[cstr_idx] - 1); @@ -928,8 +1238,74 @@ static void smooth_weights(fj_cpu_climber_t& fj_cpu) } if (fj_cpu.h_objective_weight > 0 && fj_cpu.h_incumbent_objective >= fj_cpu.h_best_objective) { - fj_cpu.h_objective_weight = max((f_t)0, fj_cpu.h_objective_weight - 1); + fj_cpu.h_objective_weight = + max(fj_cpu.seed_objective_weight, fj_cpu.h_objective_weight - 1); + } +} + +// Escalation threshold and step for the violated-row bump, in local minima without a severity gain. +constexpr int32_t fj_weight_escalate_after = 2000; +constexpr int32_t fj_weight_escalate_max = 100; + +// Satisfied neighbours sampled per violated row for the donation, and the floor a donor keeps. +constexpr int32_t fj_weight_donor_samples = 4; +constexpr double fj_weight_donation_floor = 1.0; + +// DDFW donation: reach through a variable of this violated row to a satisfied neighbour and take +// the bump back off its heavier side, so total weight stays roughly conserved. +template +static void donate_row_weight(fj_cpu_climber_t& fj_cpu, + i_t cstr_idx, + f_t delta, + raft::random::PCGenerator& rng) +{ + const auto [row_begin, row_end] = range_for_constraint(fj_cpu, cstr_idx); + const uint32_t row_width = (uint32_t)(row_end - row_begin); + // What a donor has to carry to still hold the floor once the delta comes off it. + const f_t donor_minimum = (f_t)fj_weight_donation_floor + delta; + i_t donor = -1; + bool donor_left = true; + f_t donor_weight = 0; + + for (i_t sample = 0; row_width > 0 && sample < fj_weight_donor_samples; ++sample) { + const i_t var_idx = fj_cpu.h_variables[row_begin + (i_t)(rng.next_u32() % row_width)]; + const auto [col_begin, col_end] = reverse_range_for_var(fj_cpu, var_idx); + if (col_end <= col_begin) continue; + const i_t candidate = fj_cpu.h_reverse_constraints[ + col_begin + (i_t)(rng.next_u32() % (uint32_t)(col_end - col_begin))]; + if (candidate == cstr_idx || !fj_cpu.satisfied_constraints.contains(candidate)) continue; + + const f_t left = fj_cpu.h_cstr_left_weights[candidate]; + const f_t right = fj_cpu.h_cstr_right_weights[candidate]; + const bool take_left = left >= right; + const f_t weight = take_left ? left : right; + if (weight < donor_minimum) continue; + if (donor >= 0 && weight <= donor_weight) continue; + + donor = candidate; + donor_left = take_left; + donor_weight = weight; + } + if (donor < 0) return; + + const f_t donated = donor_weight - delta; + cuopt_assert(donated >= (f_t)fj_weight_donation_floor, "donation broke the weight floor"); + if (donor_left) { + fj_cpu.h_cstr_left_weights[donor] = donated; + } else { + fj_cpu.h_cstr_right_weights[donor] = donated; } + ++fj_cpu.n_version_bumps_weights; + fj_cpu.h_cstr_version[donor]++; +} + +template +static i_t weight_escalation_delta(const fj_cpu_climber_t& fj_cpu) +{ + const i_t stall = fj_cpu.iters_since_infeasible_improve; + if (stall <= fj_weight_escalate_after) return 1; + const i_t steps = (stall - fj_weight_escalate_after) / fj_weight_escalate_after + 1; + return steps < fj_weight_escalate_max ? steps : fj_weight_escalate_max; } template @@ -941,11 +1317,15 @@ static void update_weights(fj_cpu_climber_t& fj_cpu) raft::random::PCGenerator rng(fj_cpu.settings.seed + fj_cpu.iterations, 0, 0); bool smoothing = rng.next_float() <= fj_cpu.settings.parameters.weight_smoothing_probability; + retire_var_best_moves(fj_cpu); + if (smoothing) { smooth_weights(fj_cpu); return; } + const i_t escalated_delta = weight_escalation_delta(fj_cpu); + for (auto cstr_idx : fj_cpu.violated_constraints) { f_t curr_incumbent_lhs = fj_cpu.h_lhs[cstr_idx]; f_t curr_lower_excess = @@ -963,7 +1343,7 @@ static void update_weights(fj_cpu_climber_t& fj_cpu) cuopt_assert(curr_excess_score < 0, "constraint not violated"); - i_t int_delta = 1.0; + i_t int_delta = escalated_delta; f_t delta = int_delta; f_t new_weight = old_weight + delta; @@ -977,17 +1357,23 @@ static void update_weights(fj_cpu_climber_t& fj_cpu) fj_cpu.max_weight = max(fj_cpu.max_weight, new_weight); } + // Only before this lane's first crossing: past that the search oscillates in and out of + // feasibility, and draining satisfied rows costs the objective phase. + if (fj_cpu.use_weight_donation && !fj_cpu.feasible_found) + donate_row_weight(fj_cpu, cstr_idx, delta, rng); + // Invalidate related cached move scores - auto [relvar_offset_begin, relvar_offset_end] = - range_for_constraint(fj_cpu, cstr_idx); - for (auto i = relvar_offset_begin; i < relvar_offset_end; i++) { - fj_cpu.cached_mtm_moves[i].first = 0; - } + ++fj_cpu.n_version_bumps_weights; + fj_cpu.h_cstr_version[cstr_idx]++; } if (fj_cpu.violated_constraints.empty()) { fj_cpu.h_objective_weight += 1; } } +// Bump and ceiling applied to the objective weight when a new incumbent lands. +constexpr double fj_obj_weight_incumbent_bump = 4.0; +constexpr double fj_obj_weight_incumbent_cap = 64.0; + template static void apply_move(fj_cpu_climber_t& fj_cpu, i_t var_idx, @@ -1022,79 +1408,127 @@ static void apply_move(fj_cpu_climber_t& fj_cpu, fj_cpu.n_variable_updates_window++; fj_cpu.unique_vars_accessed_window.insert(var_idx); - i_t previous_viol = fj_cpu.violated_constraints.size(); + const size_t nnz_touched = (size_t)(offset_end - offset_begin); + ++fj_cpu.n_moves_applied; + fj_cpu.apply_move_nnz += (int64_t)nnz_touched; + fj_cpu.n_version_bumps_apply += (int64_t)nnz_touched; + fj_cpu.h_reverse_constraints.byte_loads += nnz_touched * sizeof(i_t); + fj_cpu.h_reverse_coefficients.byte_loads += nnz_touched * sizeof(f_t); + fj_cpu.cached_cstr_bounds.byte_loads += nnz_touched * sizeof(std::pair); + fj_cpu.h_lhs.byte_loads += nnz_touched * sizeof(f_t); + fj_cpu.h_lhs.byte_stores += nnz_touched * sizeof(f_t); + fj_cpu.h_lhs_sumcomp.byte_loads += nnz_touched * sizeof(f_t); + fj_cpu.h_lhs_sumcomp.byte_stores += nnz_touched * sizeof(f_t); + fj_cpu.h_cstr_tolerance.byte_loads += nnz_touched * sizeof(f_t); + + const i_t* const rev_cstr = fj_cpu.view.pb.reverse_constraints.data(); + const f_t* const rev_coeff = fj_cpu.view.pb.reverse_coefficients.data(); + const std::pair* const cstr_bounds = fj_cpu.cached_cstr_bounds.data(); + const f_t* const row_tol = fj_cpu.h_cstr_tolerance.data(); + f_t* const row_lhs = fj_cpu.view.incumbent_lhs.data(); + f_t* const row_sumcomp = fj_cpu.view.incumbent_lhs_sumcomp.data(); for (auto i = offset_begin; i < offset_end; i++) { cuopt_assert(i < (i_t)fj_cpu.h_reverse_constraints.size(), ""); - auto [c_lb, c_ub] = fj_cpu.cached_cstr_bounds[i].get(); + const auto [c_lb, c_ub] = cstr_bounds[i]; - auto cstr_idx = fj_cpu.h_reverse_constraints[i]; - fj_cpu.unique_cstrs_accessed_window.insert(cstr_idx); - auto cstr_coeff = fj_cpu.h_reverse_coefficients[i]; + const i_t cstr_idx = rev_cstr[i]; + const f_t cstr_coeff = rev_coeff[i]; - f_t old_lhs = fj_cpu.h_lhs[cstr_idx]; + const f_t old_lhs = row_lhs[cstr_idx]; // Kahan compensated summation - f_t y = cstr_coeff * delta - fj_cpu.h_lhs_sumcomp[cstr_idx]; - f_t t = old_lhs + y; - fj_cpu.h_lhs_sumcomp[cstr_idx] = (t - old_lhs) - y; - fj_cpu.h_lhs[cstr_idx] = t; - f_t new_lhs = fj_cpu.h_lhs[cstr_idx]; - f_t old_cost = fj_cpu.view.excess_score(cstr_idx, old_lhs, c_lb, c_ub); - f_t new_cost = fj_cpu.view.excess_score(cstr_idx, new_lhs, c_lb, c_ub); - f_t cstr_tolerance = fj_cpu.view.get_corrected_tolerance(cstr_idx, c_lb, c_ub); + const f_t y = cstr_coeff * delta - row_sumcomp[cstr_idx]; + const f_t t = old_lhs + y; + const f_t new_sumcomp = (t - old_lhs) - y; + row_sumcomp[cstr_idx] = new_sumcomp; + row_lhs[cstr_idx] = t; + + const f_t old_cost = fj_cpu.view.excess_score(cstr_idx, old_lhs, c_lb, c_ub); + const f_t new_cost = fj_cpu.view.excess_score(cstr_idx, t, c_lb, c_ub); + const f_t cstr_tolerance = row_tol[cstr_idx]; // trigger early lhs recomputation if the sumcomp term gets too large // to avoid large numerical errors - if (fabs(fj_cpu.h_lhs_sumcomp[cstr_idx]) > BIGVAL_THRESHOLD) - fj_cpu.trigger_early_lhs_recomputation = true; + if (fabs(new_sumcomp) > BIGVAL_THRESHOLD) fj_cpu.trigger_early_lhs_recomputation = true; + + const bool was_violated = fj_cpu.violated_constraints.contains(cstr_idx); + const bool now_violated = new_cost < -cstr_tolerance; + + // total_violations sums the excess over the violated set alone, so a row crossing the boundary + // contributes its whole cost rather than a difference. Kahan compensated, as h_lhs is: this is + // now the only place the total is maintained between refreshes. + const f_t viol_delta = + (now_violated ? new_cost : f_t{0}) - (was_violated ? old_cost : f_t{0}); + if (viol_delta != f_t{0}) { + const f_t viol_old = fj_cpu.total_violations; + const f_t viol_y = viol_delta - fj_cpu.total_violations_sumcomp; + const f_t viol_t = viol_old + viol_y; + fj_cpu.total_violations_sumcomp = (viol_t - viol_old) - viol_y; + fj_cpu.total_violations = viol_t; + } - if (new_cost < -cstr_tolerance && !fj_cpu.violated_constraints.count(cstr_idx)) { + if (now_violated && !was_violated) { fj_cpu.violated_constraints.insert(cstr_idx); - cuopt_assert(fj_cpu.satisfied_constraints.count(cstr_idx) == 1, ""); - fj_cpu.satisfied_constraints.erase(cstr_idx); - } else if (!(new_cost < -cstr_tolerance) && fj_cpu.violated_constraints.count(cstr_idx)) { - cuopt_assert(fj_cpu.satisfied_constraints.count(cstr_idx) == 0, ""); - fj_cpu.violated_constraints.erase(cstr_idx); + cuopt_assert(fj_cpu.satisfied_constraints.contains(cstr_idx), ""); + fj_cpu.satisfied_constraints.remove(cstr_idx); + } else if (!now_violated && was_violated) { + cuopt_assert(!fj_cpu.satisfied_constraints.contains(cstr_idx), ""); + fj_cpu.violated_constraints.remove(cstr_idx); fj_cpu.satisfied_constraints.insert(cstr_idx); } cuopt_assert(isfinite(delta), "delta should be finite"); - cuopt_assert(isfinite(fj_cpu.h_lhs[cstr_idx]), "assignment should be finite"); + cuopt_assert(isfinite(t), "assignment should be finite"); // Invalidate related cached move scores - auto [relvar_offset_begin, relvar_offset_end] = - range_for_constraint(fj_cpu, cstr_idx); - for (auto i = relvar_offset_begin; i < relvar_offset_end; i++) { - fj_cpu.cached_mtm_moves[i].first = 0; - } - } - - if (previous_viol > 0 && fj_cpu.violated_constraints.empty()) { - fj_cpu.last_feasible_entrance_iter = fj_cpu.iterations; + fj_cpu.h_cstr_version[cstr_idx]++; } // update the assignment and objective proper fj_cpu.h_assignment[var_idx] = new_val; - fj_cpu.h_incumbent_objective += fj_cpu.h_obj_coeffs[var_idx] * delta; - if (fj_cpu.h_incumbent_objective < fj_cpu.h_best_objective && - fj_cpu.violated_constraints.empty()) { - // recompute the LHS values to cancel out accumulation errors, then check if feasibility remains - recompute_lhs(fj_cpu); + // The clamp above passes a NaN straight through, and every comparison against one is false. + cuopt_assert(fj_cpu.view.pb.check_variable_within_bounds(var_idx, new_val), + "apply_move left the variable bounds"); + + // Kahan compensated summation, as for h_lhs. The incumbent objective is reported as-is, so it + // cannot carry the drift of a long uncompensated chain of deltas. + const f_t obj_old = fj_cpu.h_incumbent_objective; + const f_t obj_y = fj_cpu.h_obj_coeffs[var_idx] * delta - fj_cpu.h_objective_sumcomp; + const f_t obj_t = obj_old + obj_y; + fj_cpu.h_objective_sumcomp = (obj_t - obj_old) - obj_y; + fj_cpu.h_incumbent_objective = obj_t; - if (fj_cpu.violated_constraints.empty() && check_variable_feasibility(fj_cpu)) { - cuopt_assert(fj_cpu.satisfied_constraints.size() == fj_cpu.view.pb.n_constraints, ""); - fj_cpu.h_best_objective = - fj_cpu.h_incumbent_objective - fj_cpu.settings.parameters.breakthrough_move_epsilon; - fj_cpu.h_best_assignment = fj_cpu.h_assignment; - fj_cpu.iterations_since_best = 0; - CUOPT_LOG_TRACE( - "%sCPUFJ: new best objective: %g", fj_cpu.log_prefix.c_str(), fj_cpu.h_incumbent_objective); - if (fj_cpu.improvement_callback) { - double current_work_units = fj_cpu.work_units_elapsed.load(std::memory_order_acquire); - fj_cpu.improvement_callback( - fj_cpu.h_incumbent_objective, fj_cpu.h_assignment, current_work_units); - } - fj_cpu.feasible_found = true; + if (fj_cpu.h_incumbent_objective < fj_cpu.h_best_objective && + fj_cpu.violated_constraints.empty() && check_variable_feasibility(fj_cpu)) { + cuopt_assert(fj_cpu.satisfied_constraints.size() == fj_cpu.view.pb.n_constraints, ""); + fj_cpu.h_best_objective = + fj_cpu.h_incumbent_objective - fj_cpu.settings.parameters.breakthrough_move_epsilon; + fj_cpu.h_best_assignment = fj_cpu.h_assignment; + fj_cpu.iterations_since_best = 0; + // DEBUG, and reporting the stored best rather than the pre-epsilon incumbent, + // so it matches the binary path and the end-of-solve incumbent audit. + CUOPT_LOG_DEBUG("%sCPUFJ new incumbent: objective %.17g", + fj_cpu.log_prefix.c_str(), + fj_cpu.h_best_objective); + if (fj_cpu.improvement_callback) { + double current_work_units = fj_cpu.work_units_elapsed.load(std::memory_order_acquire); + fj_cpu.improvement_callback( + fj_cpu.h_incumbent_objective, fj_cpu.h_assignment, current_work_units); + } + fj_cpu.feasible_found = true; + // The true objective of the assignment, not the epsilon-reduced threshold stored above, so + // another lane comparing against it is not misled into adopting something no better. + if (fj_cpu.shared_incumbent) { + fj_cpu.shared_incumbent->publish(fj_cpu.h_incumbent_objective, fj_cpu.h_assignment); + } + // Counteract the smooth_weights decay for a lane that is actively improving, and hold the + // weight at a scale where base_feas_sum still registers against it. + if (fj_cpu.h_objective_weight > 0) { + fj_cpu.h_objective_weight = + min((f_t)fj_obj_weight_incumbent_cap, + fj_cpu.h_objective_weight + (f_t)fj_obj_weight_incumbent_bump); + // The weight enters every score, and row versions cannot see it move. + retire_var_best_moves(fj_cpu); } } @@ -1113,9 +1547,46 @@ static void apply_move(fj_cpu_climber_t& fj_cpu, // CUOPT_LOG_TRACE("CPU: tabu noinc_until: %d\n", fj_cpu.h_tabu_noinc_until[var_idx]); } - std::fill(fj_cpu.flip_move_computed.begin(), fj_cpu.flip_move_computed.end(), false); - std::fill(fj_cpu.var_bitmap.begin(), fj_cpu.var_bitmap.end(), false); - fj_cpu.iter_mtm_vars.clear(); + ++fj_cpu.flip_move_epoch; +} + +// Tightest value the rows of a certified epigraph variable imply. Satisfies all of them at once and +// leaves the objective as small as they allow, which is why it is sound from an infeasible point. +template +static f_t project_epigraph_variable(fj_cpu_climber_t& fj_cpu, i_t var_idx) +{ + cuopt_assert(fj_cpu.epigraph_push[var_idx] != 0, "variable is not a certified epigraph variable"); + const bool push_up = fj_cpu.epigraph_push[var_idx] > 0; + const f_t current = fj_cpu.h_assignment[var_idx]; + const auto bounds = fj_cpu.h_var_bounds[var_idx].get(); + f_t target = push_up ? get_lower(bounds) : get_upper(bounds); + + auto [offset_begin, offset_end] = reverse_range_for_var(fj_cpu, var_idx); + const size_t nnz_read = (size_t)(offset_end - offset_begin); + fj_cpu.h_reverse_constraints.byte_loads += nnz_read * sizeof(i_t); + fj_cpu.h_reverse_coefficients.byte_loads += nnz_read * sizeof(f_t); + fj_cpu.cached_cstr_bounds.byte_loads += nnz_read * sizeof(std::pair); + fj_cpu.h_lhs.byte_loads += nnz_read * sizeof(f_t); + + const i_t* const rev_cstr = fj_cpu.view.pb.reverse_constraints.data(); + const f_t* const rev_coeff = fj_cpu.view.pb.reverse_coefficients.data(); + const f_t* const row_lhs = fj_cpu.view.incumbent_lhs.data(); + const std::pair* const cstr_bounds = fj_cpu.cached_cstr_bounds.data(); + + for (i_t p = offset_begin; p < offset_end; ++p) { + const f_t coeff = rev_coeff[p]; + if (coeff == f_t{0}) continue; + const auto [c_lb, c_ub] = cstr_bounds[p]; + const f_t rest = row_lhs[rev_cstr[p]] - coeff * current; + const f_t bound = ((coeff > f_t{0}) == push_up) ? c_lb : c_ub; + const f_t implied = (bound - rest) / coeff; + if (!isfinite(implied)) continue; + target = push_up ? max(target, implied) : min(target, implied); + } + + target = std::min(std::max(target, get_lower(bounds)), get_upper(bounds)); + cuopt_assert(isfinite(target), "epigraph projection is not finite"); + return target; } template @@ -1129,36 +1600,49 @@ static thrust::tuple find_mtm_move( fj_move_t best_move = fj_move_t{-1, 0}; fj_staged_score_t best_score = fj_staged_score_t::invalid(); - // collect all the variables that are involved in the target constraints + ++fj_cpu.n_mtm_calls; + + // Each row contributes at most its share of the sampling budget. The gate below sits inside the + // walk, so an uncapped wide row is walked in full whatever the budget says. + const i_t per_row_cap = + std::max(1, fj_cpu.nnz_samples / std::max(1, (i_t)target_cstrs.size())); + + i_t entries = 0; for (size_t cstr_idx : target_cstrs) { auto [offset_begin, offset_end] = range_for_constraint(fj_cpu, cstr_idx); - for (auto i = offset_begin; i < offset_end; i++) { - i_t var_idx = fj_cpu.h_variables[i]; - if (fj_cpu.var_bitmap[var_idx]) continue; - fj_cpu.iter_mtm_vars.push_back(var_idx); - fj_cpu.var_bitmap[var_idx] = true; - } - } - // estimate the amount of nnzs to consider - i_t nnz_sum = 0; - for (auto var_idx : fj_cpu.iter_mtm_vars) { - auto [offset_begin, offset_end] = reverse_range_for_var(fj_cpu, var_idx); - nnz_sum += offset_end - offset_begin; + const i_t width = offset_end - offset_begin; + entries += std::min(width, per_row_cap); + fj_cpu.mtm_entries_capped += (int64_t)std::max(0, width - per_row_cap); } + fj_cpu.mtm_row_entries += (int64_t)entries; + + // The exact sum over the candidate variables costs one random offset read each to set a single + // sampling rate. The mean reverse degree estimates it in constant time. + const f_t mean_reverse_degree = + (f_t)fj_cpu.h_coefficients.size() / (f_t)std::max(1, fj_cpu.view.pb.n_variables); + const f_t nnz_sum = (f_t)entries * mean_reverse_degree; f_t nnz_pick_probability = 1; - if (nnz_sum > fj_cpu.nnz_samples) nnz_pick_probability = (f_t)fj_cpu.nnz_samples / nnz_sum; + if (nnz_sum > (f_t)fj_cpu.nnz_samples) nnz_pick_probability = (f_t)fj_cpu.nnz_samples / nnz_sum; for (size_t cstr_idx : target_cstrs) { - auto c_lb = fj_cpu.h_cstr_lb[cstr_idx]; - auto c_ub = fj_cpu.h_cstr_ub[cstr_idx]; - f_t cstr_tol = fj_cpu.view.get_corrected_tolerance(cstr_idx, c_lb, c_ub); + f_t cstr_tol = fj_cpu.h_cstr_tolerance[cstr_idx]; cuopt_assert(cstr_idx < fj_cpu.h_cstr_lb.size(), "cstr_idx is out of bounds"); auto [offset_begin, offset_end] = range_for_constraint(fj_cpu, cstr_idx); - for (auto i = offset_begin; i < offset_end; i++) { + const i_t width = offset_end - offset_begin; + const i_t visit = std::min(width, per_row_cap); + const i_t start = visit == width + ? offset_begin + : offset_begin + (i_t)(rng.next_u32() % (uint32_t)width); + for (i_t q = 0, i = start; q < visit; + ++q, i = (i + 1 == offset_end ? offset_begin : i + 1)) { // early cached check - if (auto& cached_move = fj_cpu.cached_mtm_moves[i]; cached_move.first != 0) { + cuopt_assert(fj_cpu.cached_mtm_moves_version[i] <= fj_cpu.h_cstr_version[cstr_idx], + "cached move newer than its constraint"); + if (auto& cached_move = fj_cpu.cached_mtm_moves[i]; + cached_move.first != 0 && + fj_cpu.cached_mtm_moves_version[i] == fj_cpu.h_cstr_version[cstr_idx]) { if (best_score < cached_move.second) { auto var_idx = fj_cpu.h_variables[i]; if (check_variable_within_bounds( @@ -1185,23 +1669,23 @@ static thrust::tuple find_mtm_move( // Special case for binary variables if (fj_cpu.h_is_binary_variable[var_idx]) { - if (fj_cpu.flip_move_computed[var_idx]) continue; - fj_cpu.flip_move_computed[var_idx] = true; - new_val = 1 - val; + if (fj_cpu.flip_move_stamp[var_idx] == fj_cpu.flip_move_epoch) continue; + fj_cpu.flip_move_stamp[var_idx] = fj_cpu.flip_move_epoch; + new_val = 1 - val; } else { auto cstr_coeff = fj_cpu.h_coefficients[i]; f_t c_lb = fj_cpu.h_cstr_lb[cstr_idx]; f_t c_ub = fj_cpu.h_cstr_ub[cstr_idx]; auto [delta, sign, slack, cstr_tolerance] = - get_mtm_for_constraint(fj_cpu.view, - var_idx, + get_mtm_for_constraint(var_idx, cstr_idx, cstr_coeff, c_lb, c_ub, fj_cpu.h_assignment, - fj_cpu.h_lhs); + fj_cpu.h_lhs, + cstr_tol); if (is_integer_var(fj_cpu, var_idx)) { new_val = cstr_coeff * sign > 0 ? floor(val + delta + fj_cpu.view.pb.tolerances.integrality_tolerance) @@ -1228,12 +1712,14 @@ static thrust::tuple find_mtm_move( cuopt_assert(move.var_idx < fj_cpu.h_assignment.size(), "move.var_idx is out of bounds"); cuopt_assert(move.var_idx >= 0, "move.var_idx is not positive"); - auto [score, infeasibility] = compute_score(fj_cpu, var_idx, delta); - fj_cpu.cached_mtm_moves[i] = std::make_pair(delta, score); + auto [score, infeasibility] = compute_score(fj_cpu, var_idx, delta); + fj_cpu.cached_mtm_moves[i] = std::make_pair(delta, score); + fj_cpu.cached_mtm_moves_version[i] = fj_cpu.h_cstr_version[cstr_idx]; fj_cpu.miss_count++; // reject this move if it would increase the target variable to a numerically unstable value if (fj_cpu.view.move_numerically_stable( val, new_val, infeasibility, fj_cpu.total_violations)) { + record_var_best_move(fj_cpu, var_idx, score, delta); if (best_score < score) { best_score = score; best_move = move; @@ -1247,6 +1733,7 @@ static thrust::tuple find_mtm_move( fj_cpu.h_best_objective < std::numeric_limits::infinity() && fj_cpu.h_incumbent_objective >= fj_cpu.h_best_objective + fj_cpu.settings.parameters.breakthrough_move_epsilon) { + cuopt_func_call(audit_breakthrough_inputs(fj_cpu)); for (auto var_idx : fj_cpu.h_objective_vars) { f_t old_val = fj_cpu.h_assignment[var_idx]; f_t new_val = get_breakthrough_move(fj_cpu.view, var_idx); @@ -1269,6 +1756,7 @@ static thrust::tuple find_mtm_move( if (fj_cpu.view.move_numerically_stable( old_val, new_val, infeasibility, fj_cpu.total_violations)) { + record_var_best_move(fj_cpu, var_idx, score, delta); if (best_score < score) { best_score = score; best_move = move; @@ -1280,6 +1768,27 @@ static thrust::tuple find_mtm_move( return thrust::make_tuple(best_move, best_score); } +template +static void sample_with_replacement(const host_contiguous_set_t& pool, + i_t sample_size, + uint64_t seed, + std::vector& out) +{ + cuopt_assert(sample_size > 0, "invalid sample size"); + out.clear(); + const i_t pool_size = pool.size(); + if (pool_size == 0) { return; } + if (pool_size <= sample_size) { + out.assign(pool.begin(), pool.end()); + return; + } + out.reserve(sample_size); + cuopt::pcgenerator_t rng(seed); + for (i_t i = 0; i < sample_size; ++i) { + out.push_back(pool.contents[rng.next_u32() % (uint32_t)pool_size]); + } +} + template static thrust::tuple find_mtm_move_viol( fj_cpu_climber_t& fj_cpu, i_t sample_size = 100, bool localmin = false) @@ -1288,12 +1797,10 @@ static thrust::tuple find_mtm_move_viol( CPUFJ_NVTX_RANGE("CPUFJ::find_mtm_move_viol"); std::vector sampled_cstrs; - sampled_cstrs.reserve(sample_size); - std::sample(fj_cpu.violated_constraints.begin(), - fj_cpu.violated_constraints.end(), - std::back_inserter(sampled_cstrs), - sample_size, - fj_cpu.rng); + sample_with_replacement(fj_cpu.violated_constraints, + sample_size, + fj_cpu.settings.seed + fj_cpu.iterations, + sampled_cstrs); return find_mtm_move(fj_cpu, sampled_cstrs, localmin); } @@ -1306,12 +1813,10 @@ static thrust::tuple find_mtm_move_sat( CPUFJ_NVTX_RANGE("CPUFJ::find_mtm_move_sat"); std::vector sampled_cstrs; - sampled_cstrs.reserve(sample_size); - std::sample(fj_cpu.satisfied_constraints.begin(), - fj_cpu.satisfied_constraints.end(), - std::back_inserter(sampled_cstrs), - sample_size, - fj_cpu.rng); + sample_with_replacement(fj_cpu.satisfied_constraints, + sample_size, + fj_cpu.settings.seed + fj_cpu.iterations, + sampled_cstrs); return find_mtm_move(fj_cpu, sampled_cstrs); } @@ -1321,6 +1826,7 @@ static void recompute_lhs(fj_cpu_climber_t& fj_cpu) { CPUFJ_NVTX_RANGE("CPUFJ::recompute_lhs"); cuopt_assert(fj_cpu.h_lhs.size() == fj_cpu.view.pb.n_constraints, "h_lhs size mismatch"); + ++fj_cpu.n_lhs_recompute_total; // clamp to var bounds - defensive; apply_move should already have clamped appropriately for (i_t var_idx = 0; var_idx < fj_cpu.view.pb.n_variables; ++var_idx) { @@ -1331,11 +1837,10 @@ static void recompute_lhs(fj_cpu_climber_t& fj_cpu) fj_cpu.violated_constraints.clear(); fj_cpu.satisfied_constraints.clear(); - fj_cpu.total_violations = 0; + fj_cpu.total_violations = 0; + fj_cpu.total_violations_sumcomp = 0; for (i_t cstr_idx = 0; cstr_idx < fj_cpu.view.pb.n_constraints; ++cstr_idx) { auto [offset_begin, offset_end] = range_for_constraint(fj_cpu, cstr_idx); - auto c_lb = fj_cpu.h_cstr_lb[cstr_idx]; - auto c_ub = fj_cpu.h_cstr_ub[cstr_idx]; auto delta_it = thrust::make_transform_iterator(thrust::make_counting_iterator(0), [&fj_cpu](i_t j) { return fj_cpu.h_coefficients[j] * fj_cpu.h_assignment[fj_cpu.h_variables[j]]; @@ -1344,7 +1849,7 @@ static void recompute_lhs(fj_cpu_climber_t& fj_cpu) fj_kahan_babushka_neumaier_sum(delta_it + offset_begin, delta_it + offset_end); fj_cpu.h_lhs_sumcomp[cstr_idx] = 0; - f_t cstr_tolerance = fj_cpu.view.get_corrected_tolerance(cstr_idx, c_lb, c_ub); + f_t cstr_tolerance = fj_cpu.h_cstr_tolerance[cstr_idx]; f_t new_cost = fj_cpu.view.excess_score(cstr_idx, fj_cpu.h_lhs[cstr_idx]); if (new_cost < -cstr_tolerance) { fj_cpu.violated_constraints.insert(cstr_idx); @@ -1357,6 +1862,130 @@ static void recompute_lhs(fj_cpu_climber_t& fj_cpu) // compute incumbent objective fj_cpu.h_incumbent_objective = thrust::inner_product( fj_cpu.h_assignment.begin(), fj_cpu.h_assignment.end(), fj_cpu.h_obj_coeffs.begin(), 0.); + fj_cpu.h_objective_sumcomp = 0; +} + + +// Candidate draws per 2-opt lift search. +constexpr int32_t fj_2opt_candidates = 32; + +// True when flipping both variables leaves every row they touch satisfied. Both reverse ranges are +// row-ascending, so a merge handles rows containing both variables with their joint delta. +template +static bool paired_flip_keeps_feasible( + fj_cpu_climber_t& fj_cpu, i_t var1, f_t delta1, i_t var2, f_t delta2) +{ + const auto range1 = reverse_range_for_var(fj_cpu, var1); + const auto range2 = reverse_range_for_var(fj_cpu, var2); + i_t i = range1.first, ie = range1.second; + i_t j = range2.first, je = range2.second; + + while (i < ie || j < je) { + const i_t r1 = i < ie ? (i_t)fj_cpu.h_reverse_constraints[i] : std::numeric_limits::max(); + const i_t r2 = j < je ? (i_t)fj_cpu.h_reverse_constraints[j] : std::numeric_limits::max(); + const i_t r = r1 < r2 ? r1 : r2; + + f_t change = 0; + f_t c_lb = 0; + f_t c_ub = 0; + if (r1 == r) { + auto [lb, ub] = fj_cpu.cached_cstr_bounds[i].get(); + c_lb = lb; + c_ub = ub; + change += (f_t)fj_cpu.h_reverse_coefficients[i] * delta1; + ++i; + } + if (r2 == r) { + auto [lb, ub] = fj_cpu.cached_cstr_bounds[j].get(); + c_lb = lb; + c_ub = ub; + change += (f_t)fj_cpu.h_reverse_coefficients[j] * delta2; + ++j; + } + + const f_t new_lhs = fj_cpu.h_lhs[r] + (change - fj_cpu.h_lhs_sumcomp[r]); + if (fj_cpu.view.excess_score(r, new_lhs, c_lb, c_ub) < -(f_t)fj_cpu.h_cstr_tolerance[r]) + return false; + } + return true; +} + +template +static thrust::tuple find_lift_2opt_move( + fj_cpu_climber_t& fj_cpu) +{ + timing_raii_t timer(fj_cpu.find_lift_move_times); + CPUFJ_NVTX_RANGE("CPUFJ::find_lift_2opt_move"); + cuopt_assert(fj_cpu.violated_constraints.empty(), "lift moves require a feasible incumbent"); + + fj_move_t best_first = fj_move_t{-1, 0}; + fj_move_t best_second = fj_move_t{-1, 0}; + fj_staged_score_t best_score = fj_staged_score_t::zero(); + f_t best_improvement = 0; + + const i_t n_obj = (i_t)fj_cpu.h_objective_vars.size(); + if (n_obj == 0) return thrust::make_tuple(best_first, best_second, best_score); + + raft::random::PCGenerator rng(fj_cpu.settings.seed + fj_cpu.iterations, 0, 0); + const i_t n_draws = n_obj < fj_2opt_candidates ? n_obj : fj_2opt_candidates; + + for (i_t t = 0; t < n_draws; ++t) { + const i_t var1 = fj_cpu.h_objective_vars[rng.next_u32() % (uint32_t)n_obj]; + if (!fj_cpu.h_is_binary_variable[var1]) continue; + + const f_t coeff1 = fj_cpu.h_obj_coeffs[var1]; + const f_t val1 = fj_cpu.h_assignment[var1]; + const f_t delta1 = round(1.0 - 2 * val1); + if (delta1 * coeff1 >= 0) continue; + if (tabu_check(fj_cpu, var1, delta1)) continue; + + // Breaking nothing is the single-flip lift's job; breaking several rows cannot be repaired by + // one companion. + const auto range1 = reverse_range_for_var(fj_cpu, var1); + i_t broken = -1; + bool multiple = false; + for (i_t k = range1.first; k < range1.second && !multiple; ++k) { + auto [c_lb, c_ub] = fj_cpu.cached_cstr_bounds[k].get(); + const i_t r = fj_cpu.h_reverse_constraints[k]; + const f_t new_lhs = fj_cpu.h_lhs[r] + ((f_t)fj_cpu.h_reverse_coefficients[k] * delta1 - + fj_cpu.h_lhs_sumcomp[r]); + if (fj_cpu.view.excess_score(r, new_lhs, c_lb, c_ub) < -(f_t)fj_cpu.h_cstr_tolerance[r]) { + if (broken >= 0) + multiple = true; + else + broken = r; + } + } + if (multiple || broken < 0) continue; + + const auto row = range_for_constraint(fj_cpu, broken); + for (i_t k = row.first; k < row.second; ++k) { + const i_t var2 = fj_cpu.h_variables[k]; + if (var2 == var1) continue; + if (!fj_cpu.h_is_binary_variable[var2]) continue; + + const f_t coeff2 = fj_cpu.h_obj_coeffs[var2]; + const f_t val2 = fj_cpu.h_assignment[var2]; + const f_t delta2 = round(1.0 - 2 * val2); + const f_t combined = delta1 * coeff1 + delta2 * coeff2; + if (combined >= 0) continue; + if (tabu_check(fj_cpu, var2, delta2)) continue; + if (!paired_flip_keeps_feasible(fj_cpu, var1, delta1, var2, delta2)) continue; + + // Both lift operators rank on the objective gain in its own units: the score quantization + // used elsewhere counts weights, so rounding a gain below 0.5 into it discards the move. + const f_t improvement = -combined; + if (improvement > best_improvement) { + best_improvement = improvement; + best_score.base = 1; // sign only, never compared against another operator's score + best_first = fj_move_t{var1, delta1}; + best_second = fj_move_t{var2, delta2}; + } + } + } + cuopt_assert((best_first.var_idx < 0) == (best_improvement <= 0), + "pair and score must agree on whether a move was found"); + return thrust::make_tuple(best_first, best_second, best_score); } template @@ -1368,6 +1997,7 @@ static thrust::tuple find_lift_move( fj_move_t best_move = fj_move_t{-1, 0}; fj_staged_score_t best_score = fj_staged_score_t::zero(); + f_t best_improvement = 0; for (auto var_idx : fj_cpu.h_objective_vars) { cuopt_assert(var_idx < fj_cpu.h_obj_coeffs.size(), "var_idx is out of bounds"); @@ -1385,6 +2015,40 @@ static thrust::tuple find_lift_move( delta = round(1.0 - 2 * val); // flip move wouldn't improve if (delta * obj_coeff >= 0) continue; + + auto [offset_begin, offset_end] = reverse_range_for_var(fj_cpu, var_idx); + + const i_t* const rev_cstr = fj_cpu.view.pb.reverse_constraints.data(); + const f_t* const rev_coeff = fj_cpu.view.pb.reverse_coefficients.data(); + const f_t* const row_lhs = fj_cpu.view.incumbent_lhs.data(); + const f_t* const row_sumcomp = fj_cpu.view.incumbent_lhs_sumcomp.data(); + const f_t* const row_tol = fj_cpu.h_cstr_tolerance.data(); + const std::pair* const cstr_bounds = fj_cpu.cached_cstr_bounds.data(); + + bool breaks_a_row = false; + i_t scanned = 0; + for (i_t j = offset_begin; j < offset_end; ++j) { + ++scanned; + const auto [c_lb, c_ub] = cstr_bounds[j]; + const i_t cstr_idx = rev_cstr[j]; + const f_t cstr_coeff = rev_coeff[j]; + const f_t lhs = row_lhs[cstr_idx]; + const f_t sumcomp = row_sumcomp[cstr_idx]; + const f_t new_lhs = lhs + (cstr_coeff * delta - sumcomp); + if (fj_cpu.view.excess_score(cstr_idx, new_lhs, c_lb, c_ub) < -row_tol[cstr_idx]) { + breaks_a_row = true; + break; + } + } + + const size_t nnz_scanned = (size_t)scanned; + fj_cpu.h_reverse_constraints.byte_loads += nnz_scanned * sizeof(i_t); + fj_cpu.h_reverse_coefficients.byte_loads += nnz_scanned * sizeof(f_t); + fj_cpu.cached_cstr_bounds.byte_loads += nnz_scanned * sizeof(std::pair); + fj_cpu.h_lhs.byte_loads += nnz_scanned * sizeof(f_t); + fj_cpu.h_lhs_sumcomp.byte_loads += nnz_scanned * sizeof(f_t); + + if (breaks_a_row) continue; } else { f_t lfd_lb = get_lower(fj_cpu.h_var_bounds[var_idx].get()) - val; f_t lfd_ub = get_upper(fj_cpu.h_var_bounds[var_idx].get()) - val; @@ -1394,7 +2058,7 @@ static thrust::tuple find_lift_move( auto cstr_coeff = fj_cpu.h_reverse_coefficients[j]; f_t c_lb = fj_cpu.h_cstr_lb[cstr_idx]; f_t c_ub = fj_cpu.h_cstr_ub[cstr_idx]; - f_t cstr_tolerance = fj_cpu.view.get_corrected_tolerance(cstr_idx, c_lb, c_ub); + f_t cstr_tolerance = fj_cpu.h_cstr_tolerance[cstr_idx]; cuopt_assert(c_lb <= c_ub, "invalid bounds"); cuopt_assert(fj_cpu.view.cstr_satisfied(cstr_idx, fj_cpu.h_lhs[cstr_idx]), "cstr should be satisfied"); @@ -1454,51 +2118,171 @@ static thrust::tuple find_lift_move( cuopt_assert(delta * obj_coeff < 0, "lift move doesn't improve the objective!"); - // get the score - auto move = fj_move_t{var_idx, delta}; - fj_staged_score_t score = fj_staged_score_t::zero(); - f_t obj_score = -1 * obj_coeff * delta; // negated to turn this into a positive score - score.base = round(obj_score); - - if (best_score < score) { - best_score = score; - best_move = move; + const f_t improvement = -obj_coeff * delta; + if (improvement > best_improvement) { + best_improvement = improvement; + best_score.base = 1; + best_move = fj_move_t{var_idx, delta}; } } + cuopt_assert((best_move.var_idx < 0) == (best_improvement <= 0), + "move and score must agree on whether a move was found"); return thrust::make_tuple(best_move, best_score); } +// Draws a uniform in-bounds value, rounded and re-clamped for integer variables. +template +static void randomize_variable(fj_cpu_climber_t& fj_cpu, + i_t var_idx, + raft::random::PCGenerator& rng) +{ + f_t lb = std::max(get_lower(fj_cpu.h_var_bounds[var_idx].get()), -1e7); + f_t ub = std::min(get_upper(fj_cpu.h_var_bounds[var_idx].get()), 1e7); + f_t val = lb + (ub - lb) * rng.next_double(); + if (is_integer_var(fj_cpu, var_idx)) { + lb = std::ceil(lb); + ub = std::floor(ub); + val = std::round(val); + val = std::min(std::max(val, lb), ub); + } + + cuopt_assert((check_variable_within_bounds(fj_cpu, var_idx, val)), + "value is out of bounds"); + fj_cpu.h_assignment[var_idx] = val; +} + template static void perturb(fj_cpu_climber_t& fj_cpu) { CPUFJ_NVTX_RANGE("CPUFJ::perturb"); + if (fj_cpu.feasible_found) { + cuopt_assert(fj_cpu.h_assignment.size() == fj_cpu.h_best_assignment.size(), + "incumbent_assignment span would be invalidated"); + fj_cpu.h_assignment = fj_cpu.h_best_assignment; + if (fj_cpu.shared_incumbent) { + fj_cpu.shared_incumbent->adopt(fj_cpu.h_best_objective, fj_cpu.h_assignment); + cuopt_func_call(audit_assignment_bounds(fj_cpu, "shared adopt")); + } + } + // select N variables, assign them a random value between their bounds std::vector sampled_vars; std::sample(fj_cpu.h_objective_vars.begin(), fj_cpu.h_objective_vars.end(), std::back_inserter(sampled_vars), - 2, + std::max(1, fj_cpu.perturb_vars), fj_cpu.rng); raft::random::PCGenerator rng(fj_cpu.settings.seed + fj_cpu.iterations, 0, 0); - for (auto var_idx : sampled_vars) { - f_t lb = std::max(get_lower(fj_cpu.h_var_bounds[var_idx].get()), -1e7); - f_t ub = std::min(get_upper(fj_cpu.h_var_bounds[var_idx].get()), 1e7); - f_t val = lb + (ub - lb) * rng.next_double(); - if (is_integer_var(fj_cpu, var_idx)) { - lb = std::ceil(lb); - ub = std::floor(ub); - val = std::round(val); - val = std::min(std::max(val, lb), ub); + for (auto var_idx : sampled_vars) + randomize_variable(fj_cpu, var_idx, rng); + + ++fj_cpu.n_lhs_recompute_perturb; + recompute_lhs(fj_cpu); + retire_var_best_moves(fj_cpu); +} + +template +static void reset_infeasible_checkpoint(fj_cpu_climber_t& fj_cpu) +{ + fj_cpu.h_best_infeasible_assignment.clear(); + fj_cpu.best_infeasible_severity = std::numeric_limits::infinity(); + fj_cpu.checkpoint_severity = std::numeric_limits::infinity(); + fj_cpu.iters_since_infeasible_improve = 0; +} + +template +static void invalidate_mtm_cache(fj_cpu_climber_t& fj_cpu) +{ + ++fj_cpu.n_mtm_cache_invalidations; + for (size_t c = 0; c < fj_cpu.h_cstr_version.size(); ++c) + fj_cpu.h_cstr_version[c]++; +} + +template +static void restart_from_infeasible_checkpoint(fj_cpu_climber_t& fj_cpu) +{ + cuopt_assert(fj_cpu.h_assignment.size() == fj_cpu.h_best_infeasible_assignment.size(), + "incumbent_assignment span would be invalidated"); + fj_cpu.h_assignment = fj_cpu.h_best_infeasible_assignment; + ++fj_cpu.n_lhs_recompute_restart; + recompute_lhs(fj_cpu); + invalidate_mtm_cache(fj_cpu); + cuopt_func_call(audit_assignment_bounds(fj_cpu, "checkpoint restore")); +} + +// Nonzeros per extra restart window, the cap on that, and how many windows a lane waits. +constexpr int32_t fj_restart_window_nnz_scale = 100000; +constexpr int32_t fj_restart_window_scale_max = 4; +constexpr int32_t fj_restart_window_multiple = 4; + +template +static void track_infeasible_checkpoint(fj_cpu_climber_t& fj_cpu) +{ + CPUFJ_NVTX_RANGE("CPUFJ::track_infeasible_checkpoint"); + if (fj_cpu.violated_constraints.empty()) { + reset_infeasible_checkpoint(fj_cpu); + return; + } + + const f_t severity = -fj_cpu.total_violations; + cuopt_assert(severity >= 0, "violation severity should be positive or zero"); + + if (severity < fj_cpu.best_infeasible_severity) { + fj_cpu.best_infeasible_severity = severity; + fj_cpu.iters_since_infeasible_improve = 0; + fj_cpu.restores_since_improvement = 0; + if (severity < fj_cpu.checkpoint_severity * fj_cpu.infeasible_checkpoint_refresh_ratio) { + fj_cpu.h_best_infeasible_assignment = fj_cpu.h_assignment; + fj_cpu.checkpoint_severity = severity; + ++fj_cpu.n_checkpoint_snapshots; } + return; + } - cuopt_assert((check_variable_within_bounds(fj_cpu, var_idx, val)), - "value is out of bounds"); - fj_cpu.h_assignment[var_idx] = val; + // A lane that has never crossed and has exhausted its restores abandons the basin outright. + if (!fj_cpu.feasible_found) { + const i_t nnz_scale = + 1 + (i_t)fj_cpu.h_coefficients.size() / fj_restart_window_nnz_scale; + const i_t capped = nnz_scale < fj_restart_window_scale_max ? nnz_scale + : fj_restart_window_scale_max; + if (fj_cpu.iters_since_infeasible_improve >= + fj_restart_window_multiple * fj_cpu.infeasible_restart_window * capped && + fj_cpu.restores_since_improvement >= fj_cpu.infeasible_restart_max_streak) { + raft::random::PCGenerator rng(fj_cpu.settings.seed + fj_cpu.iterations, 0, 0); + for (i_t var_idx = 0; var_idx < fj_cpu.view.pb.n_variables; ++var_idx) + randomize_variable(fj_cpu, var_idx, rng); + + ++fj_cpu.n_lhs_recompute_restart; + recompute_lhs(fj_cpu); + invalidate_mtm_cache(fj_cpu); + reset_infeasible_checkpoint(fj_cpu); + fj_cpu.restores_since_improvement = 0; + cuopt_func_call(audit_assignment_bounds(fj_cpu, "randomized restart")); + + CUOPT_LOG_DEBUG("%sCPUFJ randomized restart at iteration %d", + fj_cpu.log_prefix.c_str(), + fj_cpu.iterations); + return; + } } - recompute_lhs(fj_cpu); + if (fj_cpu.restores_since_improvement >= fj_cpu.infeasible_restart_max_streak) return; + if (++fj_cpu.iters_since_infeasible_improve < fj_cpu.infeasible_restart_window) return; + if (severity <= fj_cpu.best_infeasible_severity * fj_cpu.infeasible_restart_degrade_ratio) return; + if (fj_cpu.h_best_infeasible_assignment.empty()) return; + + cuopt_assert(fj_cpu.checkpoint_severity >= fj_cpu.best_infeasible_severity, + "checkpoint cannot beat the best severity seen"); + + restart_from_infeasible_checkpoint(fj_cpu); + + ++fj_cpu.n_checkpoint_restores; + ++fj_cpu.restores_since_improvement; + if (fj_cpu.restores_since_improvement > fj_cpu.max_restores_since_improvement) + fj_cpu.max_restores_since_improvement = fj_cpu.restores_since_improvement; + fj_cpu.iters_since_infeasible_improve = 0; } template @@ -1567,6 +2351,106 @@ static void init_fj_cpu(fj_cpu_climber_t& fj_cpu, problem.tolerances); } +template +static void init_fj_cpu_from_template(fj_cpu_climber_t& fj_cpu, + const fj_cpu_climber_t& tmpl, + const std::vector& left_weights, + const std::vector& right_weights, + f_t objective_weight) +{ + const i_t n_variables = (i_t)tmpl.h_reverse_offsets.size() - 1; + const i_t n_constraints = (i_t)tmpl.h_offsets.size() - 1; + const i_t nnz = (i_t)tmpl.h_coefficients.size(); + + cuopt_assert(n_variables == tmpl.view.pb.n_variables, "template variable count mismatch"); + cuopt_assert(n_constraints == tmpl.view.pb.n_constraints, "template constraint count mismatch"); + cuopt_assert(nnz == tmpl.view.pb.nnz, "template nnz mismatch"); + cuopt_assert(left_weights.size() == static_cast(n_constraints), + "left weight size mismatch"); + cuopt_assert(right_weights.size() == static_cast(n_constraints), + "right weight size mismatch"); + + fj_cpu.view = typename fj_t::climber_data_t::view_t{}; + // Every span the host views cover is re-pointed at this climber's own arrays below. The rest of + // the problem view carries over from the template, which is also what makes this usable on + // climbers built without a problem_t at all. + fj_cpu.view.pb = tmpl.view.pb; + fj_cpu.pb_ptr = tmpl.pb_ptr; + + fj_cpu.h_reverse_coefficients = tmpl.h_reverse_coefficients; + fj_cpu.h_reverse_constraints = tmpl.h_reverse_constraints; + fj_cpu.h_reverse_offsets = tmpl.h_reverse_offsets; + fj_cpu.h_coefficients = tmpl.h_coefficients; + fj_cpu.h_offsets = tmpl.h_offsets; + fj_cpu.h_variables = tmpl.h_variables; + fj_cpu.h_obj_coeffs = tmpl.h_obj_coeffs; + fj_cpu.h_var_bounds = tmpl.h_var_bounds; + fj_cpu.h_cstr_lb = tmpl.h_cstr_lb; + fj_cpu.h_cstr_ub = tmpl.h_cstr_ub; + fj_cpu.h_var_types = tmpl.h_var_types; + fj_cpu.h_is_binary_variable = tmpl.h_is_binary_variable; + fj_cpu.h_binary_indices = tmpl.h_binary_indices; + + fj_cpu.h_cstr_left_weights = left_weights; + fj_cpu.h_cstr_right_weights = right_weights; + fj_cpu.max_weight = 1.0; + fj_cpu.h_objective_weight = objective_weight; + fj_cpu.h_assignment = tmpl.h_assignment; + fj_cpu.h_best_assignment = tmpl.h_assignment; + fj_cpu.h_tabu_nodec_until.resize(n_variables, 0); + fj_cpu.h_tabu_noinc_until.resize(n_variables, 0); + fj_cpu.h_tabu_lastdec.resize(n_variables, 0); + fj_cpu.h_tabu_lastinc.resize(n_variables, 0); + fj_cpu.iterations = 0; + + finalize_fj_cpu_host_initialization_from_template(fj_cpu, + tmpl, + n_variables, + n_constraints, + tmpl.n_integer_vars, + nnz, + tmpl.view.pb.tolerances); +} + +// Certifies the epigraph variables: continuous, in the objective, and appearing in every one of +// their rows only on the side the objective pulls away from, with that direction unbounded. +template +static void certify_epigraph_variables(fj_cpu_climber_t& fj_cpu, i_t n_variables) +{ + fj_cpu.epigraph_push.assign(n_variables, 0); + fj_cpu.epigraph_vars.clear(); + + for (i_t var = 0; var < n_variables; ++var) { + if (is_integer_var(fj_cpu, var)) continue; + const f_t obj_coeff = fj_cpu.h_obj_coeffs[var]; + if (obj_coeff == f_t{0}) continue; + + const auto [begin, end] = reverse_range_for_var(fj_cpu, var); + if (begin == end) continue; + + // A positive coefficient is minimised by pushing the variable down, so its rows must be the + // only thing holding it up, and it must be free to rise as far as they demand. + const bool push_up = obj_coeff > f_t{0}; + const auto bounds = fj_cpu.h_var_bounds[var].get(); + if (isfinite(push_up ? get_upper(bounds) : get_lower(bounds))) continue; + + bool certified = true; + for (i_t p = begin; p < end && certified; ++p) { + const i_t row = fj_cpu.h_reverse_constraints[p]; + const f_t coeff = fj_cpu.h_reverse_coefficients[p]; + const bool has_lb = isfinite((f_t)fj_cpu.h_cstr_lb[row]); + const bool has_ub = isfinite((f_t)fj_cpu.h_cstr_ub[row]); + if (coeff == f_t{0}) continue; + certified = push_up ? ((coeff > 0 && has_lb && !has_ub) || (coeff < 0 && has_ub && !has_lb)) + : ((coeff > 0 && has_ub && !has_lb) || (coeff < 0 && has_lb && !has_ub)); + } + if (!certified) continue; + + fj_cpu.epigraph_push[var] = push_up ? 1 : -1; + fj_cpu.epigraph_vars.push_back(var); + } +} + template static void set_host_data_view( fj_cpu_climber_t& fj_cpu, @@ -1610,7 +2494,7 @@ static void set_host_data_view( } template -void finalize_fj_cpu_host_initialization( +static void wire_fj_cpu_host_views( fj_cpu_climber_t& fj_cpu, i_t n_variables, i_t n_constraints, @@ -1618,8 +2502,6 @@ void finalize_fj_cpu_host_initialization( i_t nnz, const typename mip_solver_settings_t::tolerances_t& tolerances) { - raft::common::nvtx::range scope("finalize_fj_cpu_host_initialization"); - cuopt_assert(n_variables >= 0, "invalid variable count"); cuopt_assert(n_constraints >= 0, "invalid constraint count"); cuopt_assert(fj_cpu.h_offsets.size() == static_cast(n_constraints + 1), @@ -1653,30 +2535,71 @@ void finalize_fj_cpu_host_initialization( fj_cpu.view.best_objective = &fj_cpu.h_best_objective; fj_cpu.view.settings = &fj_cpu.settings; - fj_cpu.h_objective_vars.resize(n_variables); - auto end = std::copy_if( - thrust::counting_iterator(0), - thrust::counting_iterator(n_variables), - fj_cpu.h_objective_vars.begin(), - [&fj_cpu](i_t idx) { return !fj_cpu.view.pb.integer_equal(fj_cpu.h_obj_coeffs[idx], (f_t)0); }); - fj_cpu.h_objective_vars.resize(end - fj_cpu.h_objective_vars.begin()); - fj_cpu.view.objective_vars = - raft::device_span(fj_cpu.h_objective_vars.data(), fj_cpu.h_objective_vars.size()); - fj_cpu.h_best_objective = +std::numeric_limits::infinity(); // nnz count fj_cpu.cached_mtm_moves.resize(fj_cpu.h_coefficients.size(), std::make_pair(0, fj_staged_score_t::zero())); + fj_cpu.cached_mtm_moves_version.assign(fj_cpu.h_coefficients.size(), -1); + fj_cpu.h_cstr_version.assign(n_constraints, 0); - fj_cpu.cached_cstr_bounds.resize(fj_cpu.h_reverse_coefficients.size()); - for (i_t var_idx = 0; var_idx < n_variables; ++var_idx) { - auto [offset_begin, offset_end] = reverse_range_for_var(fj_cpu, var_idx); - for (i_t i = offset_begin; i < offset_end; ++i) { - fj_cpu.cached_cstr_bounds[i] = - std::make_pair(fj_cpu.h_cstr_lb[fj_cpu.h_reverse_constraints[i]], - fj_cpu.h_cstr_ub[fj_cpu.h_reverse_constraints[i]]); - } + fj_cpu.flip_move_stamp.assign(n_variables, 0); + fj_cpu.flip_move_epoch = 1; + + fj_cpu.h_cstr_tolerance.resize(n_constraints); + for (i_t row = 0; row < n_constraints; ++row) { + fj_cpu.h_cstr_tolerance[row] = + fj_cpu.view.get_corrected_tolerance(row, fj_cpu.h_cstr_lb[row], fj_cpu.h_cstr_ub[row]); + } + + certify_epigraph_variables(fj_cpu, n_variables); +} + +template +void finalize_fj_cpu_host_initialization( + fj_cpu_climber_t& fj_cpu, + i_t n_variables, + i_t n_constraints, + i_t n_integer_vars, + i_t nnz, + const typename mip_solver_settings_t::tolerances_t& tolerances) +{ + raft::common::nvtx::range scope("finalize_fj_cpu_host_initialization"); + + wire_fj_cpu_host_views(fj_cpu, n_variables, n_constraints, n_integer_vars, nnz, tolerances); + + fj_cpu.h_objective_vars.resize(n_variables); + auto end = std::copy_if( + thrust::counting_iterator(0), + thrust::counting_iterator(n_variables), + fj_cpu.h_objective_vars.begin(), + [&fj_cpu](i_t idx) { return !fj_cpu.view.pb.integer_equal(fj_cpu.h_obj_coeffs[idx], (f_t)0); }); + fj_cpu.h_objective_vars.resize(end - fj_cpu.h_objective_vars.begin()); + fj_cpu.view.objective_vars = + raft::device_span(fj_cpu.h_objective_vars.data(), fj_cpu.h_objective_vars.size()); + // get_breakthrough_move divides by the coefficient of every variable in here. + for ([[maybe_unused]] auto var_idx : fj_cpu.h_objective_vars) { + cuopt_assert(fj_cpu.h_obj_coeffs[var_idx] != f_t{0}, "null coefficient in the objective vars"); + cuopt_assert(isfinite((f_t)fj_cpu.h_obj_coeffs[var_idx]), "non-finite objective coefficient"); + } + + f_t abs_obj_sum = 0; + for (auto var_idx : fj_cpu.h_objective_vars) { + const f_t coeff = fj_cpu.h_obj_coeffs[var_idx]; + abs_obj_sum += coeff < 0 ? -coeff : coeff; + } + fj_cpu.obj_magnitude = abs_obj_sum > 0 ? abs_obj_sum / fj_cpu.h_objective_vars.size() : f_t{1}; + cuopt_assert(isfinite(fj_cpu.obj_magnitude) && fj_cpu.obj_magnitude > 0, + "objective magnitude unit must be finite and positive"); + + fj_cpu.cached_cstr_bounds.resize(fj_cpu.h_reverse_coefficients.size()); + for (i_t var_idx = 0; var_idx < n_variables; ++var_idx) { + auto [offset_begin, offset_end] = reverse_range_for_var(fj_cpu, var_idx); + for (i_t i = offset_begin; i < offset_end; ++i) { + fj_cpu.cached_cstr_bounds[i] = + std::make_pair(fj_cpu.h_cstr_lb[fj_cpu.h_reverse_constraints[i]], + fj_cpu.h_cstr_ub[fj_cpu.h_reverse_constraints[i]]); + } } // precompute the binvars-pre-row tables for 2opt @@ -1692,20 +2615,151 @@ void finalize_fj_cpu_host_initialization( } fj_cpu.h_binrow_offsets[n_constraints] = fj_cpu.h_binrow_vars.size(); - fj_cpu.flip_move_computed.resize(n_variables, false); - fj_cpu.var_bitmap.resize(n_variables, false); - fj_cpu.iter_mtm_vars.reserve(n_variables); + // Must precede recompute_lhs, which is what first populates them. + fj_cpu.violated_constraints.resize(n_constraints); + fj_cpu.satisfied_constraints.resize(n_constraints); recompute_lhs(fj_cpu); // Precompute static problem features for regression model precompute_problem_features(fj_cpu); + compute_variable_coloring(fj_cpu); +} + +template +static void finalize_fj_cpu_host_initialization_from_template( + fj_cpu_climber_t& fj_cpu, + const fj_cpu_climber_t& tmpl, + i_t n_variables, + i_t n_constraints, + i_t n_integer_vars, + i_t nnz, + const typename mip_solver_settings_t::tolerances_t& tolerances) +{ + raft::common::nvtx::range scope("finalize_fj_cpu_host_initialization_from_template"); + + cuopt_assert(tmpl.h_lhs.size() == static_cast(n_constraints), "template lhs mismatch"); + cuopt_assert(tmpl.violated_constraints.max_size() == n_constraints, + "template violated set mismatch"); + cuopt_assert(tmpl.satisfied_constraints.max_size() == n_constraints, + "template satisfied set mismatch"); + cuopt_assert(tmpl.cached_cstr_bounds.size() == fj_cpu.h_reverse_coefficients.size(), + "template cached bounds mismatch"); + + cuopt_assert(tmpl.h_binrow_offsets.size() == static_cast(n_constraints + 1), + "template binrow offsets mismatch"); + + fj_cpu.h_objective_vars = tmpl.h_objective_vars; + fj_cpu.cached_cstr_bounds = tmpl.cached_cstr_bounds; + fj_cpu.h_binrow_offsets = tmpl.h_binrow_offsets; + fj_cpu.h_binrow_vars = tmpl.h_binrow_vars; + fj_cpu.obj_magnitude = tmpl.obj_magnitude; + + fj_cpu.h_lhs = tmpl.h_lhs; + fj_cpu.h_lhs_sumcomp = tmpl.h_lhs_sumcomp; + fj_cpu.violated_constraints = tmpl.violated_constraints; + fj_cpu.satisfied_constraints = tmpl.satisfied_constraints; + fj_cpu.total_violations = tmpl.total_violations; + fj_cpu.total_violations_sumcomp = tmpl.total_violations_sumcomp; + fj_cpu.h_incumbent_objective = tmpl.h_incumbent_objective; + fj_cpu.h_objective_sumcomp = tmpl.h_objective_sumcomp; + + // The colouring is structural, so it carries over; the score table is this climber's own. + fj_cpu.h_var_color = tmpl.h_var_color; + fj_cpu.n_colors = tmpl.n_colors; + if (fj_cpu.n_colors > 0) { + fj_cpu.h_var_best_score.assign(n_variables, fj_staged_score_t::invalid()); + fj_cpu.h_var_best_delta.assign(n_variables, f_t{0}); + fj_cpu.h_var_best_stamp.assign(n_variables, 0); + fj_cpu.h_var_best_rowsum.assign(n_variables, 0); + fj_cpu.h_var_bucket_stamp.assign(n_variables, 0); + fj_cpu.batch_size_hist.assign(fj_batch_hist_bins, 0); + fj_cpu.h_color_candidates.assign(fj_cpu.n_colors, {}); + fj_cpu.h_color_epoch.assign(fj_cpu.n_colors, 0); + fj_cpu.var_best_epoch = 1; + } + + fj_cpu.n_binary_vars = tmpl.n_binary_vars; + fj_cpu.n_integer_vars = tmpl.n_integer_vars; + fj_cpu.avg_var_degree = tmpl.avg_var_degree; + fj_cpu.max_var_degree = tmpl.max_var_degree; + fj_cpu.var_degree_cv = tmpl.var_degree_cv; + fj_cpu.avg_cstr_degree = tmpl.avg_cstr_degree; + fj_cpu.max_cstr_degree = tmpl.max_cstr_degree; + fj_cpu.cstr_degree_cv = tmpl.cstr_degree_cv; + fj_cpu.problem_density = tmpl.problem_density; + + wire_fj_cpu_host_views(fj_cpu, n_variables, n_constraints, n_integer_vars, nnz, tolerances); + fj_cpu.view.objective_vars = + raft::device_span(fj_cpu.h_objective_vars.data(), fj_cpu.h_objective_vars.size()); +} + +// Slacks at and above n_structural fold into their row's bounds: a*x + alpha*s = rhs with +// s in [lo, hi] becomes rhs - max(alpha*lo, alpha*hi) <= a*x <= rhs - min(alpha*lo, alpha*hi). +template +static void eliminate_slacks(const lp_problem_t& problem, + i_t n_structural, + csr_matrix_t& csr_A, + std::vector& row_lower, + std::vector& row_upper) +{ + cuopt_assert(csr_A.m == problem.num_rows, "row count mismatch"); + cuopt_assert(csr_A.n == problem.num_cols, "column count mismatch"); + cuopt_assert(n_structural > 0, "no structural columns"); + cuopt_assert(n_structural < problem.num_cols, "no slacks to eliminate"); + cuopt_assert(problem.num_cols - n_structural <= problem.num_rows, "more slacks than rows"); + + row_lower = problem.rhs; + row_upper = problem.rhs; + + std::vector row_has_slack(problem.num_rows, 0); + for (i_t j = n_structural; j < problem.num_cols; ++j) { + cuopt_assert(problem.A.col_length(j) == 1, "slack column is not a singleton"); + + const i_t entry = problem.A.col_start[j]; + const i_t row = problem.A.i[entry]; + const f_t alpha = problem.A.x[entry]; + cuopt_assert(std::abs(alpha) == f_t{1}, "slack coefficient is not +/-1"); + cuopt_assert(!row_has_slack[row], "row has more than one slack"); + row_has_slack[row] = 1; + + const f_t scaled_lower = alpha * problem.lower[j]; + const f_t scaled_upper = alpha * problem.upper[j]; + row_lower[row] = problem.rhs[row] - std::max(scaled_lower, scaled_upper); + row_upper[row] = problem.rhs[row] - std::min(scaled_lower, scaled_upper); + cuopt_assert(std::isfinite(row_lower[row]) || std::isfinite(row_upper[row]), + "eliminated row is free on both sides"); + cuopt_assert(row_lower[row] <= row_upper[row], "eliminated row has crossed bounds"); + } + + i_t out = 0; + for (i_t row = 0; row < csr_A.m; ++row) { + const i_t row_start = csr_A.row_start[row]; + const i_t row_end = csr_A.row_start[row + 1]; + csr_A.row_start[row] = out; + for (i_t p = row_start; p < row_end; ++p) { + if (csr_A.j[p] >= n_structural) { continue; } + csr_A.j[out] = csr_A.j[p]; + csr_A.x[out] = csr_A.x[p]; + ++out; + } + } + cuopt_assert( + out == csr_A.row_start[csr_A.m] - static_cast(problem.num_cols - n_structural), + "slack elimination removed the wrong number of entries"); + + csr_A.row_start[csr_A.m] = out; + csr_A.j.resize(out); + csr_A.x.resize(out); + csr_A.nz_max = out; + csr_A.n = n_structural; } template static std::unique_ptr> init_fj_cpu_from_host_lp( const lp_problem_t& problem, const std::vector& variable_types, + i_t n_structural, const std::vector& seed_assignment, const simplex_solver_settings_t& settings, std::atomic& preemption_flag, @@ -1723,16 +2777,27 @@ static std::unique_ptr> init_fj_cpu_from_host_lp( tolerances.absolute_mip_gap = settings.absolute_mip_gap_tol; tolerances.relative_mip_gap = settings.relative_mip_gap_tol; - const i_t n_variables = problem.num_cols; const i_t n_constraints = problem.num_rows; csr_matrix_t csr_A(problem.num_rows, problem.num_cols, problem.A.nnz()); problem.A.to_compressed_row(csr_A); - std::vector coefficients = csr_A.x; - std::vector variables = csr_A.j; - std::vector offsets = csr_A.row_start; - std::vector constraint_lower_bounds = problem.rhs; - std::vector constraint_upper_bounds = problem.rhs; + + std::vector constraint_lower_bounds; + std::vector constraint_upper_bounds; + i_t n_variables; + if (n_structural > 0 && n_structural < problem.num_cols) { + eliminate_slacks(problem, n_structural, csr_A, constraint_lower_bounds, constraint_upper_bounds); + n_variables = n_structural; + } else { + n_variables = problem.num_cols; + // Standard form: every row is an equality. + constraint_lower_bounds = problem.rhs; + constraint_upper_bounds = problem.rhs; + } + + std::vector coefficients = csr_A.x; + std::vector variables = csr_A.j; + std::vector offsets = csr_A.row_start; std::vector variable_bounds(n_variables); std::vector cpufj_variable_types(n_variables); std::vector is_binary_variable(n_variables, 0); @@ -1789,8 +2854,9 @@ static std::unique_ptr> init_fj_cpu_from_host_lp( fj_cpu->h_coefficients = std::move(coefficients); fj_cpu->h_offsets = std::move(offsets); fj_cpu->h_variables = std::move(variables); - fj_cpu->h_obj_coeffs = problem.objective; - fj_cpu->h_var_bounds = std::move(variable_bounds); + fj_cpu->h_obj_coeffs = + std::vector(problem.objective.begin(), problem.objective.begin() + n_variables); + fj_cpu->h_var_bounds = std::move(variable_bounds); fj_cpu->h_cstr_lb = std::move(constraint_lower_bounds); fj_cpu->h_cstr_ub = std::move(constraint_upper_bounds); fj_cpu->h_var_types = std::move(cpufj_variable_types); @@ -1818,6 +2884,14 @@ static std::unique_ptr> init_fj_cpu_from_host_lp( template static void sanity_checks(fj_cpu_climber_t& fj_cpu) { + // Assigning any of these wrappers from a plain vector rebinds its buffer and strands the span. + cuopt_assert(fj_cpu.view.incumbent_assignment.data() == fj_cpu.h_assignment.data(), + "incumbent_assignment span no longer covers h_assignment"); + cuopt_assert(fj_cpu.view.incumbent_lhs.data() == fj_cpu.h_lhs.data(), + "incumbent_lhs span no longer covers h_lhs"); + cuopt_assert(fj_cpu.view.pb.variable_bounds.data() == fj_cpu.h_var_bounds.data(), + "variable_bounds span no longer covers h_var_bounds"); + // Check that each variable is within its bounds for (i_t var_idx = 0; var_idx < fj_cpu.view.pb.n_variables; ++var_idx) { f_t val = fj_cpu.h_assignment[var_idx]; @@ -1828,7 +2902,7 @@ static void sanity_checks(fj_cpu_climber_t& fj_cpu) // Check that each violated constraint is actually violated and not present in // satisfied_constraints for (const auto& cstr_idx : fj_cpu.violated_constraints) { - cuopt_assert(fj_cpu.satisfied_constraints.count(cstr_idx) == 0, + cuopt_assert(!fj_cpu.satisfied_constraints.contains(cstr_idx), "Violated constraint also in satisfied_constraints"); f_t lhs = fj_cpu.h_lhs[cstr_idx]; f_t tol = fj_cpu.view.get_corrected_tolerance(cstr_idx); @@ -1839,7 +2913,7 @@ static void sanity_checks(fj_cpu_climber_t& fj_cpu) // Check that each satisfied constraint is actually satisfied and not present in // violated_constraints for (const auto& cstr_idx : fj_cpu.satisfied_constraints) { - cuopt_assert(fj_cpu.violated_constraints.count(cstr_idx) == 0, + cuopt_assert(!fj_cpu.violated_constraints.contains(cstr_idx), "Satisfied constraint also in violated_constraints"); f_t lhs = fj_cpu.h_lhs[cstr_idx]; f_t tol = fj_cpu.view.get_corrected_tolerance(cstr_idx); @@ -1849,8 +2923,8 @@ static void sanity_checks(fj_cpu_climber_t& fj_cpu) // Check that each constraint is in exactly one of violated_constraints or satisfied_constraints for (i_t cstr_idx = 0; cstr_idx < fj_cpu.view.pb.n_constraints; ++cstr_idx) { - bool in_viol = fj_cpu.violated_constraints.count(cstr_idx) > 0; - bool in_sat = fj_cpu.satisfied_constraints.count(cstr_idx) > 0; + bool in_viol = fj_cpu.violated_constraints.contains(cstr_idx); + bool in_sat = fj_cpu.satisfied_constraints.contains(cstr_idx); cuopt_assert( in_viol != in_sat, "Constraint must be in exactly one of violated_constraints or satisfied_constraints"); @@ -1859,6 +2933,8 @@ static void sanity_checks(fj_cpu_climber_t& fj_cpu) cuopt_assert(fj_cpu.h_cstr_right_weights[cstr_idx] >= 0, "Weights should be positive or zero"); } cuopt_assert(fj_cpu.h_objective_weight >= 0, "Objective weight should be positive or zero"); + cuopt_assert(fj_cpu.seed_objective_weight >= 0, + "Objective weight floor should be positive or zero"); } template @@ -1890,13 +2966,443 @@ std::unique_ptr> fj_t::create_cpu_climber( return fj_cpu; // move } +constexpr int32_t fj_nnz_per_refresh_stretch = 100000; +constexpr int32_t fj_max_refresh_stretch = 8; + +// Above this a short LP spends more time moving the matrix than it can pay back as a seed, and the +// wall budget the LP is allowed out of the lane's own. +constexpr int64_t fj_lp_seed_nnz_limit = 8'000'000; +constexpr double fj_lp_pump_max_budget_s = 2.0; +constexpr double fj_lp_pump_budget_share = 0.25; +constexpr int32_t fj_lp_pump_projections = 3; + +// One dual simplex solve of a relaxation on the calling thread. Reports whether the returned point +// is usable: a vertex reached at a limit is dual feasible and still worth rounding. +template +static bool solve_lp_relaxation(const simplex::user_problem_t& relaxation, + double time_limit, + std::vector& x) +{ + simplex::lp_status_t status = simplex::lp_status_t::UNSET; + double seconds = 0; + + // solve_linear_program_advanced, whose status separates a limit -- which leaves a usable vertex + // behind -- from infeasibility. Guarded on f_t because dual simplex is only built for double. + if constexpr (std::is_same_v) { + simplex_solver_settings_t lp_settings; + lp_settings.relaxation = true; + lp_settings.time_limit = time_limit; + lp_settings.log.log = false; + // The portfolio already pins one CPU per lane, and the simplex default is + // omp_get_max_threads() - 1, which would open a second portfolio inside this lane's worker. + lp_settings.num_threads = 1; + + const f_t lp_start = tic(); + lp_problem_t converted(relaxation.handle_ptr, + relaxation.num_rows, + relaxation.num_cols, + relaxation.A.col_start[relaxation.A.n]); + std::vector new_slacks; + simplex::dualize_info_t dualize_info; + simplex::convert_user_problem(relaxation, lp_settings, converted, new_slacks, dualize_info); + + simplex::lp_solution_t lp_solution(converted.num_rows, converted.num_cols); + std::vector vstatus; + std::vector edge_norms; + status = simplex::solve_linear_program_advanced( + converted, lp_start, lp_settings, lp_solution, vstatus, edge_norms); + x = std::move(lp_solution.x); + seconds = toc(lp_start); + } + + const bool usable = status == simplex::lp_status_t::OPTIMAL || + status == simplex::lp_status_t::TIME_LIMIT || + status == simplex::lp_status_t::ITERATION_LIMIT || + status == simplex::lp_status_t::CONCURRENT_LIMIT || + status == simplex::lp_status_t::WORK_LIMIT; + CUOPT_LOG_DEBUG("CPUFJ LP relaxation: %s after %.3fs of %.3fs%s", + simplex::lp_status_to_string(status).c_str(), + seconds, + time_limit, + usable ? "" : ", discarded"); + return usable; +} + +// The L1 distance to a rounded point, as an exact LP. Every integer x gains a d with the pair +// x - d <= r and -x - d <= -r, so minimising sum(d) minimises sum(abs(x - r)). +template +static simplex::user_problem_t make_lp_distance_problem( + const simplex::user_problem_t& base, + fj_cpu_climber_t& fj_cpu, + const std::vector& rounded) +{ + std::vector integer_vars; + for (i_t var = 0; var < fj_cpu.view.pb.n_variables; ++var) + if (is_integer_var(fj_cpu, var)) integer_vars.push_back(var); + const i_t n_distance = (i_t)integer_vars.size(); + + simplex::user_problem_t result(base.handle_ptr); + result.num_rows = base.num_rows + 2 * n_distance; + result.num_cols = base.num_cols + n_distance; + + // The model's own objective is dropped: this LP measures distance alone. + result.objective.assign(result.num_cols, f_t{0}); + for (i_t k = 0; k < n_distance; ++k) result.objective[base.num_cols + k] = f_t{1}; + + result.lower = base.lower; + result.upper = base.upper; + result.lower.resize(result.num_cols, f_t{0}); + result.upper.resize(result.num_cols, std::numeric_limits::infinity()); + + result.rhs = base.rhs; + result.row_sense = base.row_sense; + result.rhs.reserve(result.num_rows); + result.row_sense.reserve(result.num_rows); + for (i_t k = 0; k < n_distance; ++k) { + result.rhs.push_back(rounded[integer_vars[k]]); + result.row_sense.push_back('L'); + result.rhs.push_back(-rounded[integer_vars[k]]); + result.row_sense.push_back('L'); + } + result.range_rows = base.range_rows; + result.range_value = base.range_value; + result.num_range_rows = base.num_range_rows; + + const i_t base_nnz = base.A.col_start[base.A.n]; + csc_matrix_t matrix(result.num_rows, result.num_cols, base_nnz + 4 * n_distance); + i_t out = 0; + i_t next_integer = 0; + for (i_t j = 0; j < base.num_cols; ++j) { + matrix.col_start[j] = out; + for (i_t p = base.A.col_start[j]; p < base.A.col_start[j + 1]; ++p) { + matrix.i[out] = base.A.i[p]; + matrix.x[out++] = base.A.x[p]; + } + if (next_integer < n_distance && integer_vars[next_integer] == j) { + const i_t row = base.num_rows + 2 * next_integer++; + matrix.i[out] = row; + matrix.x[out++] = f_t{1}; + matrix.i[out] = row + 1; + matrix.x[out++] = f_t{-1}; + } + } + for (i_t k = 0; k < n_distance; ++k) { + matrix.col_start[base.num_cols + k] = out; + const i_t row = base.num_rows + 2 * k; + matrix.i[out] = row; + matrix.x[out++] = f_t{-1}; + matrix.i[out] = row + 1; + matrix.x[out++] = f_t{-1}; + } + matrix.col_start[result.num_cols] = out; + cuopt_assert(out == base_nnz + 4 * n_distance, "distance problem nonzero count mismatch"); + result.A = std::move(matrix); + return result; +} + +constexpr int32_t fj_bound_prop_rounds = 10; +// A deduction is committed only when it moves a bound by more than this many absolute tolerances. +constexpr double fj_bound_prop_commit_scale = 1e3; + +// Raises a lower bound to a deduced limit. Returns whether the domain moved. +template +static bool tighten_lower_bound(fj_cpu_climber_t& fj_cpu, + std::vector& lower, + const std::vector& upper, + i_t var, + f_t limit, + f_t commit_threshold) +{ + if (!isfinite(limit)) return false; + if (is_integer_var(fj_cpu, var)) + limit = ceil(limit - fj_cpu.view.pb.tolerances.integrality_tolerance); + if (limit > upper[var]) return false; + if (limit <= lower[var] + commit_threshold) return false; + lower[var] = limit; + return true; +} + +// Lowers an upper bound to a deduced limit. Returns whether the domain moved. +template +static bool tighten_upper_bound(fj_cpu_climber_t& fj_cpu, + const std::vector& lower, + std::vector& upper, + i_t var, + f_t limit, + f_t commit_threshold) +{ + if (!isfinite(limit)) return false; + if (is_integer_var(fj_cpu, var)) + limit = floor(limit + fj_cpu.view.pb.tolerances.integrality_tolerance); + if (limit < lower[var]) return false; + if (limit >= upper[var] - commit_threshold) return false; + upper[var] = limit; + return true; +} + +// Narrows this lane's domains by activity propagation, then reclassifies: an integer squeezed to +// [0,1] becomes eligible for the binary engine. +template +static void apply_bound_propagation(fj_cpu_climber_t& fj_cpu) +{ + if (!fj_cpu.use_bound_prop) return; + + const i_t n_variables = fj_cpu.view.pb.n_variables; + const i_t n_constraints = fj_cpu.view.pb.n_constraints; + const f_t commit = + (f_t)fj_bound_prop_commit_scale * fj_cpu.view.pb.tolerances.absolute_tolerance; + + std::vector lower(n_variables); + std::vector upper(n_variables); + for (i_t var = 0; var < n_variables; ++var) { + auto bounds = fj_cpu.h_var_bounds[var].get(); + lower[var] = get_lower(bounds); + upper[var] = get_upper(bounds); + } + + bool changed = true; + int32_t pass = 0; + for (; changed && pass < fj_bound_prop_rounds; ++pass) { + changed = false; + for (i_t row = 0; row < n_constraints; ++row) { + const f_t row_lb = fj_cpu.h_cstr_lb[row]; + const f_t row_ub = fj_cpu.h_cstr_ub[row]; + const bool has_lb = isfinite(row_lb); + const bool has_ub = isfinite(row_ub); + if (!has_lb && !has_ub) continue; + + const i_t begin = fj_cpu.h_offsets[row]; + const i_t end = fj_cpu.h_offsets[row + 1]; + + f_t min_activity = 0; + f_t max_activity = 0; + bool finite_min = true; + bool finite_max = true; + for (i_t p = begin; p < end; ++p) { + const f_t coeff = fj_cpu.h_coefficients[p]; + if (coeff == f_t{0}) continue; + const i_t var = fj_cpu.h_variables[p]; + const f_t min_x = coeff > 0 ? lower[var] : upper[var]; + const f_t max_x = coeff > 0 ? upper[var] : lower[var]; + finite_min &= isfinite(min_x); + finite_max &= isfinite(max_x); + if (finite_min) min_activity += coeff * min_x; + if (finite_max) max_activity += coeff * max_x; + } + + const bool from_row_ub = finite_min && has_ub; + const bool from_row_lb = finite_max && has_lb; + if (!from_row_ub && !from_row_lb) continue; + + // The activities are not refreshed as the loop below narrows the row's own variables, and a + // stale bound is the looser one, so a deduction taken against it is the weaker one. + for (i_t p = begin; p < end; ++p) { + const f_t coeff = fj_cpu.h_coefficients[p]; + if (coeff == f_t{0}) continue; + const i_t var = fj_cpu.h_variables[p]; + + if (from_row_ub) { + const f_t rest = min_activity - coeff * (coeff > 0 ? lower[var] : upper[var]); + const f_t limit = (row_ub - rest) / coeff; + changed |= coeff > 0 ? tighten_upper_bound(fj_cpu, lower, upper, var, limit, commit) + : tighten_lower_bound(fj_cpu, lower, upper, var, limit, commit); + } + if (from_row_lb) { + const f_t rest = max_activity - coeff * (coeff > 0 ? upper[var] : lower[var]); + const f_t limit = (row_lb - rest) / coeff; + changed |= coeff > 0 ? tighten_lower_bound(fj_cpu, lower, upper, var, limit, commit) + : tighten_upper_bound(fj_cpu, lower, upper, var, limit, commit); + } + } + } + } + + fj_cpu.h_binary_indices.clear(); + fj_cpu.n_binary_vars = 0; + fj_cpu.n_integer_vars = 0; + i_t tightened = 0; + bool clamped = false; + for (i_t var = 0; var < n_variables; ++var) { + auto bounds = fj_cpu.h_var_bounds[var].get(); + cuopt_assert(!(lower[var] < get_lower(bounds)), "propagation widened a lower bound"); + cuopt_assert(!(upper[var] > get_upper(bounds)), "propagation widened an upper bound"); + cuopt_assert(!(lower[var] > upper[var]), "propagation emptied a domain"); + const bool moved = lower[var] != get_lower(bounds) || upper[var] != get_upper(bounds); + + // Same rule as problem_t::compute_binary_var_table, fixed binaries included: a domain narrowed + // to a point is no longer binary. + const bool integer = is_integer_var(fj_cpu, var); + const bool binary = integer && fj_cpu.view.pb.integer_equal(lower[var], (f_t)0) && + fj_cpu.view.pb.integer_equal(upper[var], (f_t)1); + fj_cpu.h_is_binary_variable[var] = binary; + if (binary) { + fj_cpu.h_binary_indices.push_back(var); + ++fj_cpu.n_binary_vars; + } else if (integer) { + ++fj_cpu.n_integer_vars; + } + if (!moved) continue; + + ++tightened; + fj_cpu.h_var_bounds[var] = typename type_2::type{lower[var], upper[var]}; + + const f_t value = fj_cpu.h_assignment[var]; + const f_t clamped_value = std::clamp(value, lower[var], upper[var]); + if (clamped_value != value) { + cuopt_assert(!integer || fj_cpu.view.pb.is_integer(clamped_value), + "bound clamp broke integrality"); + fj_cpu.h_assignment[var] = clamped_value; + clamped = true; + } + fj_cpu.h_best_assignment[var] = + std::clamp((f_t)fj_cpu.h_best_assignment[var], lower[var], upper[var]); + } + + // h_binary_indices reallocated, so the span over it would otherwise dangle. + fj_cpu.view.pb.binary_indices = + raft::device_span(fj_cpu.h_binary_indices.data(), fj_cpu.h_binary_indices.size()); + + if (clamped) recompute_lhs(fj_cpu); + cuopt_func_call(audit_assignment_bounds(fj_cpu, "bound prop")); + + CUOPT_LOG_DEBUG("%sCPUFJ bound prop: %d passes, %d domains tightened, %d binary of %d integer", + fj_cpu.log_prefix.c_str(), + pass, + tightened, + fj_cpu.n_binary_vars, + fj_cpu.n_binary_vars + fj_cpu.n_integer_vars); +} + +// A bounded feasibility pump for the LP lane, run on the lane's own thread. An integral-feasible +// projection is published; otherwise FJ starts from the least violated rounding the pump saw. +template +static void apply_lp_rounded_seed(fj_cpu_climber_t& fj_cpu, f_t lane_time_limit) +{ + if (!fj_cpu.use_lp_seed || fj_cpu.pb_ptr == nullptr) return; + if (fj_cpu.view.pb.nnz > fj_lp_seed_nnz_limit) return; + + const double budget = + std::min(fj_lp_pump_max_budget_s, fj_lp_pump_budget_share * (double)lane_time_limit); + if (budget <= 0) return; + + simplex::user_problem_t base(fj_cpu.pb_ptr->handle_ptr); + fj_cpu.pb_ptr->get_host_user_problem(base); + + const auto started = std::chrono::steady_clock::now(); + const i_t n_variables = fj_cpu.view.pb.n_variables; + + std::vector rounded; + std::vector selected; + f_t selected_violation = -std::numeric_limits::infinity(); + + for (int32_t projection = 0; projection < fj_lp_pump_projections; ++projection) { + const double remaining = + budget - std::chrono::duration(std::chrono::steady_clock::now() - started).count(); + if (remaining <= 0) break; + + // Projection 0 is the plain relaxation; the rest chase the previous rounding. + const auto distance = projection == 0 ? simplex::user_problem_t(base.handle_ptr) + : make_lp_distance_problem(base, fj_cpu, rounded); + const auto& relaxation = projection == 0 ? base : distance; + + std::vector x; + if (!solve_lp_relaxation(relaxation, remaining, x)) break; + // convert_user_problem appends slacks, so the model's own variables are the leading columns. + if ((i_t)x.size() < n_variables) break; + + rounded.resize(n_variables); + cuopt::pcgenerator_t rng(fj_cpu.settings.seed); + bool valid = true; + for (i_t var = 0; var < n_variables && valid; ++var) { + const auto bounds = fj_cpu.h_var_bounds[var].get(); + const f_t lower = get_lower(bounds); + const f_t upper = get_upper(bounds); + f_t value = std::clamp(x[var], lower, upper); + if (!isfinite(value)) { + valid = false; + break; + } + if (is_integer_var(fj_cpu, var)) { + // Rounded up with probability equal to the fractional part, so successive projections of + // the same point explore different corners. + const f_t fraction = value - floor(value); + value = rng.next_double() < fraction ? ceil(value) : floor(value); + // A variable with no integral value inside its bounds cannot be seeded at all without + // breaking the engine's integrality invariant. + valid = value >= lower && value <= upper; + } + rounded[var] = value; + } + if (!valid) break; + + // Copied in place: assigning the wrapper from a plain vector rebinds its buffer and leaves the + // incumbent_assignment span on freed memory. + std::copy(rounded.begin(), rounded.end(), fj_cpu.h_assignment.begin()); + recompute_lhs(fj_cpu); + // total_violations sums a non-positive excess, so the greater value is the closer point. + if (fj_cpu.total_violations > selected_violation) { + selected_violation = fj_cpu.total_violations; + selected = rounded; + } + + // The rounded point can already be integral-feasible. It never passed through apply_move, so + // the incumbent is recorded here through the same contract that path uses. + if (fj_cpu.violated_constraints.empty() && check_variable_feasibility(fj_cpu)) { + std::copy(rounded.begin(), rounded.end(), fj_cpu.h_best_assignment.begin()); + fj_cpu.h_best_objective = + fj_cpu.h_incumbent_objective - fj_cpu.settings.parameters.breakthrough_move_epsilon; + fj_cpu.feasible_found = true; + CUOPT_LOG_DEBUG("%sCPUFJ new incumbent: objective %.17g", + fj_cpu.log_prefix.c_str(), + fj_cpu.h_best_objective); + if (fj_cpu.improvement_callback) { + fj_cpu.improvement_callback(fj_cpu.h_incumbent_objective, + fj_cpu.h_assignment, + fj_cpu.work_units_elapsed.load(std::memory_order_acquire)); + } + if (fj_cpu.shared_incumbent) { + fj_cpu.shared_incumbent->publish(fj_cpu.h_incumbent_objective, fj_cpu.h_assignment); + } + return; + } + } + + if (selected.empty()) return; + std::copy(selected.begin(), selected.end(), fj_cpu.h_assignment.begin()); + std::copy(selected.begin(), selected.end(), fj_cpu.h_best_assignment.begin()); + recompute_lhs(fj_cpu); + cuopt_func_call(audit_assignment_bounds(fj_cpu, "lp pump")); +} + template void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double work_unit_limit) { - i_t local_mins = 0; - auto loop_start = std::chrono::high_resolution_clock::now(); + const auto solve_start = std::chrono::high_resolution_clock::now(); + // Precedes the dispatch below because a variable it squeezes to [0,1] can bring the whole model + // into the binary engine's shape. + apply_bound_propagation(*fj_cpu); + // Also ahead of the dispatch, so an all-binary model gets the same LP-derived start. + apply_lp_rounded_seed(*fj_cpu, in_time_limit); + + const bool paid_setup = fj_cpu->use_bound_prop || fj_cpu->use_lp_seed; + const f_t setup_seconds = + paid_setup + ? std::chrono::duration(std::chrono::high_resolution_clock::now() - solve_start).count() + : f_t{0}; + const f_t remaining = std::max(f_t{0}, in_time_limit - setup_seconds); + if (remaining <= f_t{0}) return; + + // problem fits the binary fastpath shape? run it (engine is solve-local) + if (try_cpufj_binary_solve(*fj_cpu, remaining, work_unit_limit)) return; + + i_t local_mins = 0; + std::vector batch_moves; + // The LP comes out of this lane's own budget; every other lane's clock starts where it did. + auto loop_start = (fj_cpu->use_lp_seed || fj_cpu->use_bound_prop) + ? solve_start + : std::chrono::high_resolution_clock::now(); auto time_limit = std::chrono::milliseconds(static_cast(std::floor(in_time_limit * 1000.0))); - auto loop_time_start = std::chrono::high_resolution_clock::now(); + auto loop_time_start = loop_start; fj_cpu->rng.seed(fj_cpu->settings.seed); @@ -1904,6 +3410,30 @@ void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double w fj_cpu->last_feature_log_time = loop_start; fj_cpu->prev_best_objective = fj_cpu->h_best_objective; fj_cpu->iterations_since_best = 0; + reset_infeasible_checkpoint(*fj_cpu); + fj_cpu->n_checkpoint_restores = 0; + fj_cpu->n_checkpoint_snapshots = 0; + fj_cpu->restores_since_improvement = 0; + fj_cpu->max_restores_since_improvement = 0; + + // The recompute is O(nnz), so a fixed period costs a growing share of the budget. + cuopt_assert(fj_cpu->settings.parameters.lhs_refresh_period > 0, + "lhs_refresh_period should be positive"); + const i_t nnz_stretch = std::min( + (i_t)fj_cpu->h_coefficients.size() / fj_nnz_per_refresh_stretch, fj_max_refresh_stretch); + const i_t refresh_period = fj_cpu->settings.parameters.lhs_refresh_period * (1 + nnz_stretch); + //const i_t refresh_period = 5000 * (1 + nnz_stretch); + cuopt_assert(refresh_period > 0, "refresh period overflowed"); + fj_cpu->lhs_refresh_period_used = refresh_period; + + // Whatever the seed left behind, these rows are satisfiable on their own, so the walk should not + // start with them in the violated set competing for the sampler's attention. + for (i_t var : fj_cpu->epigraph_vars) { + const f_t delta = project_epigraph_variable(*fj_cpu, var) - (f_t)fj_cpu->h_assignment[var]; + if (delta == f_t{0}) continue; + apply_move(*fj_cpu, var, delta, false); + ++fj_cpu->n_epigraph_projections; + } while (!fj_cpu->halted && !fj_cpu->preemption_flag.load()) { // Check if 5 seconds have passed @@ -1926,12 +3456,13 @@ void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double w // periodically recompute the LHS and violation scores // to correct any accumulated numerical errors - cuopt_assert(fj_cpu->settings.parameters.lhs_refresh_period > 0, - "lhs_refresh_period should be positive"); - if (fj_cpu->iterations % fj_cpu->settings.parameters.lhs_refresh_period == 0 || - fj_cpu->trigger_early_lhs_recomputation) { + if (fj_cpu->trigger_early_lhs_recomputation) { + ++fj_cpu->n_lhs_recompute_bigval; recompute_lhs(*fj_cpu); fj_cpu->trigger_early_lhs_recomputation = false; + } else if (fj_cpu->iterations % refresh_period == 0) { + ++fj_cpu->n_lhs_recompute_periodic; + recompute_lhs(*fj_cpu); } fj_move_t move = fj_move_t{-1, 0}; @@ -1941,9 +3472,23 @@ void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double w bool is_mtm_sat = false; // Perform lift moves + fj_move_t lift_companion = fj_move_t{-1, 0}; if (fj_cpu->violated_constraints.empty()) { thrust::tie(move, score) = find_lift_move(*fj_cpu); - if (score > fj_staged_score_t::zero()) is_lift = true; + if (score > fj_staged_score_t::zero()) { + is_lift = true; + } else { + // Pairs are only reachable once no single improving flip preserves feasibility. + fj_move_t first, second; + fj_staged_score_t pair_score; + thrust::tie(first, second, pair_score) = find_lift_2opt_move(*fj_cpu); + if (pair_score > fj_staged_score_t::zero()) { + move = first; + lift_companion = second; + score = pair_score; + is_lift = true; + } + } } // Regular MTM if (!(score > fj_staged_score_t::zero())) { @@ -1955,17 +3500,40 @@ void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double w thrust::tie(move, score) = find_mtm_move_sat(*fj_cpu, fj_cpu->mtm_sat_samples); if (score > fj_staged_score_t::zero()) is_mtm_sat = true; } + // The scorers target one row at a time, so on an epigraph variable they climb toward the bound + // its rows already imply. The projection lands there in one move at the same O(degree) cost. + if (move.var_idx >= 0 && fj_cpu->epigraph_push[move.var_idx] != 0) { + const f_t projected = project_epigraph_variable(*fj_cpu, move.var_idx) - + (f_t)fj_cpu->h_assignment[move.var_idx]; + if (projected != f_t{0}) { + move.value = projected; + ++fj_cpu->n_epigraph_projections; + } + } + // if we're in the feasible region but haven't found improvements in the last n iterations, // perturb bool should_perturb = false; if (fj_cpu->violated_constraints.empty() && - fj_cpu->iterations - fj_cpu->last_feasible_entrance_iter > fj_cpu->perturb_interval) { - should_perturb = true; - fj_cpu->last_feasible_entrance_iter = fj_cpu->iterations; + fj_cpu->iterations_since_best > fj_cpu->perturb_interval) { + should_perturb = true; + // Without this the counter stays above the interval and every later iteration perturbs. + fj_cpu->iterations_since_best = 0; } if (score > fj_staged_score_t::zero() && !should_perturb) { + // A 2-opt lift already commits two coupled moves, and its second half is scored against the + // state before both, so it stays on its own. + if (lift_companion.var_idx < 0) { + collect_move_batch(*fj_cpu, move, batch_moves); + for (const auto& batched : batch_moves) + apply_move(*fj_cpu, batched.var_idx, batched.value, false); + } apply_move(*fj_cpu, move.var_idx, move.value, false); + if (lift_companion.var_idx >= 0) { + apply_move(*fj_cpu, lift_companion.var_idx, lift_companion.value, false); + fj_cpu->n_lift_moves_window++; + } // Track move types if (is_lift) fj_cpu->n_lift_moves_window++; if (is_mtm_viol) fj_cpu->n_mtm_viol_moves_window++; @@ -1973,10 +3541,10 @@ void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double w } else { // Local Min update_weights(*fj_cpu); + track_infeasible_checkpoint(*fj_cpu); if (should_perturb) { perturb(*fj_cpu); - for (size_t i = 0; i < fj_cpu->cached_mtm_moves.size(); i++) - fj_cpu->cached_mtm_moves[i].first = 0; + invalidate_mtm_cache(*fj_cpu); } two_opt_move_t two_opt_move; @@ -1996,12 +3564,6 @@ void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double w ++fj_cpu->n_local_minima_window; } - // number of violated constraints is usually small (<100). recomputing from all LHSs is cheap - // and more numerically precise than just adding to the accumulator in apply_move - fj_cpu->total_violations = 0; - for (auto cstr_idx : fj_cpu->violated_constraints) { - fj_cpu->total_violations += fj_cpu->view.excess_score(cstr_idx, fj_cpu->h_lhs[cstr_idx]); - } if (fj_cpu->iterations % fj_cpu->log_interval == 0) { CUOPT_LOG_DEBUG( "%sCPUFJ iteration: %d/%d, local mins: %d, best_objective: %g, viol: %zu, obj weight %g, " @@ -2060,14 +3622,196 @@ void cpufj_solve(fj_cpu_climber_t* fj_cpu, f_t in_time_limit, double w CUOPT_LOG_TRACE("%sCPUFJ Average time per iteration: %.8fms", fj_cpu->log_prefix.c_str(), avg_time_per_iter * 1000.0); + CUOPT_LOG_DEBUG("%sCPUFJ checkpoint: %lld restores, %lld snapshots, max streak %d", + fj_cpu->log_prefix.c_str(), + (long long)fj_cpu->n_checkpoint_restores, + (long long)fj_cpu->n_checkpoint_snapshots, + fj_cpu->max_restores_since_improvement); + log_batch_distribution(*fj_cpu); #if CPUFJ_TIMING_TRACE // Print final timing statistics - CUOPT_LOG_TRACE("=== Final Timing Statistics ==="); + CUOPT_LOG_DEBUG("=== Final Timing Statistics ==="); print_timing_stats(*fj_cpu); #endif } +template +static std::vector copy_to_host_async(const rmm::device_uvector& input, + rmm::cuda_stream_view stream) +{ + std::vector output(input.size()); + raft::copy(output.data(), input.data(), input.size(), stream); + return output; +} + +template +std::unique_ptr> init_fj_cpu_from_optimization_problem( + const optimization_problem_t& problem, + const typename mip_solver_settings_t::tolerances_t& tolerances, + std::atomic& preemption_flag, + fj_settings_t settings) +{ + using f_t2 = typename type_2::type; + + raft::common::nvtx::range scope("init_fj_cpu_from_optimization_problem"); + + const i_t n_variables = problem.get_n_variables(); + const i_t n_constraints = problem.get_n_constraints(); + const i_t nnz = problem.get_nnz(); + auto stream = problem.get_handle_ptr()->get_stream(); + + auto coefficients = copy_to_host_async(problem.get_constraint_matrix_values(), stream); + auto variables = copy_to_host_async(problem.get_constraint_matrix_indices(), stream); + auto offsets = copy_to_host_async(problem.get_constraint_matrix_offsets(), stream); + auto objective_coefficients = copy_to_host_async(problem.get_objective_coefficients(), stream); + auto variable_lower_bounds = copy_to_host_async(problem.get_variable_lower_bounds(), stream); + auto variable_upper_bounds = copy_to_host_async(problem.get_variable_upper_bounds(), stream); + auto constraint_lower_bounds = copy_to_host_async(problem.get_constraint_lower_bounds(), stream); + auto constraint_upper_bounds = copy_to_host_async(problem.get_constraint_upper_bounds(), stream); + auto constraint_bounds = copy_to_host_async(problem.get_constraint_bounds(), stream); + auto row_types = copy_to_host_async(problem.get_row_types(), stream); + auto variable_types = copy_to_host_async(problem.get_variable_types(), stream); + problem.get_handle_ptr()->sync_stream(); + + cuopt_assert(coefficients.size() == (size_t)nnz, "coefficient size mismatch"); + cuopt_assert(variables.size() == (size_t)nnz, "variable index size mismatch"); + cuopt_assert(offsets.size() == (size_t)(n_constraints + 1), + "constraint offset size mismatch"); + cuopt_assert(!offsets.empty() && offsets.front() == 0, "invalid first constraint offset"); + cuopt_assert(offsets.back() == nnz, "invalid final constraint offset"); + cuopt_assert(std::is_sorted(offsets.begin(), offsets.end()), "unsorted constraint offsets"); + cuopt_assert( + std::all_of(variables.begin(), + variables.end(), + [n_variables](i_t variable) { return variable >= 0 && variable < n_variables; }), + "variable index out of range"); + cuopt_assert(objective_coefficients.size() == (size_t)n_variables, + "objective size mismatch"); + cuopt_assert(variable_lower_bounds.empty() || + variable_lower_bounds.size() == (size_t)n_variables, + "variable lower bound size mismatch"); + cuopt_assert(variable_upper_bounds.empty() || + variable_upper_bounds.size() == (size_t)n_variables, + "variable upper bound size mismatch"); + + if (constraint_lower_bounds.empty() && constraint_upper_bounds.empty()) { + cuopt_assert(row_types.size() == (size_t)n_constraints, "row type size mismatch"); + cuopt_assert(constraint_bounds.size() == (size_t)n_constraints, + "constraint bound size mismatch"); + constraint_lower_bounds.resize(n_constraints); + constraint_upper_bounds.resize(n_constraints); + for (i_t row = 0; row < n_constraints; ++row) { + const f_t bound = constraint_bounds[row]; + if (row_types[row] == 'E') { + constraint_lower_bounds[row] = bound; + constraint_upper_bounds[row] = bound; + } else if (row_types[row] == 'G') { + constraint_lower_bounds[row] = bound; + constraint_upper_bounds[row] = std::numeric_limits::infinity(); + } else { + cuopt_assert(row_types[row] == 'L', "invalid row type"); + constraint_lower_bounds[row] = -std::numeric_limits::infinity(); + constraint_upper_bounds[row] = bound; + } + } + } else { + cuopt_assert(constraint_lower_bounds.size() == (size_t)n_constraints, + "constraint lower bound size mismatch"); + cuopt_assert(constraint_upper_bounds.size() == (size_t)n_constraints, + "constraint upper bound size mismatch"); + } + + if (variable_lower_bounds.empty()) { variable_lower_bounds.assign(n_variables, f_t{0}); } + if (variable_upper_bounds.empty()) { + variable_upper_bounds.assign(n_variables, std::numeric_limits::infinity()); + } + if (variable_types.empty()) { variable_types.assign(n_variables, var_t::CONTINUOUS); } + cuopt_assert(variable_types.size() == (size_t)n_variables, + "variable type size mismatch"); + + if (problem.get_sense()) { + std::transform(objective_coefficients.begin(), + objective_coefficients.end(), + objective_coefficients.begin(), + std::negate{}); + } + + std::vector variable_bounds(n_variables); + std::vector is_binary_variable(n_variables, 0); + std::vector binary_indices; + binary_indices.reserve(n_variables); + i_t n_integer_vars = 0; + for (i_t variable = 0; variable < n_variables; ++variable) { + f_t lower = variable_lower_bounds[variable]; + f_t upper = variable_upper_bounds[variable]; + const bool is_integer = variable_types[variable] == var_t::INTEGER; + if (is_integer) { + lower = std::ceil(lower); + upper = std::floor(upper); + ++n_integer_vars; + } + cuopt_assert(lower <= upper, "crossing variable bounds"); + variable_bounds[variable] = f_t2{lower, upper}; + if (is_integer && lower == f_t{0} && upper == f_t{1}) { + is_binary_variable[variable] = 1; + binary_indices.push_back(variable); + } + } + + csr_matrix_t csr(n_constraints, n_variables, nnz); + csr.x = coefficients; + csr.j = variables; + csr.row_start = offsets; + csc_matrix_t csc(n_constraints, n_variables, nnz); + csr.to_compressed_col(csc); + + std::vector assignment(n_variables, f_t{0}); + for (i_t variable = 0; variable < n_variables; ++variable) { + f_t value = std::clamp( + f_t{0}, get_lower(variable_bounds[variable]), get_upper(variable_bounds[variable])); + if (variable_types[variable] == var_t::INTEGER) { value = std::round(value); } + assignment[variable] = value; + } + + auto fj_cpu = std::make_unique>(preemption_flag); + fj_cpu->view = typename fj_t::climber_data_t::view_t{}; + fj_cpu->pb_ptr = nullptr; + fj_cpu->settings = settings; + + fj_cpu->h_reverse_coefficients = std::move(csc.x); + fj_cpu->h_reverse_constraints = std::move(csc.i); + fj_cpu->h_reverse_offsets = std::move(csc.col_start); + fj_cpu->h_coefficients = std::move(coefficients); + fj_cpu->h_offsets = std::move(offsets); + fj_cpu->h_variables = std::move(variables); + fj_cpu->h_obj_coeffs = std::move(objective_coefficients); + fj_cpu->h_var_bounds = std::move(variable_bounds); + fj_cpu->h_cstr_lb = std::move(constraint_lower_bounds); + fj_cpu->h_cstr_ub = std::move(constraint_upper_bounds); + fj_cpu->h_var_types = std::move(variable_types); + fj_cpu->h_is_binary_variable = std::move(is_binary_variable); + fj_cpu->h_binary_indices = std::move(binary_indices); + fj_cpu->h_cstr_left_weights.resize(n_constraints, f_t{1}); + fj_cpu->h_cstr_right_weights.resize(n_constraints, f_t{1}); + fj_cpu->max_weight = f_t{1}; + fj_cpu->h_objective_weight = f_t{0}; + fj_cpu->h_assignment = assignment; + fj_cpu->h_best_assignment = std::move(assignment); + fj_cpu->h_lhs.resize(n_constraints); + fj_cpu->h_lhs_sumcomp.resize(n_constraints, f_t{0}); + fj_cpu->h_tabu_nodec_until.resize(n_variables, 0); + fj_cpu->h_tabu_noinc_until.resize(n_variables, 0); + fj_cpu->h_tabu_lastdec.resize(n_variables, 0); + fj_cpu->h_tabu_lastinc.resize(n_variables, 0); + fj_cpu->iterations = 0; + fj_cpu->settings.seed = cuopt::seed_generator::get_seed(); + + finalize_fj_cpu_host_initialization( + *fj_cpu, n_variables, n_constraints, n_integer_vars, nnz, tolerances); + return fj_cpu; +} + template std::unique_ptr> init_fj_cpu_standalone( problem_t& problem, @@ -2083,8 +3827,27 @@ std::unique_ptr> init_fj_cpu_standalone( // Early CPUFJ runs while presolve is still probing, so there are no implications to hand it const probing_cache_t* no_implications = nullptr; init_fj_cpu(*fj_cpu, solution, default_weights, default_weights, 0.0, no_implications); - fj_cpu->settings = settings; - fj_cpu->settings.seed = cuopt::seed_generator::get_seed(); + // settings.seed is caller-drawn: seed_generator steps a non-atomic global and this may run + // concurrently across lanes. + fj_cpu->settings = settings; + + return fj_cpu; +} + +template +std::unique_ptr> init_fj_cpu_clone( + const fj_cpu_climber_t& tmpl, + std::atomic& preemption_flag, + fj_settings_t settings) +{ + raft::common::nvtx::range scope("init_fj_cpu_clone"); + + auto fj_cpu = std::make_unique>(preemption_flag); + + std::vector default_weights(tmpl.view.pb.n_constraints, 1.0); + init_fj_cpu_from_template(*fj_cpu, tmpl, default_weights, default_weights, f_t{0}); + // See init_fj_cpu_standalone: the seed is caller-drawn, not taken from the global generator. + fj_cpu->settings = settings; return fj_cpu; } @@ -2095,23 +3858,33 @@ void fj_cpu_worker_t::fj_cpu_deleter_t::operator()(fj_cpu_climber_t +std::shared_ptr> make_fj_cpu_shared_incumbent() +{ + return std::make_shared>(); +} + template void fj_cpu_worker_t::create_worker( const lp_problem_t& problem, const std::vector& variable_types, + i_t n_structural, const std::vector& seed_assignment, const simplex_solver_settings_t& settings, std::string log_prefix, - int64_t seed) + int64_t seed, + int lane) { auto new_climber = init_fj_cpu_from_host_lp( - problem, variable_types, seed_assignment, settings, preemption_flag, seed); + problem, variable_types, n_structural, seed_assignment, settings, preemption_flag, seed); fj_cpu.reset(new_climber.release()); fj_cpu->log_prefix = std::move(log_prefix); fj_cpu->improvement_callback = improvement_callback; + fj_cpu->shared_incumbent = shared_incumbent; fj_cpu->halted = false; preemption_flag = false; is_initialized = true; + if (lane >= 0) { apply_lane_diversification(*fj_cpu, lane, fj_cpu->settings.seed); } } template @@ -2158,6 +3931,8 @@ void fj_cpu_worker_t::send_stop_signal() #if MIP_INSTANTIATE_FLOAT template class fj_t; template struct fj_cpu_worker_t; +template std::shared_ptr> +make_fj_cpu_shared_incumbent(); template void cpufj_solve(fj_cpu_climber_t* fj_cpu, float in_time_limit, double work_unit_limit); @@ -2166,6 +3941,15 @@ template std::unique_ptr> init_fj_cpu_standalone( solution_t& solution, std::atomic& preemption_flag, fj_settings_t settings); +template std::unique_ptr> init_fj_cpu_clone( + const fj_cpu_climber_t& tmpl, + std::atomic& preemption_flag, + fj_settings_t settings); +template std::unique_ptr> init_fj_cpu_from_optimization_problem( + const optimization_problem_t& problem, + const typename mip_solver_settings_t::tolerances_t& tolerances, + std::atomic& preemption_flag, + fj_settings_t settings); template void finalize_fj_cpu_host_initialization( fj_cpu_climber_t& fj_cpu, int n_variables, @@ -2178,6 +3962,8 @@ template void finalize_fj_cpu_host_initialization( #if MIP_INSTANTIATE_DOUBLE template class fj_t; template struct fj_cpu_worker_t; +template std::shared_ptr> +make_fj_cpu_shared_incumbent(); template void cpufj_solve(fj_cpu_climber_t* fj_cpu, double in_time_limit, double work_unit_limit); @@ -2186,6 +3972,15 @@ template std::unique_ptr> init_fj_cpu_standalone( solution_t& solution, std::atomic& preemption_flag, fj_settings_t settings); +template std::unique_ptr> init_fj_cpu_clone( + const fj_cpu_climber_t& tmpl, + std::atomic& preemption_flag, + fj_settings_t settings); +template std::unique_ptr> init_fj_cpu_from_optimization_problem( + const optimization_problem_t& problem, + const typename mip_solver_settings_t::tolerances_t& tolerances, + std::atomic& preemption_flag, + fj_settings_t settings); template void finalize_fj_cpu_host_initialization( fj_cpu_climber_t& fj_cpu, int n_variables, @@ -2195,4 +3990,820 @@ template void finalize_fj_cpu_host_initialization( const typename mip_solver_settings_t::tolerances_t& tolerances); #endif +// Above this the O(nnz) seed passes eat a meaningful slice of a short budget, so they are skipped. +constexpr int64_t fj_seed_nnz_limit = 8'000'000; + +// Budget for the matching seed and the widest exact-one row it will take into the graph. +constexpr double fj_matching_budget_s = 0.45; +constexpr int32_t fj_matching_max_row_width = 20000; + +// The aggressive corner pushes harder than the covering seed: more passes, a longer budget, a +// tighter clock, and it gives up as soon as a pass changes nothing. +constexpr int32_t fj_aggressive_passes = 6; +constexpr double fj_aggressive_budget_s = 0.9; + +// Cardinality-row detection: coefficient agreement tolerance and the widest row worth peeling. +constexpr double fj_exact_k_tol = 1e-6; +constexpr int32_t fj_exact_k_max_width = 20000; +constexpr double fj_exact_k_budget_s = 0.5; +// The anchor repair only runs when this fraction of the rows is violated, and gets this long. +constexpr int32_t fj_anchor_repair_violated_share = 5; +constexpr double fj_anchor_repair_budget_s = 0.1; + +// Jumps each two-sided variable to whichever bound has fewer rows locking it in that direction. +template +static void apply_lock_weighted_seed(fj_cpu_climber_t& fj_cpu) +{ + if (fj_cpu.view.pb.nnz > fj_seed_nnz_limit) return; + + const i_t n_variables = fj_cpu.view.pb.n_variables; + for (i_t var_idx = 0; var_idx < n_variables; ++var_idx) { + const f_t lb = get_lower(fj_cpu.h_var_bounds[var_idx].get()); + const f_t ub = get_upper(fj_cpu.h_var_bounds[var_idx].get()); + if (!isfinite(lb) || !isfinite(ub) || lb >= ub) continue; + + i_t lock_up = 0; + i_t lock_down = 0; + const auto range = reverse_range_for_var(fj_cpu, var_idx); + for (i_t i = range.first; i < range.second; ++i) { + const f_t coeff = fj_cpu.h_reverse_coefficients[i]; + const i_t cstr_idx = fj_cpu.h_reverse_constraints[i]; + const bool has_lb = isfinite((f_t)fj_cpu.h_cstr_lb[cstr_idx]); + const bool has_ub = isfinite((f_t)fj_cpu.h_cstr_ub[cstr_idx]); + if (coeff > 0) { + lock_up += has_ub; + lock_down += has_lb; + } else if (coeff < 0) { + lock_up += has_lb; + lock_down += has_ub; + } + } + + f_t new_val = lock_up <= lock_down ? ub : lb; + if (is_integer_var(fj_cpu, var_idx)) new_val = std::round(new_val); + fj_cpu.h_assignment[var_idx] = new_val; + } + + recompute_lhs(fj_cpu); + fj_cpu.h_best_assignment = fj_cpu.h_assignment; +} + +// Jumps each bounded objective variable to the bound that minimises its own objective term. +template +static void apply_objective_corner_seed(fj_cpu_climber_t& fj_cpu) +{ + if (fj_cpu.view.pb.nnz > fj_seed_nnz_limit) return; + + const i_t n_variables = fj_cpu.view.pb.n_variables; + for (i_t var_idx = 0; var_idx < n_variables; ++var_idx) { + const f_t coeff = fj_cpu.h_obj_coeffs[var_idx]; + if (coeff == 0) continue; + + const f_t lb = get_lower(fj_cpu.h_var_bounds[var_idx].get()); + const f_t ub = get_upper(fj_cpu.h_var_bounds[var_idx].get()); + if (!isfinite(lb) || !isfinite(ub) || lb >= ub) continue; + + f_t new_val = coeff > 0 ? lb : ub; + if (is_integer_var(fj_cpu, var_idx)) new_val = std::round(new_val); + fj_cpu.h_assignment[var_idx] = new_val; + } + + recompute_lhs(fj_cpu); + fj_cpu.h_best_assignment = fj_cpu.h_assignment; +} + +// A single-variable integer step on a row, with the magnitude of its effect on the row sum. +template +struct row_repair_move_t { + f_t effect; + i_t var; + f_t coeff; + f_t new_val; +}; + +// Collects the unit integer steps that push this row's sum in `direction`, largest effect first. +template +static void collect_row_repair_moves(fj_cpu_climber_t& fj_cpu, + i_t row_begin, + i_t row_end, + f_t direction, + f_t tol, + std::vector>& out) +{ + out.clear(); + for (i_t i = row_begin; i < row_end; ++i) { + const i_t var = fj_cpu.h_variables[i]; + if (!is_integer_var(fj_cpu, var)) continue; + + const f_t coeff = fj_cpu.h_coefficients[i]; + const f_t val = fj_cpu.h_assignment[var]; + const f_t lb = get_lower(fj_cpu.h_var_bounds[var].get()); + const f_t ub = get_upper(fj_cpu.h_var_bounds[var].get()); + const bool is_bin = fj_cpu.h_is_binary_variable[var] != 0; + + // Raising the variable shifts the sum by `direction * coeff`; lowering it by the negation. + const f_t raise = direction * coeff; + if (raise > 0 && val < ub - tol) { + const f_t new_val = is_bin ? (f_t)1 : std::floor(val) + 1; + if (new_val > val && new_val <= ub + tol) out.push_back({raise, var, coeff, new_val}); + } else if (raise < 0 && val > lb + tol) { + const f_t new_val = is_bin ? (f_t)0 : std::ceil(val) - 1; + if (new_val < val && new_val >= lb - tol) out.push_back({-raise, var, coeff, new_val}); + } + } + std::sort(out.begin(), out.end(), [](const row_repair_move_t& a, + const row_repair_move_t& b) { + return a.effect > b.effect; + }); +} + +// Time-boxed greedy row repair. Deliberately myopic, so it reverts unless it strictly reduces the +// violated-row count against the incoming anchor. +template +static void apply_greedy_covering_seed(fj_cpu_climber_t& fj_cpu) +{ + if (fj_cpu.view.pb.nnz > fj_seed_nnz_limit) return; + + recompute_lhs(fj_cpu); + const i_t baseline_violated = fj_cpu.violated_constraints.size(); + const auto anchor_assignment = fj_cpu.h_assignment; + + const i_t n_constraints = fj_cpu.view.pb.n_constraints; + std::vector row_order(n_constraints); + for (i_t i = 0; i < n_constraints; ++i) + row_order[i] = i; + std::sort(row_order.begin(), row_order.end(), [&](i_t a, i_t b) { + return (fj_cpu.h_offsets[a + 1] - fj_cpu.h_offsets[a]) < + (fj_cpu.h_offsets[b + 1] - fj_cpu.h_offsets[b]); + }); + + const auto started = std::chrono::steady_clock::now(); + const double time_budget_s = 0.4; + const f_t tol = 1e-6; + const i_t max_passes = 2; + std::vector> candidates; + bool out_of_time = false; + + for (i_t pass = 0; pass < max_passes && !out_of_time; ++pass) { + for (i_t k = 0; k < n_constraints; ++k) { + if ((k & 0xFFF) == 0 && + std::chrono::duration(std::chrono::steady_clock::now() - started).count() > + time_budget_s) { + out_of_time = true; + break; + } + const i_t cstr_idx = row_order[k]; + const i_t row_begin = fj_cpu.h_offsets[cstr_idx]; + const i_t row_end = fj_cpu.h_offsets[cstr_idx + 1]; + if (row_begin == row_end) continue; + + const f_t lb = fj_cpu.h_cstr_lb[cstr_idx]; + const f_t ub = fj_cpu.h_cstr_ub[cstr_idx]; + const bool has_lb = isfinite(lb); + const bool has_ub = isfinite(ub); + if (!has_lb && !has_ub) continue; + + f_t sum = 0; + for (i_t i = row_begin; i < row_end; ++i) + sum += (f_t)fj_cpu.h_coefficients[i] * (f_t)fj_cpu.h_assignment[fj_cpu.h_variables[i]]; + + // Equality rows are driven to their bound; one-sided rows only to the side they violate. + const bool is_equality = has_lb && has_ub && std::abs(lb - ub) < tol; + f_t direction = 0; + f_t target = 0; + if (is_equality && std::abs(sum - lb) > tol) { + direction = sum < lb ? (f_t)1 : (f_t)-1; + target = lb; + } else if (has_lb && sum < lb - tol) { + direction = 1; + target = lb; + } else if (has_ub && sum > ub + tol) { + direction = -1; + target = ub; + } else { + continue; + } + + collect_row_repair_moves(fj_cpu, row_begin, row_end, direction, tol, candidates); + for (const auto& m : candidates) { + if (direction > 0 ? sum >= target - tol : sum <= target + tol) break; + const f_t delta = m.new_val - (f_t)fj_cpu.h_assignment[m.var]; + sum += m.coeff * delta; + fj_cpu.h_assignment[m.var] = m.new_val; + } + } + } + + recompute_lhs(fj_cpu); + if ((i_t)fj_cpu.violated_constraints.size() >= baseline_violated) { + fj_cpu.h_assignment = anchor_assignment; + recompute_lhs(fj_cpu); + } + fj_cpu.h_best_assignment = fj_cpu.h_assignment; +} + +// Repeated one-sided row repair in CSR order. Unlike the covering seed it revisits rows until a +// pass changes nothing, so a repair that breaks a row already visited gets another chance. +template +static void apply_aggressive_constraint_seed(fj_cpu_climber_t& fj_cpu) +{ + if (fj_cpu.view.pb.nnz > fj_seed_nnz_limit) return; + + const auto started = std::chrono::steady_clock::now(); + auto timed_out = [&] { + return std::chrono::duration(std::chrono::steady_clock::now() - started).count() > + fj_aggressive_budget_s; + }; + + recompute_lhs(fj_cpu); + const i_t baseline = fj_cpu.violated_constraints.size(); + const auto anchor = fj_cpu.h_assignment; + const f_t tol = 1e-6; + std::vector> candidates; + + for (i_t pass = 0; pass < fj_aggressive_passes && !timed_out(); ++pass) { + i_t moves = 0; + for (i_t row = 0; row < fj_cpu.view.pb.n_constraints; ++row) { + if ((row & 0xFF) == 0 && timed_out()) break; + + const i_t begin = fj_cpu.h_offsets[row]; + const i_t end = fj_cpu.h_offsets[row + 1]; + if (begin == end) continue; + + const f_t lb = fj_cpu.h_cstr_lb[row]; + const f_t ub = fj_cpu.h_cstr_ub[row]; + const bool has_lb = isfinite(lb); + const bool has_ub = isfinite(ub); + if (!has_lb && !has_ub) continue; + + f_t sum = 0; + for (i_t p = begin; p < end; ++p) + sum += (f_t)fj_cpu.h_coefficients[p] * (f_t)fj_cpu.h_assignment[fj_cpu.h_variables[p]]; + + f_t direction = 0; + f_t target = 0; + if (has_lb && sum < lb - tol) { + direction = 1; + target = lb; + } else if (has_ub && sum > ub + tol) { + direction = -1; + target = ub; + } else { + continue; + } + + collect_row_repair_moves(fj_cpu, begin, end, direction, tol, candidates); + for (const auto& move : candidates) { + if (direction > 0 ? sum >= target - tol : sum <= target + tol) break; + const f_t delta = move.new_val - (f_t)fj_cpu.h_assignment[move.var]; + sum += move.coeff * delta; + fj_cpu.h_assignment[move.var] = move.new_val; + ++moves; + } + } + if (moves == 0) break; + } + + recompute_lhs(fj_cpu); + if ((i_t)fj_cpu.violated_constraints.size() >= baseline) { + fj_cpu.h_assignment = anchor; + recompute_lhs(fj_cpu); + } + fj_cpu.h_best_assignment = fj_cpu.h_assignment; +} + +// Treats the exact-one rows as a graph in which each variable is an edge between the two rows it +// appears in. A component that is bipartite and has equally many rows on each side admits a perfect +// matching, and the cheapest one is the assignment satisfying every row in the component at least +// cost. Solved per component as min-cost flow by successive shortest paths, which needs no +// potentials here because augmenting along shortest paths keeps the residual free of negative +// cycles. Components that are not of that shape are left to the search. +template +static void apply_bipartite_matching_seed(fj_cpu_climber_t& fj_cpu) +{ + if (fj_cpu.view.pb.nnz > fj_seed_nnz_limit) return; + + const auto started = std::chrono::steady_clock::now(); + auto timed_out = [&] { + return std::chrono::duration(std::chrono::steady_clock::now() - started).count() > + fj_matching_budget_s; + }; + const f_t tol = 1e-6; + + struct exact_one_row_t { + i_t begin, end; + }; + std::vector rows; + for (i_t row = 0; row < fj_cpu.view.pb.n_constraints; ++row) { + if ((row & 0xFFF) == 0 && timed_out()) return; + + const f_t lb = fj_cpu.h_cstr_lb[row]; + const f_t ub = fj_cpu.h_cstr_ub[row]; + if (!isfinite(lb) || !isfinite(ub) || std::abs(lb - ub) > tol) continue; + + const i_t begin = fj_cpu.h_offsets[row]; + const i_t end = fj_cpu.h_offsets[row + 1]; + if (begin == end || end - begin > fj_matching_max_row_width) continue; + + const f_t scale = fj_cpu.h_coefficients[begin]; + if (!isfinite(scale) || std::abs(scale) <= tol || std::abs(lb / scale - 1) > 1e-5) continue; + + bool uniform_binary = true; + for (i_t p = begin; p < end && uniform_binary; ++p) { + const f_t coeff = fj_cpu.h_coefficients[p]; + const f_t agreement = tol * std::max((f_t)1, std::abs(scale)); + uniform_binary = fj_cpu.h_is_binary_variable[fj_cpu.h_variables[p]] && + std::abs(coeff - scale) <= agreement; + } + if (uniform_binary) rows.push_back({begin, end}); + } + if (rows.size() < 2 || timed_out()) return; + + const i_t n_rows = (i_t)rows.size(); + const i_t n_variables = fj_cpu.view.pb.n_variables; + std::vector degree(n_variables, 0); + std::vector endpoint_a(n_variables, -1); + std::vector endpoint_b(n_variables, -1); + for (i_t row = 0; row < n_rows; ++row) { + for (i_t p = rows[row].begin; p < rows[row].end; ++p) { + const i_t var = fj_cpu.h_variables[p]; + if (degree[var] == 0) endpoint_a[var] = row; + else if (degree[var] == 1) endpoint_b[var] = row; + ++degree[var]; + } + } + + struct edge_t { + int to, reverse, capacity; + f_t cost; + i_t var; + }; + auto add_edge = [](std::vector>& graph, int from, int to, f_t cost, i_t var) { + const int back = (int)graph[to].size(); + graph[from].push_back({to, back, 1, cost, var}); + graph[to].push_back({from, (int)graph[from].size() - 1, 0, -cost, -1}); + }; + + std::vector color(n_rows, -1); + std::vector state(n_variables, -1); + std::vector side_index(n_rows, -1); + std::vector component_rows, component_vars, left, right; + std::queue pending; + bool installed = false; + + for (i_t root = 0; root < n_rows && !timed_out(); ++root) { + if (color[root] >= 0) continue; + + component_rows.clear(); + component_vars.clear(); + color[root] = 0; + pending.push(root); + bool valid = true; + while (!pending.empty()) { + const i_t row = pending.front(); + pending.pop(); + component_rows.push_back(row); + for (i_t p = rows[row].begin; p < rows[row].end; ++p) { + const i_t var = fj_cpu.h_variables[p]; + // A variable outside exactly two rows is not an edge, and a self-loop cannot be 2-coloured. + if (degree[var] != 2 || endpoint_a[var] == endpoint_b[var]) { + valid = false; + continue; + } + if (endpoint_a[var] == row) component_vars.push_back(var); + const i_t other = endpoint_a[var] == row ? endpoint_b[var] : endpoint_a[var]; + if (color[other] < 0) { + color[other] = 1 - color[row]; + pending.push(other); + } else if (color[other] == color[row]) { + valid = false; + } + } + if ((component_rows.size() & 0x3FF) == 0 && timed_out()) return; + } + if (!valid || component_vars.empty()) continue; + + left.clear(); + right.clear(); + for (i_t row : component_rows) + (color[row] == 0 ? left : right).push_back(row); + if (left.size() != right.size()) continue; + for (i_t k = 0; k < (i_t)left.size(); ++k) + side_index[left[k]] = k; + for (i_t k = 0; k < (i_t)right.size(); ++k) + side_index[right[k]] = k; + + const int side = (int)left.size(); + const int source = 2 * side; + const int sink = source + 1; + std::vector> graph(sink + 1); + for (int k = 0; k < side; ++k) { + add_edge(graph, source, k, 0, -1); + add_edge(graph, side + k, sink, 0, -1); + } + // Every perfect matching uses exactly one variable edge per row, so shifting all of them by a + // constant moves every matching's cost equally and leaves the cheapest one unchanged. Shifting + // the negatives away is what lets the potentials below start at zero. + f_t cheapest = 0; + for (i_t var : component_vars) { + const f_t cost = fj_cpu.h_obj_coeffs[var]; + if (!isfinite(cost)) { + valid = false; + break; + } + cheapest = std::min(cheapest, cost); + } + if (!valid) continue; + const f_t shift = -cheapest; + + for (i_t var : component_vars) { + i_t a = endpoint_a[var]; + i_t b = endpoint_b[var]; + if (color[a] == 1) std::swap(a, b); + add_edge(graph, side_index[a], side + side_index[b], fj_cpu.h_obj_coeffs[var] + shift, var); + } + + // Node potentials hold every reduced cost at or above zero, which is what makes Dijkstra + // applicable. All shifted costs start non-negative, so the potentials start at zero. Rounding + // can still leave a tree edge fractionally negative once the potentials move, so relaxation + // below skips settled nodes: that keeps every predecessor older than its successor in + // settlement order, which is what makes the retrace terminate. + int flow = 0; + std::vector potential(graph.size(), 0); + std::vector distance(graph.size()); + std::vector previous_node(graph.size()); + std::vector previous_edge(graph.size()); + std::vector settled(graph.size()); + using heap_entry_t = std::pair; + + while (flow < side && !timed_out()) { + std::fill(distance.begin(), distance.end(), std::numeric_limits::infinity()); + std::fill(previous_node.begin(), previous_node.end(), -1); + std::fill(settled.begin(), settled.end(), 0); + distance[source] = 0; + std::priority_queue, std::greater> heap; + heap.push({0, source}); + + while (!heap.empty()) { + const auto [reached_at, from] = heap.top(); + heap.pop(); + if (settled[from]) continue; + settled[from] = 1; + for (int e = 0; e < (int)graph[from].size(); ++e) { + const auto& edge = graph[from][e]; + if (!edge.capacity || settled[edge.to]) continue; + const f_t reduced = edge.cost + potential[from] - potential[edge.to]; + cuopt_assert(reduced >= -1e-9 * std::max((f_t)1, std::abs(edge.cost)), + "potentials failed to keep the reduced cost non-negative"); + if (reached_at + reduced >= distance[edge.to]) continue; + distance[edge.to] = reached_at + reduced; + previous_node[edge.to] = from; + previous_edge[edge.to] = e; + heap.push({distance[edge.to], edge.to}); + } + } + if (previous_node[sink] < 0) break; + + for (int node = 0; node < (int)graph.size(); ++node) + if (isfinite(distance[node])) potential[node] += distance[node]; + + for (int node = sink; node != source; node = previous_node[node]) { + cuopt_assert(previous_node[node] >= 0, "augmenting path is broken"); + auto& edge = graph[previous_node[node]][previous_edge[node]]; + --edge.capacity; + ++graph[node][edge.reverse].capacity; + } + ++flow; + } + if (flow != side) continue; + + for (i_t var : component_vars) + state[var] = 0; + for (int node = 0; node < side; ++node) + for (const auto& edge : graph[node]) + if (edge.var >= 0 && edge.capacity == 0) state[edge.var] = 1; + installed = true; + } + if (!installed) return; + + recompute_lhs(fj_cpu); + const i_t baseline = fj_cpu.violated_constraints.size(); + const auto anchor = fj_cpu.h_assignment; + for (i_t var = 0; var < n_variables; ++var) + if (state[var] >= 0) fj_cpu.h_assignment[var] = state[var]; + + recompute_lhs(fj_cpu); + const i_t candidate = fj_cpu.violated_constraints.size(); + // Kept when it reaches feasibility outright, otherwise only on a strict gain. + if (candidate != 0 && candidate >= baseline) { + fj_cpu.h_assignment = anchor; + recompute_lhs(fj_cpu); + } + fj_cpu.h_best_assignment = fj_cpu.h_assignment; +} + +// Every variable to its lower bound, or its upper where the lower is infinite. +template +static void apply_lower_bound_seed(fj_cpu_climber_t& fj_cpu) +{ + if (fj_cpu.view.pb.nnz > fj_seed_nnz_limit) return; + + for (i_t var_idx = 0; var_idx < fj_cpu.view.pb.n_variables; ++var_idx) { + auto bounds = fj_cpu.h_var_bounds[var_idx].get(); + const f_t lower = get_lower(bounds); + const f_t upper = get_upper(bounds); + if (!isfinite(lower) && !isfinite(upper)) continue; + + f_t new_val = isfinite(lower) ? lower : upper; + if (is_integer_var(fj_cpu, var_idx)) new_val = std::round(new_val); + fj_cpu.h_assignment[var_idx] = new_val; + } + + recompute_lhs(fj_cpu); + fj_cpu.h_best_assignment = fj_cpu.h_assignment; +} + +// Constructively satisfies the equality rows that read as sum(x) = k over binaries sharing one +// coefficient: pick k members of each, narrowest rows first so the wide ones inherit the choices, +// and within a row the variables appearing in fewest other such rows. +template +static void apply_exact_k_seed(fj_cpu_climber_t& fj_cpu) +{ + if (fj_cpu.view.pb.nnz > fj_seed_nnz_limit) return; + + const auto started = std::chrono::steady_clock::now(); + auto timed_out = [&] { + return std::chrono::duration(std::chrono::steady_clock::now() - started).count() > + fj_exact_k_budget_s; + }; + + struct exact_k_row_t { + i_t k, begin, end; + }; + std::vector rows; + for (i_t row = 0; row < fj_cpu.view.pb.n_constraints; ++row) { + if ((row & 0xFFF) == 0 && timed_out()) return; + + const f_t lb = fj_cpu.h_cstr_lb[row]; + const f_t ub = fj_cpu.h_cstr_ub[row]; + if (!isfinite(lb) || !isfinite(ub) || std::abs(lb - ub) > fj_exact_k_tol) continue; + + const i_t begin = fj_cpu.h_offsets[row]; + const i_t end = fj_cpu.h_offsets[row + 1]; + if (end - begin < 2 || end - begin > fj_exact_k_max_width) continue; + + const f_t scale = fj_cpu.h_coefficients[begin]; + if (scale <= 0) continue; + bool uniform_binary = true; + for (i_t p = begin; p < end && uniform_binary; ++p) { + const i_t var = fj_cpu.h_variables[p]; + const f_t coeff = fj_cpu.h_coefficients[p]; + const f_t agreement = fj_exact_k_tol * std::max((f_t)1, std::abs(scale)); + uniform_binary = + fj_cpu.h_is_binary_variable[var] && coeff > 0 && std::abs(coeff - scale) <= agreement; + } + if (!uniform_binary) continue; + + const double cardinality = (double)lb / scale; + const i_t k = (i_t)std::lround(cardinality); + if (std::abs(cardinality - k) <= 1e-4 && k >= 0 && k <= end - begin) + rows.push_back({k, begin, end}); + } + if (rows.empty()) return; + + std::sort(rows.begin(), rows.end(), [](const exact_k_row_t& a, const exact_k_row_t& b) { + return a.end - a.begin < b.end - b.begin; + }); + + const i_t n_variables = fj_cpu.view.pb.n_variables; + std::vector degree(n_variables, 0); + for (const auto& row : rows) + for (i_t p = row.begin; p < row.end; ++p) + ++degree[fj_cpu.h_variables[p]]; + + std::vector state(n_variables, -1); + std::vector free_vars; + for (size_t index = 0; index < rows.size(); ++index) { + if ((index & 0xFFF) == 0 && timed_out()) break; + const auto& row = rows[index]; + + i_t selected = 0; + free_vars.clear(); + for (i_t p = row.begin; p < row.end; ++p) { + const i_t var = fj_cpu.h_variables[p]; + selected += state[var] == 1; + if (state[var] < 0) free_vars.push_back(var); + } + const i_t needed = row.k - selected; + if (needed < 0 || (i_t)free_vars.size() < needed) continue; + + std::sort(free_vars.begin(), free_vars.end(), [°ree](i_t a, i_t b) { + return degree[a] < degree[b]; + }); + for (i_t p = 0; p < (i_t)free_vars.size(); ++p) + state[free_vars[p]] = (int8_t)(p < needed); + } + + recompute_lhs(fj_cpu); + const i_t baseline = fj_cpu.violated_constraints.size(); + const auto anchor = fj_cpu.h_assignment; + for (i_t var = 0; var < n_variables; ++var) + if (state[var] >= 0) fj_cpu.h_assignment[var] = state[var]; + + recompute_lhs(fj_cpu); + if ((i_t)fj_cpu.violated_constraints.size() >= baseline) { + fj_cpu.h_assignment = anchor; + recompute_lhs(fj_cpu); + } + fj_cpu.h_best_assignment = fj_cpu.h_assignment; +} + +// One repair pass over the violated rows of a start that is mostly violated. Row sums are read from +// the lhs computed on entry, so a row does not see the repairs made for earlier rows; the revert +// below is what keeps that myopia from costing anything. +template +static void repair_difficult_anchor(fj_cpu_climber_t& fj_cpu) +{ + recompute_lhs(fj_cpu); + const i_t baseline = fj_cpu.violated_constraints.size(); + if (baseline == 0 || baseline <= fj_cpu.view.pb.n_constraints / fj_anchor_repair_violated_share) + return; + + const auto started = std::chrono::steady_clock::now(); + const auto anchor = fj_cpu.h_assignment; + const std::vector violated(fj_cpu.violated_constraints.begin(), + fj_cpu.violated_constraints.end()); + std::vector> candidates; + + for (i_t row : violated) { + if (std::chrono::duration(std::chrono::steady_clock::now() - started).count() > + fj_anchor_repair_budget_s) + break; + + const f_t lb = fj_cpu.h_cstr_lb[row]; + const f_t ub = fj_cpu.h_cstr_ub[row]; + f_t sum = fj_cpu.h_lhs[row]; + f_t target = 0; + f_t direction = 0; + if (sum < lb) { + direction = 1; + target = lb; + } else if (sum > ub) { + direction = -1; + target = ub; + } else { + continue; + } + + collect_row_repair_moves(fj_cpu, + fj_cpu.h_offsets[row], + fj_cpu.h_offsets[row + 1], + direction, + fj_exact_k_tol, + candidates); + for (const auto& move : candidates) { + if (direction > 0 ? sum >= target : sum <= target) break; + const f_t delta = move.new_val - (f_t)fj_cpu.h_assignment[move.var]; + sum += move.coeff * delta; + fj_cpu.h_assignment[move.var] = move.new_val; + } + } + + recompute_lhs(fj_cpu); + if ((i_t)fj_cpu.violated_constraints.size() >= baseline) { + fj_cpu.h_assignment = anchor; + recompute_lhs(fj_cpu); + } + fj_cpu.h_best_assignment = fj_cpu.h_assignment; +} + +// What makes one lane of a CPUFJ portfolio behave differently from another: which corner it starts +// from, how it samples, and how hard it pulls on the objective. Lane 0 keeps the anchor assignment +// so it is the lane every clone is built from. +template +void apply_lane_diversification(fj_cpu_climber_t& climber, int lane, int64_t base_seed) +{ + // Objective pressure across the portfolio, indexed by lane. Lanes 0 and 3 stay pure feasibility + // seekers until they cross, since the objective term only enters the score once the weight is + // positive; their nonzero floor then keeps a pull on the objective afterwards rather than letting + // smooth_weights decay it back to nothing. + const f_t obj_weight_ladder[4] = {0, 4, 32, 0}; + const f_t obj_weight_floor[4] = {1, 4, 32, 1}; + + // One structural start per lane; lanes 0, 4 and 7 keep the shared anchor here. Lane 4's is + // replaced inside its own task by the LP pump, so construction does not wait on an LP. + climber.use_lp_seed = lane % 8 == 4; + + // Half the portfolio searches the propagated model, half the model as parsed. + climber.use_bound_prop = lane % 2 == 0; + + climber.use_weight_donation = (lane % 8 == 5) || (lane % 8 == 6); + + // Only where the colouring came out; n_colors is zero when the structure declined it. + climber.use_move_batching = + climber.n_colors > 0 && ((lane % 8 == 2) || (lane % 8 == 6)); + climber.use_move_batching = true; + if (climber.n_colors == 0) climber.use_move_batching = false; + switch (lane % 8) { + case 1: apply_lock_weighted_seed(climber); break; + case 2: apply_aggressive_constraint_seed(climber); break; + case 3: apply_greedy_covering_seed(climber); break; + case 5: apply_bipartite_matching_seed(climber); break; + case 6: apply_objective_corner_seed(climber); break; + default: break; + } + + // Default: every climber identical apart from its seed and a random draw of the + // four sampling parameters. Diversification, decorrelated from the value RNG. + std::mt19937 rng(base_seed + 7919u * lane); + climber.mtm_viol_samples = std::uniform_int_distribution(15, 50)(rng); + climber.mtm_sat_samples = std::uniform_int_distribution(10, 30)(rng); + climber.nnz_samples = std::uniform_int_distribution(2000, 15000)(rng); + climber.perturb_interval = std::uniform_int_distribution(50, 500)(rng); + //climber.perturb_vars = std::uniform_int_distribution(2, 8)(rng); + + // The objective weight below is inert until a lane crosses, so without these the whole portfolio + // runs one weight decay, one tabu tenure and one restart policy while it is still infeasible. + // const double smoothing_ladder[8] = {0.0003, 0.0, 0.001, 0.003, 0.0001, 0.0006, 0.002, 0.0003}; + // const int tabu_min_ladder[8] = {3, 1, 5, 3, 2, 6, 4, 3}; + // const int tabu_max_ladder[8] = {13, 7, 21, 13, 10, 25, 17, 13}; + // const i_t restart_window_ladder[8] = {300, 150, 500, 300, 200, 600, 400, 300}; + // const f_t degrade_ratio_ladder[8] = {1.15, 1.05, 1.30, 1.15, 1.08, 1.40, 1.20, 1.15}; + // climber.settings.parameters.weight_smoothing_probability = smoothing_ladder[lane % 8]; + // climber.settings.parameters.tabu_tenure_min = tabu_min_ladder[lane % 8]; + // climber.settings.parameters.tabu_tenure_max = tabu_max_ladder[lane % 8]; + // climber.infeasible_restart_window = restart_window_ladder[lane % 8]; + // climber.infeasible_restart_degrade_ratio = degrade_ratio_ladder[lane % 8]; + + climber.enable_infeasible_repair = (lane % 8 == 1) || (lane % 8 == 5); + + climber.h_objective_weight = obj_weight_ladder[lane % 4]; + //climber.seed_objective_weight = obj_weight_floor[lane % 4]; +} + +// Portfolio construction for the standalone benchmark. Host logic, but it lives +// in a .cu because fj_cpu.cuh pulls in raft/util/cuda_dev_essentials.cuh through +// solution.cuh, which does not compile under the host compiler. Kept out of the +// header regardless: editing this file rebuilds one translation unit rather than +// the fifteen that including headers pull in. +template +void build_climber_portfolio(problem_t& problem, + solution_t& solution, + std::vector>& preemption_flags, + std::vector>>& climbers, + int64_t base_seed) +{ + const int n_climbers = static_cast(climbers.size()); + + for (int k = 0; k < n_climbers; ++k) + preemption_flags[k].store(false); + + // cuopt::seed_generator::get_seed() steps a non-atomic global, so every lane's seed is drawn here + // in lane order before any concurrent construction below. + std::vector lane_seed(n_climbers); + for (int k = 0; k < n_climbers; ++k) + lane_seed[k] = cuopt::seed_generator::get_seed(); + + // Lane 0 is a genuine dependency: it host-copies the problem and every other lane clones it. + { + fj_settings_t settings; + settings.seed = (int)lane_seed[0]; + climbers[0] = init_fj_cpu_standalone(problem, solution, preemption_flags[0], settings); + // Runs before the clones are taken, so every lane starts from the repaired anchor. + apply_exact_k_seed(*climbers[0]); + repair_difficult_anchor(*climbers[0]); + apply_lane_diversification(*climbers[0], 0, base_seed); + } + + // The remaining lanes depend only on lane 0's finished, read-only template, and the O(nnz) clone + // and seed passes are otherwise paid serially on one thread while the other pinned CPUs idle. +#ifdef _OPENMP +#pragma omp parallel for num_threads(std::max(1, n_climbers - 1)) schedule(static) +#endif + for (int k = 1; k < n_climbers; ++k) { + fj_settings_t settings; + settings.seed = (int)lane_seed[k]; + climbers[k] = init_fj_cpu_clone(*climbers[0], preemption_flags[k], settings); + apply_lane_diversification(*climbers[k], k, base_seed); + } + + auto shared = std::make_shared>(); + for (int k = 0; k < n_climbers; ++k) + climbers[k]->shared_incumbent = shared; +} + +#if MIP_INSTANTIATE_FLOAT +template void apply_lane_diversification(fj_cpu_climber_t&, int, int64_t); +template void build_climber_portfolio( + problem_t&, solution_t&, std::vector>&, + std::vector>>&, int64_t); +#endif + +#if MIP_INSTANTIATE_DOUBLE +template void apply_lane_diversification(fj_cpu_climber_t&, int, int64_t); +template void build_climber_portfolio( + problem_t&, solution_t&, std::vector>&, + std::vector>>&, int64_t); +#endif + } // namespace cuopt::mathematical_optimization::mip diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh index 411b4083f7..df86602d45 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu.cuh @@ -8,8 +8,10 @@ #pragma once #include +#include #include #include +#include #include #include #include @@ -24,6 +26,102 @@ namespace cuopt::mathematical_optimization::mip { template class probing_cache_t; +template +struct host_contiguous_set_t { + void resize(i_t max_size) + { + cuopt_assert(max_size >= 0, "invalid max size"); + contents.clear(); + contents.reserve(max_size); + index_map.assign(max_size, -1); + is_member.assign(max_size, 0); + } + + void clear() + { + for (i_t val : contents) { + index_map[val] = -1; + is_member[val] = 0; + } + contents.clear(); + } + + void insert(i_t val) + { + cuopt_assert(val >= 0 && val < max_size(), "Value is out of bounds"); + cuopt_assert(!contains(val), "Value already exists"); + index_map[val] = contents.size(); + is_member[val] = 1; + contents.push_back(val); + } + + void remove(i_t val) + { + cuopt_assert(val >= 0 && val < max_size(), "Value is out of bounds"); + cuopt_assert(contains(val), "Value not found"); + const i_t idx = index_map[val]; + const i_t last_val = contents.back(); + contents[idx] = last_val; + index_map[last_val] = idx; + contents.pop_back(); + index_map[val] = -1; + is_member[val] = 0; + } + + bool contains(i_t val) const + { + cuopt_assert(val >= 0 && val < max_size(), "Value is out of bounds"); + return is_member[val] != 0; + } + + auto begin() const { return contents.begin(); } + auto end() const { return contents.end(); } + i_t size() const { return contents.size(); } + i_t max_size() const { return index_map.size(); } + bool empty() const { return contents.empty(); } + + std::vector contents; + std::vector index_map; + std::vector is_member; +}; + +constexpr double fj_obj_mult_min = 0.25; +constexpr double fj_obj_mult_max = 4.0; + +// Best feasible assignment found by any lane of one portfolio. A lane publishes its own +// improvements and adopts a better one when it perturbs, so a lane that has stalled resumes from +// the portfolio's progress instead of its own. Lanes run concurrently, so which lane observes +// which incumbent depends on scheduling: a portfolio that shares is not run-to-run reproducible. +template +struct fj_cpu_shared_incumbent_t { + // True when the candidate beat the shared best, in which case it was stored. + bool publish(f_t candidate_objective, const std::vector& candidate) + { + // Unlocked reject first: the publish sites are hot on instances that improve in tiny steps. + if (!(candidate_objective < objective.load(std::memory_order_relaxed))) return false; + std::lock_guard lock(guard); + if (!(candidate_objective < objective.load(std::memory_order_relaxed))) return false; + assignment = candidate; + objective.store(candidate_objective, std::memory_order_relaxed); + return true; + } + + // True when the shared best beat local_objective, in which case it was copied into destination. + bool adopt(f_t local_objective, std::vector& destination) + { + if (!(objective.load(std::memory_order_relaxed) < local_objective)) return false; + std::lock_guard lock(guard); + if (!(objective.load(std::memory_order_relaxed) < local_objective)) return false; + cuopt_assert(assignment.size() == destination.size(), "shared incumbent size mismatch"); + destination = assignment; + return true; + } + + std::mutex guard; + std::vector assignment; + std::atomic objective{std::numeric_limits::infinity()}; +}; + // NOTE: this seems an easy pick for reflection/xmacros once this is available (C++26?) // Maintaining a single source of truth for all members would be nice template @@ -44,6 +142,7 @@ struct fj_cpu_climber_t { ADD_INSTRUMENTED(h_var_bounds), ADD_INSTRUMENTED(h_cstr_lb), ADD_INSTRUMENTED(h_cstr_ub), + ADD_INSTRUMENTED(h_cstr_tolerance), ADD_INSTRUMENTED(h_var_types), ADD_INSTRUMENTED(h_is_binary_variable), ADD_INSTRUMENTED(h_objective_vars), @@ -64,8 +163,8 @@ struct fj_cpu_climber_t { ADD_INSTRUMENTED(h_cstr_right_weights), ADD_INSTRUMENTED(h_assignment), ADD_INSTRUMENTED(h_best_assignment), - ADD_INSTRUMENTED(cached_cstr_bounds), - ADD_INSTRUMENTED(iter_mtm_vars)}; + ADD_INSTRUMENTED(h_best_infeasible_assignment), + ADD_INSTRUMENTED(cached_cstr_bounds)}; #undef ADD_INSTRUMENTED } @@ -90,6 +189,8 @@ struct fj_cpu_climber_t { ins_vector::type> h_var_bounds; ins_vector h_cstr_lb; ins_vector h_cstr_ub; + // get_corrected_tolerance of each row, held because the bounds it derives from never move. + ins_vector h_cstr_tolerance; ins_vector h_var_types; ins_vector h_is_binary_variable; ins_vector h_objective_vars; @@ -118,15 +219,50 @@ struct fj_cpu_climber_t { ins_vector h_assignment; ins_vector h_best_assignment; f_t h_objective_weight; + // Lower bound h_objective_weight decays to, so a lane seeded with objective pressure keeps it. + f_t seed_objective_weight{0}; + // Mean absolute nonzero objective coefficient; the unit of the objective score term. + f_t obj_magnitude{1}; f_t h_incumbent_objective; + // Kahan compensation for h_incumbent_objective, mirroring h_lhs_sumcomp. Reset wherever the + // objective is re-derived from the assignment. + f_t h_objective_sumcomp{0}; f_t h_best_objective; - i_t last_feasible_entrance_iter{0}; i_t iterations; - std::unordered_set violated_constraints; - std::unordered_set satisfied_constraints; + host_contiguous_set_t violated_constraints; + host_contiguous_set_t satisfied_constraints; bool feasible_found{false}; bool trigger_early_lhs_recomputation{false}; + + // Move batching over a colouring of the variable co-occurrence graph, where each row is a clique. + // Same colour means no shared row, so a batch of same-coloured moves has disjoint row support. + bool use_move_batching{false}; + i_t n_colors{0}; + std::vector h_var_color; + // Per variable, the best move seen since the epoch below, and the sum of its incident row + // versions at that moment. The entry is usable while both still match. + std::vector h_var_best_score; + std::vector h_var_best_delta; + std::vector h_var_best_stamp; + std::vector h_var_best_rowsum; + int64_t var_best_epoch{1}; + // Variables that entered the table with a positive score, bucketed by colour. Stale entries are + // skipped at selection, so each bucket carries the epoch it was last cleared in. + std::vector> h_color_candidates; + std::vector h_color_epoch; + // Membership is stamped separately from validity: a variable consumed by a batch is invalidated + // while staying in its bucket, so it cannot be enqueued twice in one epoch. + std::vector h_var_bucket_stamp; + int64_t n_batch_attempts{0}; + int64_t n_batched_moves{0}; + // Companions per attempt, in unit bins. The last bin saturates, so max_batch_size carries the + // tail exactly. + std::vector batch_size_hist; + int64_t max_batch_size{0}; f_t total_violations{0}; + // Kahan compensation for total_violations, mirroring h_lhs_sumcomp. Reset wherever the total is + // re-derived from the violated set. + f_t total_violations_sumcomp{0}; // Timing data structures std::vector find_lift_move_times; @@ -136,35 +272,89 @@ struct fj_cpu_climber_t { std::vector update_weights_times; std::vector compute_score_times; - i_t hit_count{0}; - i_t miss_count{0}; + int64_t hit_count{0}; + int64_t miss_count{0}; i_t candidate_move_hits[3] = {0}; i_t candidate_move_misses[3] = {0}; - // vector is actually likely beneficial here since we're memory bound - std::vector flip_move_computed; + // Hot-loop accounting, reported off the clock by the standalone harness. + int64_t n_moves_applied{0}; + int64_t apply_move_nnz{0}; + int64_t n_mtm_calls{0}; + // Row entries find_mtm_move visits, and the ones the per-row cap kept it from visiting. + int64_t mtm_row_entries{0}; + int64_t mtm_entries_capped{0}; + int64_t n_compute_score_calls{0}; + int64_t compute_score_nnz{0}; + int64_t n_version_bumps_apply{0}; + int64_t n_version_bumps_weights{0}; + int64_t n_mtm_cache_invalidations{0}; + int64_t n_lhs_recompute_total{0}; + int64_t n_lhs_recompute_periodic{0}; + int64_t n_lhs_recompute_bigval{0}; + int64_t n_lhs_recompute_perturb{0}; + int64_t n_lhs_recompute_restart{0}; + i_t lhs_refresh_period_used{0}; + + // A variable's flip move has already been considered when its stamp equals flip_move_epoch, + // which advances once per applied move. An epoch avoids clearing an n_variables bitmap per move. + std::vector flip_move_stamp; + int64_t flip_move_epoch{1}; + + // Continuous objective variables bounded by their rows only opposite the objective's pull, so + // the tightest row bound is their value. epigraph_push is +1 pushing up, -1 pushing down. + std::vector epigraph_push; + std::vector epigraph_vars; + int64_t n_epigraph_projections{0}; // CSR nnz offset -> (delta, score) std::vector> cached_mtm_moves; + // Entry i is live only while cached_mtm_moves_version[i] == h_cstr_version of i's row. + std::vector cached_mtm_moves_version; + std::vector h_cstr_version; + // CSC (transposed!) nnz-offset-indexed constraint bounds (lb, ub) // std::pair better compile down to 16 bytes!! GCC do your job! ins_vector> cached_cstr_bounds; - std::vector var_bitmap; - ins_vector iter_mtm_vars; - // Scratch reused by the binary 2-opt search, which runs at every local minimum std::vector two_opt_target_cstrs; std::vector two_opt_first_vars; std::vector> two_opt_partners; std::vector> two_opt_row_deltas; + ins_vector h_best_infeasible_assignment; + f_t best_infeasible_severity{std::numeric_limits::infinity()}; + f_t checkpoint_severity{std::numeric_limits::infinity()}; + i_t iters_since_infeasible_improve{0}; + i_t restores_since_improvement{0}; + i_t max_restores_since_improvement{0}; + int64_t n_checkpoint_restores{0}; + int64_t n_checkpoint_snapshots{0}; + i_t mtm_viol_samples{25}; i_t mtm_sat_samples{15}; i_t nnz_samples{50000}; i_t perturb_interval{100}; + // Number of variables randomized by one perturbation. + i_t perturb_vars{2}; + // One lane replaces its start with a rounded LP relaxation, solved inside that lane's own task so + // portfolio construction does not wait on an LP. + bool use_lp_seed{false}; + // Half the lanes narrow their own domains by activity propagation before searching, so the + // portfolio covers both the propagated and the as-parsed model. + bool use_bound_prop{false}; + // Two lanes move weight from satisfied rows into the violated ones while still infeasible. + bool use_weight_donation{false}; + // Enables the binary engine's infeasible-phase pair repair. Per lane, since the pair scan costs + // iterations that a well-tuned single-flip lane would rather spend elsewhere. + bool enable_infeasible_repair{false}; + i_t infeasible_restart_window{300}; + i_t infeasible_restart_max_streak{20}; + f_t infeasible_restart_degrade_ratio{1.15}; + f_t infeasible_checkpoint_refresh_ratio{0.99}; i_t log_interval{1000}; i_t diversity_callback_interval{3000}; @@ -176,6 +366,10 @@ struct fj_cpu_climber_t { std::function&)> diversity_callback{nullptr}; std::string log_prefix{""}; + // Held with the other lanes of the same portfolio. Null when the climber runs alone, which is + // what keeps a solo climber reproducible. + std::shared_ptr> shared_incumbent; + // Work unit tracking for deterministic synchronization std::atomic work_units_elapsed{0.0}; double work_unit_bias{1.5}; // Bias factor to keep CPUFJ ahead of B&B @@ -195,8 +389,8 @@ struct fj_cpu_climber_t { i_t iterations_since_best{0}; // Cache and locality tracking - i_t hit_count_window_start{0}; - i_t miss_count_window_start{0}; + int64_t hit_count_window_start{0}; + int64_t miss_count_window_start{0}; std::unordered_set unique_cstrs_accessed_window; std::unordered_set unique_vars_accessed_window; @@ -231,4 +425,36 @@ std::unique_ptr> init_fj_cpu_standalone( std::atomic& preemption_flag, fj_settings_t settings = fj_settings_t{}); +// Copies a climber that has already paid the O(nnz) problem construction. Everything the engine +// reads is host-owned, so this needs neither a problem handle nor any GPU work. +template +std::unique_ptr> init_fj_cpu_clone( + const fj_cpu_climber_t& tmpl, + std::atomic& preemption_flag, + fj_settings_t settings = fj_settings_t{}); + +// Per-lane behaviour for a CPUFJ portfolio, shared by every caller that races several climbers so +// the composition cannot drift between them. +template +void apply_lane_diversification(fj_cpu_climber_t& climber, int lane, int64_t base_seed); + +// Builds the climber portfolio the standalone benchmark races: how many distinct +// behaviours, what parameters each gets, whether they are randomized or +// specialized. Defined in fj_cpu_portfolio.cpp -- host code, compiled by the host +// compiler, so editing it is markedly cheaper than editing this header. Runs +// inside the measured window. +template +void build_climber_portfolio(problem_t& problem, + solution_t& solution, + std::vector>& preemption_flags, + std::vector>>& climbers, + int64_t base_seed); + +template +std::unique_ptr> init_fj_cpu_from_optimization_problem( + const optimization_problem_t& problem, + const typename mip_solver_settings_t::tolerances_t& tolerances, + std::atomic& preemption_flag, + fj_settings_t settings = fj_settings_t{}); + } // namespace cuopt::mathematical_optimization::mip diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu new file mode 100644 index 0000000000..7fe3bb1e39 --- /dev/null +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cu @@ -0,0 +1,2176 @@ +/* clang-format off */ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +/* clang-format on */ + +#include "fj_cpu_binary.cuh" + +#include "feasibility_jump.cuh" +#include "fj_cpu.cuh" + +#include +#include +#include + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cuopt::mathematical_optimization::mip { + +const char* fj_binary_reject_name(fj_binary_reject_t reason) +{ + switch (reason) { + case fj_binary_reject_t::none: return "none"; + case fj_binary_reject_t::empty_problem: return "empty problem"; + case fj_binary_reject_t::non_binary_var: return "non-binary variable"; + case fj_binary_reject_t::fractional_coefficient: return "fractional coefficient"; + case fj_binary_reject_t::coefficient_out_of_range: return "coefficient wider than int16"; + case fj_binary_reject_t::fractional_row_bound: return "fractional row bound"; + case fj_binary_reject_t::row_bound_out_of_range: return "row bound outside int32"; + case fj_binary_reject_t::lhs_headroom: return "row sum|coef| exceeds int32 headroom"; + case fj_binary_reject_t::narrow_check_failed: return "narrowing check failed"; + } + return "unknown"; +} + +// work unit proxy. will likely require a lot of tuning +constexpr double fj_bin_bytes_per_nnz = 16.0; + +// Tabu for binary variables, expressed as a ring buffer +// There can be at most max_tenure tabu'd variables at any given time. +// since max_tenure << n_vars, it's cheaper to maintain a ring buffer than a full array +// and it allows smaller instances to become L1 resident +struct fj_bin_tabu_t { + static constexpr int32_t ring_size = 16; + static constexpr int32_t max_tenure = ring_size; + // Headroom so iter + tenure - iter_bias still fits uint16 when iter - iter_bias is at the rebase + // threshold. + static constexpr int32_t window = + (int32_t)std::numeric_limits::max() - max_tenure; + + std::vector flip_until; + std::vector last_flip; + int32_t iter_bias{0}; + + int32_t ring_var[ring_size]; + int32_t ring_expiry[ring_size]; + + + void resize(int32_t n) + { + flip_until.assign(n, 0); + last_flip.assign(n, 0); + clear_ring(); + iter_bias = 0; + } + + void clear(int32_t iter) + { + std::fill(flip_until.begin(), flip_until.end(), (uint16_t)0); + std::fill(last_flip.begin(), last_flip.end(), 0); + clear_ring(); + iter_bias = iter; + } + + void clear_ring() + { + for (int32_t i = 0; i < ring_size; ++i) { + ring_var[i] = -1; + ring_expiry[i] = 0; + } + } + + void on_flip(int32_t v, int32_t iter, int32_t tenure) + { + flip_until[v] = (uint16_t)(iter + tenure - iter_bias); + last_flip[v] = iter; + + // keep only one tabu entry per var + for (int32_t i = 0; i < ring_size; ++i) { + if (ring_var[i] == v) ring_var[i] = -1; + } + + const int32_t slot = iter & (ring_size - 1); + ring_var[slot] = v; + ring_expiry[slot] = iter + tenure; + } + + // replace the scores of tabu'd variable with sentinel values + int32_t block_tabu(int32_t iter, + int64_t* var_score, + int32_t (&saved_var)[ring_size], + int64_t (&saved_score)[ring_size]) const + { + int32_t k = 0; + for (int32_t i = 0; i < ring_size; ++i) { + const int32_t v = ring_var[i]; + if (v >= 0 && ring_expiry[i] > iter) { + saved_var[k] = v; + saved_score[k] = var_score[v]; + var_score[v] = fj_bin_score_invalid; + ++k; + } + } + return k; + } + + // reverse the above operation. + static void unblock_tabu(int32_t k, + int64_t* var_score, + const int32_t (&saved_var)[ring_size], + const int64_t (&saved_score)[ring_size]) + { + for (int32_t i = k - 1; i >= 0; --i) var_score[saved_var[i]] = saved_score[i]; + } + + + bool blocked(int32_t v, int32_t iter, bool localmin) const + { + return localmin ? (iter == last_flip[v] + 1) + : ((uint16_t)(iter - iter_bias) < flip_until[v]); + } + + // rebase the iteration bias value every 64k iter + void maybe_rebase(int32_t iter) + { + if ((int64_t)iter - iter_bias <= window) return; + const uint16_t shift = (uint16_t)(iter - iter_bias); + for (uint16_t& fu : flip_until) fu = (fu > shift) ? (uint16_t)(fu - shift) : (uint16_t)0; + iter_bias = iter; + } +}; + +// Narrowed problem: one-sided rows, integer coefficients, CSR plus its transpose. +template +struct fj_bin_problem_t { + int32_t n_variables{0}; + int32_t n_constraints{0}; + int32_t nnz{0}; + + std::vector offsets; + std::vector variables; + std::vector coefficients; + + std::vector reverse_offsets; + std::vector reverse_constraints; + std::vector reverse_to_csr; + + // Per incidence, for the vectorized row walk: the coefficient and the row's cmax, both replicated + // in transpose order so the walk reads them at unit stride instead of gathering per row. Both are + // structural. + std::vector reverse_coefficients; + std::vector incident_row_cmax; + + std::vector bound; + std::vector cmax; + std::vector initial_weight; + + std::vector objective; + std::vector objective_vars; + + // Cardinality census, for the repair-pair gate. A cardinality row is an equality over binaries + // sharing one coefficient, so a variable of degree two across them can only be switched on by + // switching exactly one other off: the exchange a pair can represent. + int32_t n_exchange_vars{0}; + int32_t max_card_degree{0}; + + // Empty unless encoded, when every engine variable is one bit of a bounded general integer and + // original[j] = var_offset[j] + sum of bit_weight[b] * assign[b] over the bits b owned by j. + bool encoded{false}; + int32_t n_original{0}; + std::vector var_offset; + std::vector bit_owner; + std::vector bit_weight; + std::vector orig_objective; +}; + +// Result of the width-independent eligibility scan. +struct fj_bin_scan_t { + fj_binary_reject_t reject{fj_binary_reject_t::none}; + int coefficient_bits{0}; + int32_t n_split_constraints{0}; + int32_t bad_row{-1}; + int32_t bad_var{-1}; + std::vector row_scale; +}; + +constexpr int64_t fj_bin_scale_cap = std::numeric_limits::max(); + +// DDFW and restart have no general-path equivalent, so their defaults live here until there is a +// reason to promote them alongside the other FJ knobs. +constexpr int32_t fj_bin_ddfw_init = 10; // initial weight, also the donation floor +constexpr int32_t fj_bin_ddfw_transfer = 1; +constexpr int32_t fj_bin_ddfw_donor_samples = 4; +constexpr int32_t fj_bin_restart_period = 5000000; + +// Escalation threshold and step, in infeasible local minima without a severity improvement. +constexpr int32_t fj_bin_ddfw_escalate_after = 2000; +constexpr int32_t fj_bin_ddfw_escalate_max = 100; + +// The same, in feasible local minima without a best-objective improvement. +constexpr int32_t fj_bin_obj_stall_after = 50; +constexpr int32_t fj_bin_obj_escalate_max = 10; + +// Infeasible-region kick: stall, cooldown, post-restart quiet window, rows drawn, flips per row. +constexpr int32_t fj_bin_kick_after = 200; +constexpr int32_t fj_bin_kick_cooldown = 200; +constexpr int32_t fj_bin_kick_restart_guard = 50; +constexpr int32_t fj_bin_kick_rows = 3; +constexpr int32_t fj_bin_kick_vars_per_row = 2; + +// Infeasible-phase pair repair: iterations between attempts, violated rows sampled per attempt, +// and the pool size the O(pool^2) pair scan is capped to. +constexpr int32_t fj_bin_repair_interval = 20; +constexpr int32_t fj_bin_repair_max_rows = 4; +constexpr int32_t fj_bin_repair_max_vars = 12; + +// Structure the pair repair needs before it is worth running: enough variables that are shared by +// exactly two cardinality rows, and no variable shared by so many that closing the exchange takes a +// chain rather than a pair. +constexpr int32_t fj_bin_repair_min_exchange_vars = 64; +constexpr int32_t fj_bin_repair_max_card_degree = 4; + +// Candidate draws per 2-opt lift search. +constexpr int32_t fj_bin_2opt_candidates = 64; +// prefetch distance +// TODO: check if it actually matters at all for performance +constexpr int32_t fj_bin_pf_dist = 8; + +constexpr int32_t fj_bin_base_limit = 1 << 16; +constexpr int32_t fj_bin_bonus_limit = 1 << 14; + +static inline bool fj_bin_in_int32(double v) +{ + return v >= (double)INT32_MIN && v <= (double)INT32_MAX; +} + +// Tile width of the argmax sweep, in variables: min(algorithm target, L1-residency cap). +// +// The target is about the shape of the sweep rather than cache capacity -- it sets how often the +// running maximum is raised, which is what bounds the index re-scan -- and 256 is the measured +// optimum. The cap is a residency guard, and it is the reason this is not simply a constant: the +// re-scan pays off only because it revisits a tile that is still L1-hot, so the tile must not be +// wide enough to spill. It bites only on a small L1, where an unguarded 256 would push the re-scan +// out to L2 and cost more than the split saves. +// +// Bytes per variable is the score array alone. Tabu does not appear: the sweep reads var_score +// only, with the handful of tabu variables held at the invalid sentinel across it, so flip_until is +// never touched here. +constexpr int32_t fj_bin_argmax_tile_target = 256; +constexpr int32_t fj_bin_argmax_tile_cap_k = 4; + +static int32_t fj_bin_argmax_tile() +{ +#ifdef _SC_LEVEL1_DCACHE_SIZE + long l1 = sysconf(_SC_LEVEL1_DCACHE_SIZE); +#else + long l1 = 0; +#endif + if (l1 <= 0) l1 = 32768; // fallback: 32 KiB, the common x86 L1d + const int32_t bpv = (int32_t)sizeof(int32_t); + const int32_t cap = (int32_t)(l1 / (fj_bin_argmax_tile_cap_k * bpv)); + int32_t t = fj_bin_argmax_tile_target < cap ? fj_bin_argmax_tile_target : cap; + t &= ~15; // whole vectors + return t < 16 ? 16 : t; +} + +// Width-independent eligibility scan over the climber's host mirrors. Mutates nothing. +template +static fj_bin_scan_t fj_bin_scan(const fj_cpu_climber_t& c) +{ + fj_bin_scan_t out; + const int32_t n = c.view.pb.n_variables; + const int32_t m = c.view.pb.n_constraints; + if (n <= 0 || m <= 0) { + out.reject = fj_binary_reject_t::empty_problem; + return out; + } + + const double tol = c.view.pb.tolerances.integrality_tolerance; + const auto& is_binary_variable = c.h_is_binary_variable; + cuopt_assert((int32_t)is_binary_variable.size() == n, "is_binary_variable size mismatch"); + + for (int32_t v = 0; v < n; ++v) { + // Populated at climber init with integer_equal on [0,1] bounds. + if (!is_binary_variable[v]) { + out.reject = fj_binary_reject_t::non_binary_var; + out.bad_var = v; + return out; + } + } + + const auto& offsets = c.h_offsets; + const auto& reverse_offsets = c.h_reverse_offsets; + const auto& reverse_constraints = c.h_reverse_constraints; + const auto& coeffs = c.h_coefficients; + const auto& cstr_lb = c.h_cstr_lb; + const auto& cstr_ub = c.h_cstr_ub; + + cuopt_assert( + thrust::all_of( + thrust::host, + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(n), + [&reverse_offsets, &reverse_constraints](int32_t v) { + const auto first = reverse_constraints.begin() + reverse_offsets[v]; + const auto last = reverse_constraints.begin() + reverse_offsets[v + 1]; + return std::adjacent_find(first, last) == last; + }), + "duplicate variable in CSR row"); + + double max_abs_coefficient = 0; + std::vector row_values; + for (int32_t r = 0; r < m; ++r) { + const double lb = cstr_lb[r]; + const double ub = cstr_ub[r]; + const bool lb_fin = std::isfinite(lb); + const bool ub_fin = std::isfinite(ub); + const double sides[2] = {lb, ub}; + const bool finite[2] = {lb_fin, ub_fin}; + + bool fractional_coefficient_seen = false; + bool integral = true; + for (int32_t k = offsets[r]; k < offsets[r + 1]; ++k) { + if (!is_integer(coeffs[k], tol)) { + fractional_coefficient_seen = true; + integral = false; + break; + } + } + for (int s = 0; s < 2 && integral; ++s) { + if (finite[s] && !is_integer(sides[s], tol)) integral = false; + } + + double row_s = 1.0; + if (!integral) { + row_values.clear(); + for (int32_t k = offsets[r]; k < offsets[r + 1]; ++k) row_values.push_back(coeffs[k]); + for (int s = 0; s < 2; ++s) { + if (finite[s]) row_values.push_back(sides[s]); + } + row_s = find_scaling_rational(row_values, + /*maxscale=*/1.0 / tol, + /*maxdnom=*/fj_bin_scale_cap, + /*maxfinal=*/(double)fj_bin_scale_cap, + /*intcheck_tol=*/tol); + if (!std::isfinite(row_s) || row_s <= 0.0) { + out.reject = fractional_coefficient_seen ? fj_binary_reject_t::fractional_coefficient + : fj_binary_reject_t::fractional_row_bound; + out.bad_row = r; + return out; + } + if (out.row_scale.empty()) out.row_scale.assign(m, 1.0); + out.row_scale[r] = row_s; + } + + double row_abs_sum = 0; + double row_lhs_min = 0; + double row_lhs_max = 0; + for (int32_t k = offsets[r]; k < offsets[r + 1]; ++k) { + const double a = row_s * coeffs[k]; + cuopt_assert(is_integer(a, tol), "row scaling left a fractional coefficient"); + const double integral_a = std::round(a); + const double abs_a = std::fabs(integral_a); + row_abs_sum += abs_a; + if (integral_a < 0) { + row_lhs_min += integral_a; + } else { + row_lhs_max += integral_a; + } + if (abs_a > max_abs_coefficient) max_abs_coefficient = abs_a; + } + + // A binary assignment can drive lhs to sum|coef|; keep that inside the int32 accumulator with + // room to spare. The int8-only reference engine never needed this bound. + if (row_abs_sum > (double)(INT32_MAX / 2)) { + out.reject = fj_binary_reject_t::lhs_headroom; + out.bad_row = r; + return out; + } + + for (int s = 0; s < 2; ++s) { + if (!finite[s]) continue; + const double scaled_side = row_s * sides[s]; + cuopt_assert(is_integer(scaled_side, tol), "row scaling left a fractional row bound"); + if (!fj_bin_in_int32(std::round(scaled_side))) { + out.reject = fj_binary_reject_t::row_bound_out_of_range; + out.bad_row = r; + return out; + } + const double integral_side = std::round(scaled_side); + const double min_slack = + s == 0 ? row_lhs_min - integral_side : integral_side - row_lhs_max; + const double max_slack = + s == 0 ? row_lhs_max - integral_side : integral_side - row_lhs_min; + if (!fj_bin_in_int32(min_slack) || !fj_bin_in_int32(max_slack)) { + out.reject = fj_binary_reject_t::lhs_headroom; + out.bad_row = r; + return out; + } + } + // Free rows are dropped: trivially satisfied, contributing nothing to the search. + out.n_split_constraints += (int32_t)lb_fin + (int32_t)ub_fin; + } + + if (out.n_split_constraints <= 0) { + out.reject = fj_binary_reject_t::empty_problem; + return out; + } + + if (max_abs_coefficient <= 127.0) { + out.coefficient_bits = 8; + } else if (max_abs_coefficient <= 32767.0) { + out.coefficient_bits = 16; + } else { + out.reject = fj_binary_reject_t::coefficient_out_of_range; + } + return out; +} + +// Build the narrowed, one-sided problem. Called only after fj_bin_scan cleared the instance, so a +// failing check here is a self-consistency bug and refuses the fast path rather than truncating. +template +static bool fj_bin_narrow(const fj_cpu_climber_t& c, + const fj_bin_scan_t& scan, + fj_bin_problem_t& pb) +{ + const int32_t n_split = scan.n_split_constraints; + const int32_t n = c.view.pb.n_variables; + const int32_t m = c.view.pb.n_constraints; + const double tol = c.view.pb.tolerances.integrality_tolerance; + + const auto& offsets = c.h_offsets; + const auto& variables = c.h_variables; + const auto& coeffs = c.h_coefficients; + const auto& cstr_lb = c.h_cstr_lb; + const auto& cstr_ub = c.h_cstr_ub; + const auto& left_w = c.h_cstr_left_weights; + const auto& right_w = c.h_cstr_right_weights; + const auto& obj = c.h_obj_coeffs; + + pb.n_variables = n; + pb.n_constraints = n_split; + pb.offsets.assign(1, 0); + pb.offsets.reserve(n_split + 1); + pb.bound.reserve(n_split); + pb.cmax.reserve(n_split); + pb.initial_weight.reserve(n_split); + + std::vector incoming_weight; + incoming_weight.reserve(n_split); + + // Each split row inherits the weight of the side it came from: left is the lower-bound side, + // right the upper. + // + // Both sides are stored as a'x <= b. The lower-bound side is negated on the way in, which costs + // nothing because each side already gets its own copy of the row, and it leaves the slack as + // bound - lhs everywhere -- so no per-row sign reaches the engine at all. Negation is safe on both + // fields: the scan admits |coef| up to 127 for int8 and 32767 for int16, and the bound is checked + // for int32 range after negating. + auto emit = [&](int32_t r, double side_bound, long side, double weight) -> bool { + const double s = scan.row_scale.empty() ? 1.0 : scan.row_scale[r]; + coef_t row_cmax = 1; + for (int32_t k = offsets[r]; k < offsets[r + 1]; ++k) { + const double a = s * coeffs[k]; + const long ai = side * std::lround(a); + if (!is_integer(a, tol) || ai < std::numeric_limits::min() || + ai > std::numeric_limits::max()) { + return false; + } + pb.variables.push_back(variables[k]); + pb.coefficients.push_back((coef_t)ai); + const coef_t abs_a = (coef_t)std::labs(ai); + if (abs_a > row_cmax) row_cmax = abs_a; + } + const long b = side * std::lround(s * side_bound); + if (!fj_bin_in_int32((double)b)) return false; + pb.offsets.push_back((int32_t)pb.variables.size()); + pb.bound.push_back((int32_t)b); + pb.cmax.push_back(row_cmax); + incoming_weight.push_back(weight); + return true; + }; + + for (int32_t r = 0; r < m; ++r) { + const double lb = cstr_lb[r]; + const double ub = cstr_ub[r]; + if (std::isfinite(lb) && !emit(r, lb, -1, left_w[r])) return false; + if (std::isfinite(ub) && !emit(r, ub, 1, right_w[r])) return false; + } + if ((int32_t)pb.bound.size() != n_split) return false; + pb.nnz = (int32_t)pb.variables.size(); + + // One vector of padding past nnz, so the row kernel can load and store whole vectors at the last + // row without running off the end and can therefore mask its remainder rather than peeling it + // into a scalar tail. The padding is never read as data: every lane past a row's end is excluded + // from the gather, the scatter and the store by the row-length mask. + pb.variables.resize(pb.nnz + fj_bin_simd_padding, 0); + pb.coefficients.resize(pb.nnz + fj_bin_simd_padding, (coef_t)0); + + // Scale the incoming weights into the DDFW band by one global factor, so relative structure + // survives while every row clears the donation floor. Capped so the largest scaled weight stays + // clear of packed-score saturation; where the cap binds, the smallest rows sit below the floor. + // TODO: bound the scaled weights by derivation instead of leaving them open. The packed score + // holds while a variable's aggregate base stays under 2^16, and that aggregate is bounded by the + // sum of weights over the rows the variable appears in, so 2^16 / max_var_degree gives a per-row + // bound computable here from the transpose. Left uncapped for now, matching the reference + // engine, which shipped with its weight cap disabled and relied on the end-of-solve saturation + // report to say whether a bound was needed. + double w_min = std::numeric_limits::infinity(); + for (double w : incoming_weight) { + if (w > 0 && w < w_min) w_min = w; + } + double scale = 1.0; + if (std::isfinite(w_min) && w_min > 0) { + scale = (double)fj_bin_ddfw_init / w_min; + if (scale < 1.0) scale = 1.0; + } + for (double w : incoming_weight) { + int32_t scaled = w > 0 ? (int32_t)std::lround(w * scale) : fj_bin_ddfw_init; + if (scaled < 1) scaled = 1; + pb.initial_weight.push_back(scaled); + } + + // Transpose, plus the reverse-nnz to CSR-nnz map the apply path uses to store the flipped + // variable's own score delta. + pb.reverse_offsets.assign(n + 1, 0); + for (int32_t k = 0; k < pb.nnz; ++k) pb.reverse_offsets[pb.variables[k] + 1]++; + for (int32_t v = 0; v < n; ++v) pb.reverse_offsets[v + 1] += pb.reverse_offsets[v]; + pb.reverse_constraints.resize(pb.nnz); + pb.reverse_coefficients.resize(pb.nnz); + pb.reverse_to_csr.resize(pb.nnz); + pb.incident_row_cmax.resize(pb.nnz); + { + std::vector cursor(pb.reverse_offsets.begin(), pb.reverse_offsets.begin() + n); + for (int32_t r = 0; r < n_split; ++r) { + for (int32_t k = pb.offsets[r]; k < pb.offsets[r + 1]; ++k) { + const int32_t slot = cursor[pb.variables[k]]++; + pb.reverse_constraints[slot] = r; + pb.reverse_coefficients[slot] = pb.coefficients[k]; + pb.reverse_to_csr[slot] = k; + pb.incident_row_cmax[slot] = pb.cmax[r]; + } + } + } + // Lookahead room for the row walk: a vector of overhang for the kernel's unit-stride loads, and + // the prefetch distance the scalar path uses. Reads land on row 0, harmlessly, and every lane past + // a variable's range is masked out of the gather, the scatter and the compress. + const int32_t rpad = + fj_bin_pf_dist > fj_bin_simd_padding ? fj_bin_pf_dist : fj_bin_simd_padding; + pb.reverse_constraints.resize(pb.nnz + rpad, 0); + pb.reverse_coefficients.resize(pb.nnz + rpad, (coef_t)0); + pb.incident_row_cmax.resize(pb.nnz + rpad, (coef_t)1); + + pb.objective.resize(n); + for (int32_t v = 0; v < n; ++v) { + pb.objective[v] = obj[v]; + if (pb.objective[v] != 0.0) pb.objective_vars.push_back(v); + } + + // Every variable here is binary, so an equality row whose members share one coefficient reads as + // a cardinality constraint. Counted on the unscaled row: the row scale multiplies bound and + // coefficients alike and leaves the ratio alone. + { + std::vector card_degree(n, 0); + for (int32_t r = 0; r < m; ++r) { + const double lb = cstr_lb[r]; + const double ub = cstr_ub[r]; + if (!std::isfinite(lb) || !std::isfinite(ub) || std::fabs(lb - ub) > tol) continue; + + const int32_t begin = offsets[r]; + const int32_t end = offsets[r + 1]; + if (end - begin < 2) continue; + + const double shared = coeffs[begin]; + if (std::fabs(shared) <= tol) continue; + const double k = lb / shared; + if (k < 1.0 - tol || std::fabs(k - std::round(k)) > tol) continue; + + bool uniform = true; + for (int32_t p = begin; p < end && uniform; ++p) { + const double a = coeffs[p]; + uniform = std::fabs(a - shared) <= tol * std::max(1.0, std::fabs(shared)); + } + if (!uniform) continue; + + for (int32_t p = begin; p < end; ++p) + card_degree[variables[p]]++; + } + for (int32_t v = 0; v < n; ++v) { + if (card_degree[v] == 2) ++pb.n_exchange_vars; + if (card_degree[v] > pb.max_card_degree) pb.max_card_degree = card_degree[v]; + } + } + return true; +} + + +// Bit budget for one general integer's domain. +constexpr int32_t fj_bin_encode_max_bits = 16; +// Cap on the bit-variable count relative to the model's variable count, bounding the SIMD sweep. +constexpr int64_t fj_bin_encode_max_growth = 6; + +// Bits needed to represent the integers 0..W inclusive. +static inline int32_t fj_bin_encode_nbits(int64_t W) +{ + int32_t bits = 0; + while (((int64_t)1 << bits) - 1 < W) ++bits; + return bits; +} + +// Encodes an all-integer model with bounded general integers into bits: x in [L,U] becomes +// x = L + sum_k w_k b_k over weights 1, 2, ..., 2^(nbits-2), R, with R closing the range at W = U-L. +template +static bool fj_bin_encode(const fj_cpu_climber_t& c, + fj_bin_problem_t& pb, + int& coefficient_bits) +{ + const int32_t n = c.view.pb.n_variables; + const int32_t m = c.view.pb.n_constraints; + if (n <= 0 || m <= 0) return false; + + const double tol = c.view.pb.tolerances.integrality_tolerance; + + const auto& var_bounds = c.h_var_bounds; + const auto& var_types = c.h_var_types; + const auto& offsets = c.h_offsets; + const auto& variables = c.h_variables; + const auto& coeffs = c.h_coefficients; + const auto& cstr_lb = c.h_cstr_lb; + const auto& cstr_ub = c.h_cstr_ub; + const auto& left_w = c.h_cstr_left_weights; + const auto& right_w = c.h_cstr_right_weights; + const auto& obj = c.h_obj_coeffs; + + std::vector lower(n); + std::vector upper(n); + std::vector nbits(n); + std::vector bit_start(n); + int64_t total_bits = 0; + for (int32_t v = 0; v < n; ++v) { + if (var_types[v] != var_t::INTEGER) return false; + auto bounds = var_bounds[v]; + const double x = (double)cuopt::get_lower(bounds); + const double y = (double)cuopt::get_upper(bounds); + if (!std::isfinite(x) || !std::isfinite(y) || y < x) return false; + if (!is_integer(x, tol) || !is_integer(y, tol)) return false; + + lower[v] = std::round(x); + upper[v] = std::round(y); + const int64_t W = (int64_t)(upper[v] - lower[v]); + + nbits[v] = fj_bin_encode_nbits(W); + if (nbits[v] > fj_bin_encode_max_bits) return false; + bit_start[v] = (int32_t)total_bits; + total_bits += nbits[v]; + } + if (total_bits <= 0 || total_bits > (int64_t)INT32_MAX / 2) return false; + if (total_bits > fj_bin_encode_max_growth * (int64_t)n) return false; + + const int32_t n_bits = (int32_t)total_bits; + + pb.encoded = true; + pb.n_original = n; + pb.var_offset = lower; + pb.orig_objective.assign(n, 0.0); + pb.bit_owner.assign(n_bits, 0); + pb.bit_weight.assign(n_bits, 0.0); + for (int32_t v = 0; v < n; ++v) { + int64_t covered = 0; + const int64_t W = (int64_t)(upper[v] - lower[v]); + for (int32_t k = 0; k < nbits[v]; ++k) { + const int64_t w = k + 1 < nbits[v] ? (int64_t)1 << k : W - covered; + covered += w; + pb.bit_owner[bit_start[v] + k] = v; + pb.bit_weight[bit_start[v] + k] = (double)w; + } + cuopt_assert(covered == W, "bit weights do not close the domain exactly"); + } + + pb.n_variables = n_bits; + pb.offsets.assign(1, 0); + pb.bound.clear(); + pb.cmax.clear(); + pb.initial_weight.clear(); + pb.variables.clear(); + pb.coefficients.clear(); + + std::vector incoming_weight; + std::vector row_values; + double max_abs_coefficient = 0; + + // One side of one row, as a'b <= bound in bit space with sum(a_j L_j) folded into the bound. + auto emit = [&](int32_t r, double side_bound, long side, double weight) -> bool { + double fixed = 0; + for (int32_t k = offsets[r]; k < offsets[r + 1]; ++k) + fixed += coeffs[k] * lower[variables[k]]; + const double folded_bound = side_bound - fixed; + + row_values.clear(); + bool integral = is_integer(folded_bound, tol); + for (int32_t k = offsets[r]; k < offsets[r + 1]; ++k) { + row_values.push_back(coeffs[k]); + if (!is_integer(coeffs[k], tol)) integral = false; + } + row_values.push_back(folded_bound); + + double s = 1.0; + if (!integral) { + s = find_scaling_rational( + row_values, 1.0 / tol, fj_bin_scale_cap, (double)fj_bin_scale_cap, tol); + if (!std::isfinite(s) || s <= 0.0) return false; + } + + coef_t row_cmax = 1; + double row_abs_sum = 0; + for (int32_t k = offsets[r]; k < offsets[r + 1]; ++k) { + const int32_t v = variables[k]; + const double a = s * coeffs[k]; + if (!is_integer(a, tol)) return false; + const long ai = std::lround(a); + for (int32_t bk = 0; bk < nbits[v]; ++bk) { + const int32_t bit = bit_start[v] + bk; + const long scaled = side * ai * std::lround(pb.bit_weight[bit]); + const long abs_a = std::labs(scaled); + // Bounded by magnitude, so cmax below and the negated side both stay representable. + if (abs_a > (long)std::numeric_limits::max()) return false; + pb.variables.push_back(bit); + pb.coefficients.push_back((coef_t)scaled); + + if (abs_a > (long)row_cmax) row_cmax = (coef_t)abs_a; + row_abs_sum += (double)abs_a; + if ((double)abs_a > max_abs_coefficient) max_abs_coefficient = (double)abs_a; + } + } + if (row_abs_sum > (double)(INT32_MAX / 2)) return false; + + const double scaled_bound = side * s * folded_bound; + if (!is_integer(scaled_bound, tol)) return false; + const double bound = std::round(scaled_bound); + if (!fj_bin_in_int32(bound)) return false; + // A bit assignment can drive lhs anywhere in [-row_abs_sum, row_abs_sum]. + if (!fj_bin_in_int32(bound - row_abs_sum) || !fj_bin_in_int32(bound + row_abs_sum)) return false; + + pb.offsets.push_back((int32_t)pb.variables.size()); + pb.bound.push_back((int32_t)bound); + pb.cmax.push_back(row_cmax); + incoming_weight.push_back(weight); + return true; + }; + + for (int32_t r = 0; r < m; ++r) { + const double lb = cstr_lb[r]; + const double ub = cstr_ub[r]; + if (std::isfinite(lb) && !emit(r, lb, -1, left_w[r])) return false; + if (std::isfinite(ub) && !emit(r, ub, 1, right_w[r])) return false; + } + pb.n_constraints = (int32_t)pb.bound.size(); + if (pb.n_constraints <= 0) return false; + pb.nnz = (int32_t)pb.variables.size(); + + if (max_abs_coefficient <= 127.0) { + coefficient_bits = 8; + } else if (max_abs_coefficient <= 32767.0) { + coefficient_bits = 16; + } else { + return false; + } + + pb.variables.resize(pb.nnz + fj_bin_simd_padding, 0); + pb.coefficients.resize(pb.nnz + fj_bin_simd_padding, (coef_t)0); + + double w_min = std::numeric_limits::infinity(); + for (double w : incoming_weight) { + if (w > 0 && w < w_min) w_min = w; + } + double scale = 1.0; + if (std::isfinite(w_min) && w_min > 0) { + scale = (double)fj_bin_ddfw_init / w_min; + if (scale < 1.0) scale = 1.0; + } + for (double w : incoming_weight) { + int32_t scaled = w > 0 ? (int32_t)std::lround(w * scale) : fj_bin_ddfw_init; + if (scaled < 1) scaled = 1; + pb.initial_weight.push_back(scaled); + } + + pb.reverse_offsets.assign(n_bits + 1, 0); + for (int32_t k = 0; k < pb.nnz; ++k) pb.reverse_offsets[pb.variables[k] + 1]++; + for (int32_t v = 0; v < n_bits; ++v) pb.reverse_offsets[v + 1] += pb.reverse_offsets[v]; + pb.reverse_constraints.resize(pb.nnz); + pb.reverse_coefficients.resize(pb.nnz); + pb.reverse_to_csr.resize(pb.nnz); + pb.incident_row_cmax.resize(pb.nnz); + { + std::vector cursor(pb.reverse_offsets.begin(), pb.reverse_offsets.begin() + n_bits); + for (int32_t r = 0; r < pb.n_constraints; ++r) { + for (int32_t k = pb.offsets[r]; k < pb.offsets[r + 1]; ++k) { + const int32_t slot = cursor[pb.variables[k]]++; + pb.reverse_constraints[slot] = r; + pb.reverse_coefficients[slot] = pb.coefficients[k]; + pb.reverse_to_csr[slot] = k; + pb.incident_row_cmax[slot] = pb.cmax[r]; + } + } + } + const int32_t rpad = fj_bin_pf_dist > fj_bin_simd_padding ? fj_bin_pf_dist : fj_bin_simd_padding; + pb.reverse_constraints.resize(pb.nnz + rpad, 0); + pb.reverse_coefficients.resize(pb.nnz + rpad, (coef_t)0); + pb.incident_row_cmax.resize(pb.nnz + rpad, (coef_t)1); + + pb.objective.assign(n_bits, 0.0); + pb.objective_vars.clear(); + for (int32_t v = 0; v < n; ++v) { + pb.orig_objective[v] = obj[v]; + if (obj[v] == 0.0) continue; + for (int32_t bk = 0; bk < nbits[v]; ++bk) { + const int32_t bit = bit_start[v] + bk; + pb.objective[bit] = obj[v] * pb.bit_weight[bit]; + if (pb.objective[bit] != 0.0) pb.objective_vars.push_back(bit); + } + } + + // The cardinality census only reads as a count on rows of plain binaries. + pb.n_exchange_vars = 0; + pb.max_card_degree = 0; + return true; +} + +// The integer engine. Feasibility is an exact compare against one bound per row, so there is no +// tolerance arithmetic and no compensated summation anywhere below. +template +struct fj_bin_engine_t { + fj_bin_problem_t pb; + // The only mutable per-row state besides the slack. Everything else the apply path once read + // per row now reaches it at unit stride: bound stayed in pb, where only the rebuild paths need + // it, and cmax went to pb.incident_row_cmax, replicated per incidence. + std::vector row_weight; + + // Per row, bound - lhs: negative exactly when the row is violated, and moved by a flip by exactly + // -reverse_coefficients. The only mutable state the vectorized walk gathers. + std::vector row_slack; + + std::vector assign; + std::vector best_assign; + std::shared_ptr> shared_incumbent; + // Staging for an adopted assignment, which arrives as f_t. Sized only when sharing is on. + std::vector adopt_buffer; + std::vector seed_assign; // restart target + std::vector assign_i32; // gather mirror for the SIMD patch (Batch B) + + std::vector best_infeasible_assign; + int64_t best_infeasible_severity{std::numeric_limits::max()}; + int64_t checkpoint_severity{std::numeric_limits::max()}; + int32_t iters_since_infeasible_improve{0}; + int32_t restores_since_improvement{0}; + + std::vector var_score; // live feasibility score of flipping each variable + std::vector nnz_score_delta; // per CSR nnz: last score delta of variables[k] in its row + + // Objective half of the move score, held live so a weighted global scan can stay vectorized. + // Its support is pb.objective_vars, so entries outside that set are zero for the whole solve. + std::vector obj_base_score; + std::vector combined_score; + // Objective weight obj_base_score was built for; -1 marks it stale. + int32_t obj_base_weight{-1}; + + fj_bin_tabu_t tabu; + + std::vector is_violated; + std::vector violated_list; + std::vector vpos; + // Duplicate guard for find_move_in_rows, its only reader. Zero everywhere outside that function, + // which clears what it set before returning. + std::vector var_bitmap; + + // One generator advanced across the whole search, rather than one re-seeded per call site per + // iteration. Re-seeding from `seed + iters` gave every call site in an iteration the identical + // stream, and a 624-word Mersenne state was being built and discarded on every move selection. + raft::random::PCGenerator rng{0, 0, 0}; + std::vector sample_buf; // move-selection row sample, reused to keep the loop allocation-free + + int32_t objective_weight{0}; + int32_t seed_objective_weight{0}; + // Feasible local minima since best_objective last moved, and the value it was last seen at. + int32_t iterations_at_same_objective{0}; + double last_best_objective{std::numeric_limits::infinity()}; + // Mean absolute nonzero objective coefficient; the unit of the objective score term. + double obj_magnitude{1.0}; + double incumbent_objective{0}; + // sum(obj_j * L_j), folded out of the encoded objective and carried here so both tracked + // objectives hold the model's own value. Zero on the all-binary path. + double objective_offset{0}; + double best_objective{std::numeric_limits::infinity()}; + int32_t max_weight{1}; + bool feasible_found{false}; + + int32_t iters{0}; + // Iterations since best_objective last moved. Counts iterations, unlike + // iterations_at_same_objective, so it is comparable against perturb_interval. + int32_t iters_since_best{0}; + int32_t last_restart_iter{0}; + int32_t last_kick_iter{0}; + int64_t nnz_touched{0}; + + // Denominator for the ops-per-nnz roofline: nonzeros the row kernel actually processes, and the + // rows walked to find them. Unlike nnz_touched these are not mixed with the full-matrix rebuilds. + int64_t nnz_patched{0}; + int64_t rows_walked{0}; + + int64_t n_checkpoint_restores{0}; + int64_t n_checkpoint_snapshots{0}; + int32_t max_restores_since_improvement{0}; + + // Tile width for the argmax sweep, in variables. Set at init from fj_bin_argmax_tile(). + int32_t argmax_tile{fj_bin_argmax_tile_target}; + + // Settings read at solve entry, where the climber carries populated values. + int32_t seed{0}; + int32_t tabu_tenure_min{3}; + int32_t tabu_tenure_max{13}; + int32_t perturb_interval{100}; + int32_t mtm_viol_samples{25}; + int32_t mtm_sat_samples{15}; + bool enable_infeasible_repair{false}; + int32_t last_repair_iter{0}; + int32_t infeasible_restart_window{300}; + int32_t infeasible_restart_max_streak{20}; + double infeasible_restart_degrade_ratio{1.15}; + double infeasible_checkpoint_refresh_ratio{0.99}; + double breakthrough_margin{1e-4}; + + int32_t max_aggregate_base{0}; + int32_t max_aggregate_bonus{0}; + + int coefficient_bits() const { return 8 * (int)sizeof(coef_t); } + + // Largest per-variable aggregate base and bonus under the final weights and assignment, in raw + // int32. The packed representation is only order-preserving while these stay inside their + // limits, and weights grow without a cap, so this is the reading that says whether the packing + // survived the run. + // Independent audit of the incumbent at end of solve. Recomputes every row's lhs and the objective + // from best_assign alone, trusting nothing the incremental path maintained: not the live lhs, not + // violated_list, not the running incumbent_objective. Accumulates in int64 so an int32 lhs + // overflow the eligibility scan was supposed to preclude would show up here rather than wrap + // silently. Runs once per solve, so its cost is not on any hot path. + void verify_incumbent(fj_cpu_climber_t& climber) const + { + if (!feasible_found) return; + + int32_t n_violated = 0; + int64_t worst = 0; + bool lhs_overflow = false; + for (int32_t r = 0; r < pb.n_constraints; ++r) { + int64_t lhs = 0; + for (int32_t k = pb.offsets[r]; k < pb.offsets[r + 1]; ++k) { + lhs += (int64_t)pb.coefficients[k] * (int64_t)best_assign[pb.variables[k]]; + } + if (lhs < INT32_MIN || lhs > INT32_MAX) lhs_overflow = true; + const int64_t slack = (int64_t)pb.bound[r] - lhs; + if (slack < 0) { + ++n_violated; + if (-slack > worst) worst = -slack; + } + } + + double objective = objective_offset; + for (int32_t v = 0; v < pb.n_variables; ++v) objective += pb.objective[v] * (double)best_assign[v]; + const double drift = std::fabs(objective - best_objective); + + if (n_violated != 0 || lhs_overflow || drift > 1e-6) { + CUOPT_LOG_ERROR( + "%sCPUFJ[bin%d] incumbent audit FAILED: %d violated rows (worst %lld), lhs overflow %d, " + "objective recomputed %.17g vs tracked %.17g (drift %g)", + climber.log_prefix.c_str(), + coefficient_bits(), + n_violated, + (long long)worst, + (int)lhs_overflow, + objective, + best_objective, + drift); + } else { + CUOPT_LOG_DEBUG("%sCPUFJ[bin%d] incumbent audit ok: feasible, objective %.17g (drift %g)", + climber.log_prefix.c_str(), + coefficient_bits(), + objective, + drift); + } + } + + void compute_saturation() + { + int32_t peak_base = 0, peak_bonus = 0; + for (int32_t v = 0; v < pb.n_variables; ++v) { + const int8_t flip = (int8_t)(1 - 2 * assign[v]); + int32_t agg_base = 0, agg_bonus = 0; + for (int32_t i = pb.reverse_offsets[v]; i < pb.reverse_offsets[v + 1]; ++i) { + const int32_t r = pb.reverse_constraints[i]; + const int32_t os = row_slack[r]; + const int32_t ns = os - (int32_t)pb.reverse_coefficients[i] * flip; + int32_t base = 0, bonus = 0; + fj_bin_score_delta_parts(os, ns, row_weight[r], base, bonus); + agg_base += base; + agg_bonus += bonus; + } + const int32_t abs_base = agg_base < 0 ? -agg_base : agg_base; + const int32_t abs_bonus = agg_bonus < 0 ? -agg_bonus : agg_bonus; + if (abs_base > peak_base) peak_base = abs_base; + if (abs_bonus > peak_bonus) peak_bonus = abs_bonus; + } + max_aggregate_base = peak_base; + max_aggregate_bonus = peak_bonus; + } + + void set_violated(int32_t r) + { + if (!is_violated[r]) { + is_violated[r] = 1; + vpos[r] = (int32_t)violated_list.size(); + violated_list.push_back(r); + } + } + + void set_satisfied(int32_t r) + { + if (is_violated[r]) { + is_violated[r] = 0; + const int32_t p = vpos[r]; + const int32_t last = violated_list.back(); + violated_list[p] = last; + vpos[last] = p; + violated_list.pop_back(); + vpos[r] = -1; + } + } + + void rebuild_scores() + { + std::fill(var_score.begin(), var_score.end(), 0); + for (int32_t r = 0; r < pb.n_constraints; ++r) { + const int32_t weight = row_weight[r]; + const int32_t os = row_slack[r]; + for (int32_t k = pb.offsets[r]; k < pb.offsets[r + 1]; ++k) { + const int32_t v = pb.variables[k]; + const int32_t flip = 1 - 2 * assign[v]; + const int32_t ns = os - (int32_t)pb.coefficients[k] * flip; + const int64_t p = fj_bin_packed_score_delta(os, ns, weight); + nnz_score_delta[k] = p; + var_score[v] += p; + } + } + nnz_touched += pb.nnz; + } + + void recompute_slack() + { + violated_list.clear(); + std::fill(is_violated.begin(), is_violated.end(), (uint8_t)0); + for (int32_t r = 0; r < pb.n_constraints; ++r) { + int32_t lhs = 0; + for (int32_t k = pb.offsets[r]; k < pb.offsets[r + 1]; ++k) + lhs += (int32_t)pb.coefficients[k] * assign[pb.variables[k]]; + const int32_t slack = pb.bound[r] - lhs; + row_slack[r] = slack; + if (slack < 0) set_violated(r); + } + incumbent_objective = objective_offset; + for (int32_t v = 0; v < pb.n_variables; ++v) incumbent_objective += pb.objective[v] * assign[v]; + nnz_touched += pb.nnz; + rebuild_scores(); + // Every caller of this reached it by replacing the assignment wholesale, so the cached + // per-variable flip directions no longer describe it. + obj_base_weight = -1; + } + + // Base field of the objective term: the weight, signed by the direction of the gain and scaled by + // how large that gain is against the model's typical coefficient. Depends only on the variable's + // own value and the weight, which is what lets a global scan cache it. + int64_t objective_base(int32_t v, int8_t delta) const + { + const double obj_diff = pb.objective[v] * delta; + if (obj_diff == 0) return 0; + cuopt_assert(obj_magnitude > 0, "objective magnitude unit must be positive"); + const double rel = std::fabs(obj_diff) / obj_magnitude; + const double mult = + rel < fj_obj_mult_min ? fj_obj_mult_min : (rel > fj_obj_mult_max ? fj_obj_mult_max : rel); + const double raw = objective_weight * mult; + cuopt_assert(fj_bin_in_int32(raw), "scaled objective weight out of int32 range"); + const int32_t scaled = (int32_t)std::lround(raw); + return (int64_t)(obj_diff < 0 ? scaled : -scaled) * fj_bin_score_k; + } + + int64_t objective_terms(int32_t v, int8_t delta) const + { + const double obj_diff = pb.objective[v] * delta; + int32_t bonus = 0; + const bool old_better = incumbent_objective < best_objective; + const bool new_better = incumbent_objective + obj_diff < best_objective; + if (!old_better && new_better) { + bonus += objective_weight; + } else if (old_better && !new_better) { + bonus -= objective_weight; + } + return objective_base(v, delta) + bonus; + } + + int64_t flip_objective_base(int32_t v) const + { + return objective_base(v, (int8_t)(1 - 2 * assign[v])); + } + + // Only the objective variables are written: the rest of the array is zero from init onwards. + void ensure_objective_base() + { + if (obj_base_weight == objective_weight) return; + for (int32_t v : pb.objective_vars) obj_base_score[v] = flip_objective_base(v); + obj_base_weight = objective_weight; + } + + int64_t full_score(int32_t v, int8_t delta) const + { + if (objective_weight == 0) return var_score[v]; + return var_score[v] + objective_terms(v, delta); + } + + bool tabu_blocked(int32_t v, bool localmin) const { return tabu.blocked(v, iters, localmin); } + + void apply_move(int32_t var, int8_t delta, fj_cpu_climber_t& climber) + { + const int8_t new_val = (int8_t)(assign[var] + delta); + const int8_t new_flip = (int8_t)(1 - 2 * new_val); + const int32_t ob = pb.reverse_offsets[var], oe = pb.reverse_offsets[var + 1]; + int64_t own_score = 0; + + // The tail writes a score delta through int32_t* and calls out to the patch, either of which may + // alias a vector's internal pointer as far as the compiler can prove. Without these locals it + // reloads every base pointer below out of `this` on each visit. + int32_t* const weight_p = row_weight.data(); + int32_t* const slack_p = row_slack.data(); + const int32_t* const rcon_p = pb.reverse_constraints.data(); + const coef_t* const skv_p = pb.reverse_coefficients.data(); + const coef_t* const rcmax_p = pb.incident_row_cmax.data(); + const int32_t* const rcsr_p = pb.reverse_to_csr.data(); + const int32_t* const offsets_p = pb.offsets.data(); + const int32_t* const vars_p = pb.variables.data(); + const coef_t* const coefs_p = pb.coefficients.data(); + int64_t* const var_score_p = var_score.data(); + int64_t* const nnz_delta_p = nnz_score_delta.data(); + const int32_t* const assign_p = assign_i32.data(); + + // Everything a visit still needs once its slack has been advanced. Shared by the two arms below + // so the walk's shape is the only thing that differs between them. + auto finish = [&](int32_t ii) { + const int32_t r = rcon_p[ii]; + const int32_t weight = weight_p[r]; + const int32_t skv = (int32_t)skv_p[ii]; + const int32_t new_slack = slack_p[r]; + const int32_t old_slack = new_slack + skv * delta; + + // A row can only cross its boundary if the flip moves it by at least the distance to it, so + // every transition is inside this list and none was lost with the rows the walk absorbed. + if (new_slack < 0 && old_slack >= 0) { + set_violated(r); + } else if (new_slack >= 0 && old_slack < 0) { + set_satisfied(r); + } + + // The mirror of the walk's deep_sat test. Kept here rather than there because it fires on + // 0.02% of visits and guards the widest rows in the matrix: measured, moving it into the + // vector loop costs more in the 85% case than it saves in the 0.02% one. + const int32_t margin = (int32_t)rcmax_p[ii]; + if (!(old_slack < -margin && new_slack < -margin)) { + const int32_t kb = offsets_p[r], ke = offsets_p[r + 1]; + // TODO: check that this may not cause AVX512 powerdown overheads if the AVX2 row/AVX512 row ratio is unbalanced + fj_bin_patch_row(vars_p, + coefs_p, + kb, + ke, + var_score_p, + nnz_delta_p, + assign_p, + weight, + new_slack, + var); + nnz_touched += ke - kb; + nnz_patched += ke - kb; + } + + // The flipped variable's own score delta. Zero on the rows the walk absorbed -- deeply + // satisfied both ways -- and already stored as zero there. + const int64_t pv = fj_bin_packed_score_delta(new_slack, new_slack - skv * new_flip, weight); + own_score += pv; + nnz_delta_p[rcsr_p[ii]] = pv; + }; + + // A tile at a time: the kernel advances every slack in the tile and reports back only the visits + // that left the row within reach of its boundary, which on supportcase22 is 15.1% of them. The + // buffer is a stack array rather than one sized to the widest reverse degree because the tail + // runs between tiles, which is also what keeps the patch calls out of the vector loop. + // + // Unconditional: a scalar arm for short ranges was tried and never won. Sweeping the degree + // below which apply_move walked the rows itself, bnatt400 degraded monotonically from 14.43M to + // 14.19M iterations/s as the threshold rose from 0 to 64, and crypt16 and supportcase22 were + // flat. At a median degree of 13 and 7 respectively, one gather still beats that many dependent + // scalar load-modify-stores, because it breaks the dependence chain through row_slack rather + // than following it. + int32_t tile_incidence[fj_bin_walk_tile]; + for (int32_t t0 = ob; t0 < oe; t0 += fj_bin_walk_tile) { + const int32_t t1 = (t0 + fj_bin_walk_tile < oe) ? t0 + fj_bin_walk_tile : oe; + const int32_t n_tail = + fj_bin_walk_rows(slack_p, rcon_p, skv_p, rcmax_p, t0, t1, delta, tile_incidence); + for (int32_t j = 0; j < n_tail; ++j) finish(tile_incidence[j]); + } + nnz_touched += oe - ob; + rows_walked += oe - ob; + + assign[var] = new_val; + assign_i32[var] = new_val; + var_score[var] = own_score; + incumbent_objective += pb.objective[var] * delta; + // Only this variable's flip direction moved, so a live cache needs one entry rewritten. + if (obj_base_weight == objective_weight && pb.objective[var] != 0) + obj_base_score[var] = flip_objective_base(var); + + if (violated_list.empty() && incumbent_objective < best_objective) { + best_objective = incumbent_objective; + best_assign = assign; + feasible_found = true; + iters_since_best = 0; + report_incumbent(climber); + } + + const int32_t tenure = + tabu_tenure_min + (int32_t)(rng.next_u32() % (uint32_t)(tabu_tenure_max - tabu_tenure_min)); + tabu.on_flip(var, iters, tenure); + } + + // Publish a new best into the climber, which owns the reporting contract. + void report_incumbent(fj_cpu_climber_t& climber) + { + auto& h_assign = climber.h_assignment; + auto& h_best = climber.h_best_assignment; + if (pb.encoded) { + for (int32_t v = 0; v < pb.n_original; ++v) { + h_assign[v] = (f_t)pb.var_offset[v]; + h_best[v] = (f_t)pb.var_offset[v]; + } + for (int32_t b = 0; b < pb.n_variables; ++b) { + if (!assign[b]) continue; + const int32_t v = pb.bit_owner[b]; + h_assign[v] += (f_t)pb.bit_weight[b]; + h_best[v] += (f_t)pb.bit_weight[b]; + } + } else { + for (int32_t v = 0; v < pb.n_variables; ++v) { + h_assign[v] = (f_t)assign[v]; + h_best[v] = (f_t)assign[v]; + } + } + climber.h_incumbent_objective = (f_t)incumbent_objective; + climber.h_best_objective = (f_t)best_objective; + climber.feasible_found = true; + if (shared_incumbent) { shared_incumbent->publish((f_t)best_objective, h_best); } + // Emitted once per improvement so the benchmark harness can reconstruct the + // incumbent trajectory exactly, rather than sampling it at log_interval. + CUOPT_LOG_DEBUG("%sCPUFJ[bin%d] new incumbent: objective %.17g", + climber.log_prefix.c_str(), + coefficient_bits(), + best_objective); + if (climber.improvement_callback) { + const double work_units = climber.work_units_elapsed.load(std::memory_order_acquire); + climber.improvement_callback((f_t)best_objective, h_best, work_units); + } + } + + void reweight_constraint(int32_t r, int32_t new_weight) + { + if (new_weight == row_weight[r]) return; + row_weight[r] = new_weight; + if (new_weight > max_weight) max_weight = new_weight; + // The slack is unchanged here, and no variable is excluded, so skip_var matches no index. + const int32_t kb = pb.offsets[r], ke = pb.offsets[r + 1]; + fj_bin_patch_row(pb.variables.data(), + pb.coefficients.data(), + kb, + ke, + var_score.data(), + nnz_score_delta.data(), + assign_i32.data(), + new_weight, + row_slack[r], + -1); + nnz_touched += ke - kb; + nnz_patched += ke - kb; + } + + // DDFW: every violated row gains weight taken from a satisfied neighbour above the donation + // floor, so total weight is roughly conserved and differentiation stays local to the hard region. + // Unit transfers stop moving the landscape on a long stall, so the amount grows with the stall. + int32_t ddfw_transfer() const + { + if (iters_since_infeasible_improve <= fj_bin_ddfw_escalate_after) return fj_bin_ddfw_transfer; + const int32_t over = iters_since_infeasible_improve - fj_bin_ddfw_escalate_after; + const int32_t steps = over / fj_bin_ddfw_escalate_after + 1; + const int32_t scale = steps < fj_bin_ddfw_escalate_max ? steps : fj_bin_ddfw_escalate_max; + return fj_bin_ddfw_transfer * scale; + } + + void update_weights() + { + const int32_t transfer = ddfw_transfer(); + // Donors must stay above the floor, or weights go negative and every base score inverts. + const int32_t donor_floor = fj_bin_ddfw_init + transfer - 1; + + for (int32_t cf : violated_list) { + reweight_constraint(cf, row_weight[cf] + transfer); + const int32_t vo = pb.offsets[cf], ve = pb.offsets[cf + 1]; + if (ve <= vo) continue; + int32_t best_donor = -1, best_w = donor_floor; + for (int32_t s = 0; s < fj_bin_ddfw_donor_samples; ++s) { + const int32_t v = pb.variables[vo + (int32_t)(rng.next_u32() % (uint32_t)(ve - vo))]; + const int32_t no = pb.reverse_offsets[v], ne = pb.reverse_offsets[v + 1]; + if (ne <= no) continue; + const int32_t d = + pb.reverse_constraints[no + (int32_t)(rng.next_u32() % (uint32_t)(ne - no))]; + if (d != cf && !is_violated[d] && row_weight[d] > best_w) { + best_w = row_weight[d]; + best_donor = d; + } + } + if (best_donor >= 0) { + const int32_t donated = row_weight[best_donor] - transfer; + cuopt_assert(donated >= fj_bin_ddfw_init, "donation broke the weight floor"); + reweight_constraint(best_donor, donated); + } + } + if (violated_list.empty()) { + if (best_objective < last_best_objective) { + iterations_at_same_objective = 0; + last_best_objective = best_objective; + } else { + ++iterations_at_same_objective; + } + objective_weight += objective_weight_increment(); + } + track_infeasible_checkpoint(); + } + + // Stall-escalation for the objective weight, the feasible-region counterpart of ddfw_transfer: + // a lane that keeps reaching local minima without moving its best objective needs more + // objective pressure than one that is still improving. + int32_t objective_weight_increment() const + { + if (iterations_at_same_objective <= fj_bin_obj_stall_after) return 1; + const int32_t steps = + 1 + (iterations_at_same_objective - fj_bin_obj_stall_after) / fj_bin_obj_stall_after; + return steps < fj_bin_obj_escalate_max ? steps : fj_bin_obj_escalate_max; + } + + void reset_infeasible_checkpoint() + { + best_infeasible_assign.clear(); + best_infeasible_severity = std::numeric_limits::max(); + checkpoint_severity = std::numeric_limits::max(); + iters_since_infeasible_improve = 0; + } + + void track_infeasible_checkpoint() + { + if (violated_list.empty()) { + reset_infeasible_checkpoint(); + return; + } + + int64_t severity = 0; + for (int32_t r : violated_list) { + cuopt_assert(row_slack[r] < 0, "row in violated_list is not violated"); + severity -= (int64_t)row_slack[r]; + } + + if (severity < best_infeasible_severity) { + best_infeasible_severity = severity; + iters_since_infeasible_improve = 0; + restores_since_improvement = 0; + if ((double)severity < (double)checkpoint_severity * infeasible_checkpoint_refresh_ratio) { + best_infeasible_assign = assign; + checkpoint_severity = severity; + ++n_checkpoint_snapshots; + } + return; + } + + if (restores_since_improvement >= infeasible_restart_max_streak) return; + if (++iters_since_infeasible_improve < infeasible_restart_window) return; + if ((double)severity <= (double)best_infeasible_severity * infeasible_restart_degrade_ratio) + return; + if (best_infeasible_assign.empty()) return; + + cuopt_assert(checkpoint_severity >= best_infeasible_severity, + "checkpoint cannot beat the best severity seen"); + + assign = best_infeasible_assign; + for (int32_t v = 0; v < pb.n_variables; ++v) + assign_i32[v] = assign[v]; + recompute_slack(); + + ++n_checkpoint_restores; + ++restores_since_improvement; + if (restores_since_improvement > max_restores_since_improvement) + max_restores_since_improvement = restores_since_improvement; + iters_since_infeasible_improve = 0; + } + + // Global argmax over every variable, affordable because var_score is maintained live. While the + // objective weight is zero the full score is exactly var_score; above zero the sweep runs over + // var_score plus the cached objective base. Only the local-minimum path falls to the scalar loop. + std::pair find_move_global(bool localmin) + { + if (!localmin && objective_weight == 0) { + // The sweep reads var_score alone; the handful of tabu variables are held at the invalid + // sentinel across it rather than tested per variable. + int32_t saved_var[fj_bin_tabu_t::ring_size]; + int64_t saved_score[fj_bin_tabu_t::ring_size]; + const int32_t blocked = tabu.block_tabu(iters, var_score.data(), saved_var, saved_score); + + int32_t v = -1; + int64_t s = fj_bin_score_invalid; + fj_bin_argmax(var_score.data(), pb.n_variables, argmax_tile, v, s); + + fj_bin_tabu_t::unblock_tabu(blocked, var_score.data(), saved_var, saved_score); + return {v, s}; + } + + if (!localmin) { + // The breakthrough bonus is deliberately absent from the ranking: it depends on + // incumbent_objective, so no per-variable form of it survives a move, and it occupies the low + // field where it can only separate variables already tied on the base. The winner's score is + // then taken from full_score so the caller sees the true value. + ensure_objective_base(); + int64_t* const comb_p = combined_score.data(); + fj_bin_add_scores(var_score.data(), obj_base_score.data(), pb.n_variables, comb_p); + + int32_t saved_var[fj_bin_tabu_t::ring_size]; + int64_t saved_score[fj_bin_tabu_t::ring_size]; + const int32_t blocked = tabu.block_tabu(iters, comb_p, saved_var, saved_score); + + int32_t v = -1; + int64_t s = fj_bin_score_invalid; + fj_bin_argmax(comb_p, pb.n_variables, argmax_tile, v, s); + + fj_bin_tabu_t::unblock_tabu(blocked, comb_p, saved_var, saved_score); + if (v >= 0) s = full_score(v, (int8_t)(1 - 2 * assign[v])); + return {v, s}; + } + + int32_t best_v = -1; + int64_t best_s = fj_bin_score_invalid; + for (int32_t v = 0; v < pb.n_variables; ++v) { + if (tabu_blocked(v, localmin)) continue; + const int64_t s = full_score(v, (int8_t)(1 - 2 * assign[v])); + if (s > best_s) { + best_s = s; + best_v = v; + } + } + return {best_v, best_s}; + } + + std::pair find_move_in_rows(const std::vector& target_rows, + bool localmin) + { + int32_t best_v = -1; + int64_t best_s = fj_bin_score_invalid; + for (int32_t r : target_rows) { + for (int32_t k = pb.offsets[r]; k < pb.offsets[r + 1]; ++k) { + const int32_t v = pb.variables[k]; + if (var_bitmap[v]) continue; + var_bitmap[v] = 1; + if (tabu_blocked(v, localmin)) continue; + const int64_t s = full_score(v, (int8_t)(1 - 2 * assign[v])); + if (s > best_s) { + best_s = s; + best_v = v; + } + } + } + // Restore the all-zero invariant by revisiting only what was set: the sampled rows hold a few + // dozen variables against n in the thousands, so this is far cheaper than clearing the array. + for (int32_t r : target_rows) { + for (int32_t k = pb.offsets[r]; k < pb.offsets[r + 1]; ++k) var_bitmap[pb.variables[k]] = 0; + } + return {best_v, best_s}; + } + + std::pair find_move_violated(int32_t sample_size, bool localmin) + { + // Draw the rows directly instead of reservoir-sampling the violated list: `std::sample` is + // linear in the population, so it walked every violated row to keep a handful. Sampling with + // replacement is what `find_move_satisfied` already does, and `find_move_in_rows` deduplicates + // variables through `var_bitmap`, so a repeated row costs a bitmap sweep and no scoring. + const int32_t n = (int32_t)violated_list.size(); + const std::vector* sampled = &violated_list; + if (n > sample_size) { + sample_buf.clear(); + for (int32_t i = 0; i < sample_size; ++i) { + sample_buf.push_back(violated_list[rng.next_u32() % (uint32_t)n]); + } + sampled = &sample_buf; + } + auto move = find_move_in_rows(*sampled, localmin); + + // Breakthrough moves: once a feasible solution exists, allow objective-driven jumps. + if (feasible_found && incumbent_objective >= best_objective + breakthrough_margin) { + for (int32_t v : pb.objective_vars) { + const double step = (best_objective - incumbent_objective) / pb.objective[v]; + double target = pb.objective[v] > 0 ? std::floor(assign[v] + step) + : std::ceil(assign[v] + step); + if (target < 0) target = 0; + if (target > 1) target = 1; + if ((int8_t)target == assign[v]) continue; + if (tabu_blocked(v, false)) continue; + const int64_t s = full_score(v, (int8_t)((int8_t)target - assign[v])); + if (s > move.second) move = {v, s}; + } + } + return move; + } + + std::pair find_move_satisfied(int32_t sample_size) + { + sample_buf.clear(); + for (int32_t tries = 0; (int32_t)sample_buf.size() < sample_size && tries < sample_size * 8; + ++tries) { + const int32_t r = (int32_t)(rng.next_u32() % (uint32_t)pb.n_constraints); + if (!is_violated[r]) sample_buf.push_back(r); + } + return find_move_in_rows(sample_buf, false); + } + + // True when flipping both variables leaves every row they touch satisfied. Both reverse ranges + // are row-ascending, so shared rows are handled jointly by merging them. + bool paired_flip_keeps_feasible( + int32_t var1, int8_t delta1, int32_t var2, int8_t delta2) const + { + int32_t i = pb.reverse_offsets[var1], ie = pb.reverse_offsets[var1 + 1]; + int32_t j = pb.reverse_offsets[var2], je = pb.reverse_offsets[var2 + 1]; + + while (i < ie || j < je) { + const int32_t r1 = i < ie ? pb.reverse_constraints[i] : INT32_MAX; + const int32_t r2 = j < je ? pb.reverse_constraints[j] : INT32_MAX; + const int32_t r = r1 < r2 ? r1 : r2; + + int32_t change = 0; + if (r1 == r) change += (int32_t)pb.reverse_coefficients[i++] * delta1; + if (r2 == r) change += (int32_t)pb.reverse_coefficients[j++] * delta2; + if (row_slack[r] - change < 0) return false; + } + return true; + } + + // Net change in the violated-row count from flipping both variables. Positive is an improvement. + // Both reverse ranges are row-ascending, so shared rows are counted once with their joint delta. + int32_t paired_flip_violation_delta(int32_t var1, + int8_t delta1, + int32_t var2, + int8_t delta2) const + { + int32_t i = pb.reverse_offsets[var1], ie = pb.reverse_offsets[var1 + 1]; + int32_t j = pb.reverse_offsets[var2], je = pb.reverse_offsets[var2 + 1]; + int32_t net = 0; + + while (i < ie || j < je) { + const int32_t r1 = i < ie ? pb.reverse_constraints[i] : INT32_MAX; + const int32_t r2 = j < je ? pb.reverse_constraints[j] : INT32_MAX; + const int32_t r = r1 < r2 ? r1 : r2; + + int32_t change = 0; + if (r1 == r) change += (int32_t)pb.reverse_coefficients[i++] * delta1; + if (r2 == r) change += (int32_t)pb.reverse_coefficients[j++] * delta2; + + const bool was_violated = row_slack[r] < 0; + const bool now_violated = row_slack[r] - change < 0; + if (was_violated && !now_violated) + ++net; + else if (!was_violated && now_violated) + --net; + } + return net; + } + + // Draws a few violated rows and searches their members for a joint flip that strictly reduces the + // violated-row count. The single-flip path cannot see these: each half may be neutral or worsening + // on its own. Rate-limited by the caller because the pair scan is quadratic in the pool. + std::pair find_infeasible_pair_repair() + { + const std::pair none{-1, -1}; + if (violated_list.empty()) return none; + + sample_buf.clear(); + const int32_t n_viol = (int32_t)violated_list.size(); + const int32_t n_rows = n_viol < fj_bin_repair_max_rows ? n_viol : fj_bin_repair_max_rows; + for (int32_t t = 0; t < n_rows; ++t) + sample_buf.push_back(violated_list[rng.next_u32() % (uint32_t)n_viol]); + + int32_t pool[fj_bin_repair_max_vars]; + int32_t n_pool = 0; + for (int32_t r : sample_buf) { + const int32_t begin = pb.offsets[r]; + const int32_t width = pb.offsets[r + 1] - begin; + if (width == 0) continue; + + // A random cyclic start rather than the CSR prefix. At a repeated local minimum the prefix + // makes the neighbourhood deterministic and leaves the tail of a wide covering row permanently + // invisible, at the same pool size and cost. + const int32_t start = (int32_t)(rng.next_u32() % (uint32_t)width); + for (int32_t q = 0; q < width && n_pool < fj_bin_repair_max_vars; ++q) { + const int32_t v = pb.variables[begin + (start + q) % width]; + bool dup = false; + for (int32_t p = 0; p < n_pool && !dup; ++p) + dup = pool[p] == v; + if (!dup) pool[n_pool++] = v; + } + } + + std::pair best_pair = none; + int32_t best_net = 0; + for (int32_t a = 0; a < n_pool; ++a) { + const int32_t v1 = pool[a]; + if (tabu_blocked(v1, false)) continue; + const int8_t delta1 = (int8_t)(1 - 2 * assign[v1]); + + for (int32_t b = a + 1; b < n_pool; ++b) { + const int32_t v2 = pool[b]; + if (tabu_blocked(v2, false)) continue; + const int8_t delta2 = (int8_t)(1 - 2 * assign[v2]); + const int32_t net = paired_flip_violation_delta(v1, delta1, v2, delta2); + if (net > best_net) { + best_net = net; + best_pair = {v1, v2}; + } + } + } + cuopt_assert(best_pair.first < 0 || best_net > 0, "accepted a repair that gains no row"); + return best_pair; + } + + std::pair, int64_t> find_lift_2opt_move() + { + cuopt_assert(violated_list.empty(), "lift moves require a feasible incumbent"); + + std::pair best_pair = {-1, -1}; + int64_t best_s = 0; + double best_improvement = 0; + if (pb.objective_vars.empty()) return {best_pair, best_s}; + + const uint32_t n_obj = (uint32_t)pb.objective_vars.size(); + const int32_t n_draws = n_obj < (uint32_t)fj_bin_2opt_candidates ? (int32_t)n_obj + : fj_bin_2opt_candidates; + + for (int32_t t = 0; t < n_draws; ++t) { + const int32_t var1 = pb.objective_vars[rng.next_u32() % n_obj]; + const int8_t delta1 = (int8_t)(1 - 2 * assign[var1]); + if ((double)delta1 * pb.objective[var1] >= 0) continue; + if (tabu_blocked(var1, false)) continue; + + // Only pairs are useful here: a flip breaking nothing is already the single-flip lift's job, + // and one breaking several rows cannot be repaired by a single companion. + int32_t broken = -1; + bool multiple = false; + for (int32_t i = pb.reverse_offsets[var1]; i < pb.reverse_offsets[var1 + 1] && !multiple; + ++i) { + const int32_t r = pb.reverse_constraints[i]; + if (row_slack[r] - (int32_t)pb.reverse_coefficients[i] * delta1 < 0) { + if (broken >= 0) + multiple = true; + else + broken = r; + } + } + if (multiple || broken < 0) continue; + + for (int32_t k = pb.offsets[broken]; k < pb.offsets[broken + 1]; ++k) { + const int32_t var2 = pb.variables[k]; + if (var2 == var1) continue; + + const int8_t delta2 = (int8_t)(1 - 2 * assign[var2]); + const double combined = (double)delta1 * pb.objective[var1] + + (double)delta2 * pb.objective[var2]; + if (combined >= 0) continue; + if (tabu_blocked(var2, false)) continue; + if (!paired_flip_keeps_feasible(var1, delta1, var2, delta2)) continue; + + // Both lift operators rank on the objective gain in its own units: the packed score counts + // weights, and this engine requires an integral matrix but not integral objective terms. + const double improvement = -combined; + if (improvement > best_improvement) { + best_improvement = improvement; + best_s = 1; // sign only, never compared against another operator's score + best_pair = {var1, var2}; + } + } + } + cuopt_assert((best_pair.first < 0) == (best_improvement <= 0), + "pair and score must agree on whether a move was found"); + return {best_pair, best_s}; + } + + std::pair find_lift_move() const + { + cuopt_assert(violated_list.empty(), "lift moves require a feasible incumbent"); + + int32_t best_v = -1; + int64_t best_s = 0; + double best_improvement = 0; + for (int32_t v : pb.objective_vars) { + const int8_t delta = (int8_t)(1 - 2 * assign[v]); + if ((double)delta * pb.objective[v] >= 0) continue; + if (tabu_blocked(v, false)) continue; + // Base field is zero iff the flip breaks no row; K/2 splits it while |bonus| < 2^31. + if (var_score[v] <= -(fj_bin_score_k / 2)) continue; + const double improvement = -pb.objective[v] * (double)delta; + if (improvement > best_improvement) { + best_improvement = improvement; + best_s = 1; + best_v = v; + } + } + cuopt_assert((best_v < 0) == (best_improvement <= 0), + "move and score must agree on whether a move was found"); + return {best_v, best_s}; + } + + // Flips a few variables drawn from violated rows, to leave a basin the weights cannot escape. + void infeasible_region_kick() + { + const int32_t n_viol = (int32_t)violated_list.size(); + cuopt_assert(n_viol > 0, "kick requires a violated row"); + + int32_t flipped[fj_bin_kick_rows * fj_bin_kick_vars_per_row]; + int32_t n_flipped = 0; + + for (int32_t i = 0; i < fj_bin_kick_rows; ++i) { + const int32_t r = violated_list[rng.next_u32() % (uint32_t)n_viol]; + const int32_t row_begin = pb.offsets[r]; + const int32_t row_end = pb.offsets[r + 1]; + if (row_begin >= row_end) continue; + + for (int32_t j = 0; j < fj_bin_kick_vars_per_row; ++j) { + const int32_t k = row_begin + (int32_t)(rng.next_u32() % (uint32_t)(row_end - row_begin)); + const int32_t v = pb.variables[k]; + + bool already = false; + for (int32_t f = 0; f < n_flipped && !already; ++f) + already = flipped[f] == v; + if (already) continue; + + cuopt_assert(n_flipped < fj_bin_kick_rows * fj_bin_kick_vars_per_row, "flip list overflow"); + flipped[n_flipped++] = v; + assign[v] = (int8_t)(1 - assign[v]); + assign_i32[v] = assign[v]; + } + } + recompute_slack(); + } + + void perturb() + { + if (pb.objective_vars.empty()) return; + if (feasible_found) { + cuopt_assert((int32_t)best_assign.size() == pb.n_variables, "incumbent size mismatch"); + assign = best_assign; + // The shared buffer holds decoded integers, so the flat 0/1 read below only lines up when + // engine variables are the model's own variables. + if (!pb.encoded && shared_incumbent && + shared_incumbent->adopt((f_t)best_objective, adopt_buffer)) { + for (int32_t v = 0; v < pb.n_variables; ++v) + assign[v] = (int8_t)(adopt_buffer[v] >= 0.5 ? 1 : 0); + } + for (int32_t v = 0; v < pb.n_variables; ++v) assign_i32[v] = assign[v]; + } + const uint32_t n = (uint32_t)pb.objective_vars.size(); + for (int i = 0; i < 2; ++i) { + const int32_t v = pb.objective_vars[rng.next_u32() % n]; + assign[v] = (int8_t)(rng.next_u32() & 1u); + assign_i32[v] = assign[v]; + } + recompute_slack(); + } + + // Restart returns the assignment to the seed the climber was constructed with, leaving the + // recorded best and the global iteration counter intact. + void do_restart() + { + assign = seed_assign; + for (int32_t v = 0; v < pb.n_variables; ++v) assign_i32[v] = assign[v]; + for (int32_t r = 0; r < pb.n_constraints; ++r) row_weight[r] = pb.initial_weight[r]; + max_weight = fj_bin_ddfw_init; + objective_weight = seed_objective_weight; + reset_infeasible_checkpoint(); + tabu.clear(iters); + recompute_slack(); + last_restart_iter = iters; + // The restarted walk gets a full window before the stall gate can perturb it. + iters_since_best = 0; + } + + void init(fj_cpu_climber_t& climber) + { + const auto& params = climber.settings.parameters; + seed = climber.settings.seed; + rng = raft::random::PCGenerator((uint64_t)seed, 0, 0); + tabu_tenure_min = params.tabu_tenure_min; + tabu_tenure_max = params.tabu_tenure_max; + breakthrough_margin = params.breakthrough_move_epsilon; + perturb_interval = climber.perturb_interval; + mtm_viol_samples = climber.mtm_viol_samples; + mtm_sat_samples = climber.mtm_sat_samples; + enable_infeasible_repair = climber.enable_infeasible_repair && + pb.n_exchange_vars >= fj_bin_repair_min_exchange_vars && + pb.max_card_degree <= fj_bin_repair_max_card_degree; + last_repair_iter = 0; + + infeasible_restart_window = climber.infeasible_restart_window; + infeasible_restart_max_streak = climber.infeasible_restart_max_streak; + infeasible_restart_degrade_ratio = (double)climber.infeasible_restart_degrade_ratio; + infeasible_checkpoint_refresh_ratio = (double)climber.infeasible_checkpoint_refresh_ratio; + cuopt_assert(infeasible_restart_window > 0, "invalid infeasible restart window"); + cuopt_assert(infeasible_restart_max_streak > 0, "invalid infeasible restart streak cap"); + cuopt_assert(infeasible_restart_degrade_ratio >= 1.0, "degrade ratio should be at least one"); + cuopt_assert( + infeasible_checkpoint_refresh_ratio > 0.0 && infeasible_checkpoint_refresh_ratio <= 1.0, + "checkpoint refresh ratio should be in (0, 1]"); + + if (tabu_tenure_max <= tabu_tenure_min) tabu_tenure_max = tabu_tenure_min + 1; + + // The tabu ring is indexed by iteration modulo its size, so a slot is reused after ring_size + // iterations. A tenure that long would be overwritten while the variable is still tabu, and the + // argmax would stop excluding it. Clamped as well as asserted: release builds compile the assert + // out, and silently dropping tabu entries is worse than a shorter tenure. + cuopt_assert(tabu_tenure_max <= fj_bin_tabu_t::max_tenure, + "tabu tenure exceeds the tabu ring, live entries would be evicted"); + if (tabu_tenure_max > fj_bin_tabu_t::max_tenure) tabu_tenure_max = fj_bin_tabu_t::max_tenure; + + const int32_t n = pb.n_variables, m = pb.n_constraints; + const auto& h_assign = climber.h_assignment; + assign.assign(n, 0); + if (pb.encoded) { + // Descending weight, so the bit pattern reproduces the start value wherever it is + // representable: with exact closure that is every integer of the domain. + std::vector> bits_of(pb.n_original); + for (int32_t b = 0; b < n; ++b) bits_of[pb.bit_owner[b]].push_back(b); + for (int32_t v = 0; v < pb.n_original; ++v) { + long residual = std::lround((double)h_assign[v] - pb.var_offset[v]); + if (residual < 0) residual = 0; + auto& bits = bits_of[v]; + std::sort(bits.begin(), bits.end(), [&](int32_t a, int32_t b) { + return pb.bit_weight[a] > pb.bit_weight[b]; + }); + for (int32_t b : bits) { + const long w = std::lround(pb.bit_weight[b]); + if (w <= residual) { + assign[b] = 1; + residual -= w; + } + } + cuopt_assert(residual == 0, "greedy bit encode left the start value unrepresented"); + } + } else { + for (int32_t v = 0; v < n; ++v) { + const double val = (double)h_assign[v]; + assign[v] = (int8_t)(val >= 0.5 ? 1 : 0); + } + } + seed_assign = assign; + best_assign = assign; + shared_incumbent = climber.shared_incumbent; + if (shared_incumbent) adopt_buffer.assign(n, 0); + reset_infeasible_checkpoint(); + assign_i32.assign(n, 0); + for (int32_t v = 0; v < n; ++v) assign_i32[v] = assign[v]; + + row_weight.assign(pb.initial_weight.begin(), pb.initial_weight.end()); + row_slack.assign(m, 0); + + var_score.assign(n, 0); + nnz_score_delta.assign(pb.nnz + fj_bin_simd_padding, 0); + // Zeroed once: ensure_objective_base only ever rewrites the objective variables. + obj_base_score.assign(n, 0); + combined_score.assign(n, 0); + obj_base_weight = -1; + tabu.resize(n); + is_violated.assign(m, 0); + vpos.assign(m, -1); + violated_list.clear(); + var_bitmap.assign(n, 0); + + const int32_t seeded_weight = (int32_t)std::lround(climber.h_objective_weight); + cuopt_assert(seeded_weight >= 0, "objective weight should be positive or zero"); + + double abs_obj_sum = 0; + for (int32_t v : pb.objective_vars) abs_obj_sum += std::fabs(pb.objective[v]); + obj_magnitude = abs_obj_sum > 0 ? abs_obj_sum / (double)pb.objective_vars.size() : 1.0; + cuopt_assert(std::isfinite(obj_magnitude) && obj_magnitude > 0, + "objective magnitude unit must be finite and positive"); + + objective_offset = 0; + if (pb.encoded) { + for (int32_t v = 0; v < pb.n_original; ++v) + objective_offset += pb.orig_objective[v] * pb.var_offset[v]; + } + + argmax_tile = fj_bin_argmax_tile(); + objective_weight = seeded_weight > 0 ? seeded_weight : 0; + seed_objective_weight = objective_weight; + max_weight = fj_bin_ddfw_init; + incumbent_objective = 0; + best_objective = std::numeric_limits::infinity(); + last_best_objective = std::numeric_limits::infinity(); + iterations_at_same_objective = 0; + feasible_found = false; + iters = 0; + iters_since_best = 0; + last_restart_iter = 0; + last_kick_iter = 0; + recompute_slack(); + } + + void solve(fj_cpu_climber_t& climber, f_t time_limit, double work_unit_limit) + { + init(climber); + + const auto loop_start = std::chrono::high_resolution_clock::now(); + const auto limit = + std::chrono::milliseconds((int64_t)std::floor((double)time_limit * 1000.0)); + const bool bounded_time = std::isfinite((double)time_limit); + + while (!climber.halted && !climber.preemption_flag.load()) { + if (bounded_time && std::chrono::high_resolution_clock::now() - loop_start > limit) break; + if (iters >= climber.settings.iteration_limit) break; + if (iters - last_restart_iter >= fj_bin_restart_period) do_restart(); + tabu.maybe_rebase(iters); + + int32_t move_var = -1; + int64_t score = fj_bin_score_invalid; + std::pair pair2 = {-1, -1}; + if (violated_list.empty()) { + std::tie(move_var, score) = find_lift_move(); + // Pairs are only reachable once no single improving flip preserves feasibility. + if (score <= 0) { + int64_t pair_score; + std::tie(pair2, pair_score) = find_lift_2opt_move(); + if (pair_score > 0) score = pair_score; + } + } + if (pair2.first < 0 && score <= 0) std::tie(move_var, score) = find_move_global(false); + if (pair2.first < 0 && feasible_found && score <= 0) + std::tie(move_var, score) = find_move_satisfied(mtm_sat_samples); + + bool perturb_now = false; + if (violated_list.empty() && iters_since_best > perturb_interval) { + perturb_now = true; + // Without this the counter stays above the interval and every later iteration perturbs. + iters_since_best = 0; + } + + if (pair2.first >= 0 && !perturb_now) { + apply_move(pair2.first, (int8_t)(1 - 2 * assign[pair2.first]), climber); + apply_move(pair2.second, (int8_t)(1 - 2 * assign[pair2.second]), climber); + } else if (score > 0 && move_var >= 0 && !perturb_now) { + apply_move(move_var, (int8_t)(1 - 2 * assign[move_var]), climber); + } else { + // A pair that reduces the violated count takes precedence over reweighting: the weights + // exist to escape a minimum no move can improve, and this found one that can. + bool repaired = false; + if (enable_infeasible_repair && !violated_list.empty() && + iters - last_repair_iter >= fj_bin_repair_interval) { + last_repair_iter = iters; + const auto repair_pair = find_infeasible_pair_repair(); + if (repair_pair.first >= 0) { + apply_move(repair_pair.first, (int8_t)(1 - 2 * assign[repair_pair.first]), climber); + apply_move(repair_pair.second, (int8_t)(1 - 2 * assign[repair_pair.second]), climber); + repaired = true; + } + } + + if (!repaired) { + update_weights(); + const bool kick_ready = !violated_list.empty() && + iters_since_infeasible_improve >= fj_bin_kick_after && + iters - last_kick_iter >= fj_bin_kick_cooldown && + iters - last_restart_iter >= fj_bin_kick_restart_guard; + if (kick_ready) { + infeasible_region_kick(); + last_kick_iter = iters; + } else if (perturb_now) { + perturb(); + } + std::tie(move_var, score) = find_move_violated(1, true); + const int32_t v = move_var >= 0 ? move_var : 0; + apply_move(v, (int8_t)(1 - 2 * assign[v]), climber); + } + } + + if (iters % climber.log_interval == 0) { + CUOPT_LOG_DEBUG("%sCPUFJ[bin%d] iteration: %d, viol: %zu, best: %g, maxw: %d", + climber.log_prefix.c_str(), + coefficient_bits(), + iters, + violated_list.size(), + best_objective, + max_weight); + } + if (iters % climber.diversity_callback_interval == 0 && climber.diversity_callback) { + auto& h_assign = climber.h_assignment; + if (pb.encoded) { + for (int32_t v = 0; v < pb.n_original; ++v) h_assign[v] = (f_t)pb.var_offset[v]; + for (int32_t b = 0; b < pb.n_variables; ++b) + if (assign[b]) h_assign[pb.bit_owner[b]] += (f_t)pb.bit_weight[b]; + } else { + for (int32_t v = 0; v < pb.n_variables; ++v) h_assign[v] = (f_t)assign[v]; + } + climber.diversity_callback((f_t)incumbent_objective, h_assign); + } + + // Work-unit proxy. nnz_touched is cumulative, reproducing the accumulation shape the general + // path gets from its cumulative byte counters. + if (iters % 100 == 0 && iters > 0) { + const double work = (double)nnz_touched * fj_bin_bytes_per_nnz * climber.work_unit_bias / 1e10; + climber.work_units_elapsed.store(work, std::memory_order_release); + if (climber.producer_sync != nullptr) climber.producer_sync->notify_progress(); + if (work >= work_unit_limit) break; + } + + ++iters; + ++iters_since_best; + } + + compute_saturation(); + verify_incumbent(climber); + climber.iterations = (i_t)iters; + CUOPT_LOG_DEBUG( + "%sCPUFJ[bin%d] done: %d iterations, best %g, max weight %d, aggregate base %d/%d, bonus %d/%d", + climber.log_prefix.c_str(), + coefficient_bits(), + iters, + best_objective, + max_weight, + max_aggregate_base, + fj_bin_base_limit, + max_aggregate_bonus, + fj_bin_bonus_limit); + CUOPT_LOG_DEBUG("%sCPUFJ[bin%d] work: nnz_patched %lld, rows_walked %lld", + climber.log_prefix.c_str(), + coefficient_bits(), + (long long)nnz_patched, + (long long)rows_walked); + CUOPT_LOG_DEBUG("%sCPUFJ[bin%d] checkpoint: %lld restores, %lld snapshots, max streak %d", + climber.log_prefix.c_str(), + coefficient_bits(), + (long long)n_checkpoint_restores, + (long long)n_checkpoint_snapshots, + max_restores_since_improvement); + } +}; + +template +bool try_cpufj_binary_solve(fj_cpu_climber_t& climber, + f_t time_limit, + double work_unit_limit) +{ + // Escape hatch for A/B against the general path on an instance the fast path would take. The two + // paths are meant to search identically, so any divergence is a bug in this one; setting this is + // how that gets bisected without editing the eligibility scan. + static const bool disabled = std::getenv("CUOPT_NO_BINFJ") != nullptr; + if (disabled) return false; + + const fj_bin_scan_t scan = fj_bin_scan(climber); + if (scan.reject != fj_binary_reject_t::none) { + // A non-binary variable is the one rejection the encoding can answer: the model may still be + // all-integer with finite domains. Every other reason fails the encoded model just the same. + if (scan.reject == fj_binary_reject_t::non_binary_var) { + // The width the encoded coefficients need is only known once they are built, so probe with + // int16 and rebuild on int8 for the narrower kernel when that is enough. + fj_bin_engine_t probe; + int bits = 0; + if (fj_bin_encode(climber, probe.pb, bits)) { + if (bits == 8) { + fj_bin_engine_t engine8; + int bits8 = 0; + if (fj_bin_encode(climber, engine8.pb, bits8)) { + CUOPT_LOG_DEBUG("%sCPUFJ binary fast path enabled (encoded int8): %d bits, %d rows", + climber.log_prefix.c_str(), + engine8.pb.n_variables, + engine8.pb.n_constraints); + engine8.solve(climber, time_limit, work_unit_limit); + return true; + } + } + CUOPT_LOG_DEBUG("%sCPUFJ binary fast path enabled (encoded int16): %d bits, %d rows", + climber.log_prefix.c_str(), + probe.pb.n_variables, + probe.pb.n_constraints); + probe.solve(climber, time_limit, work_unit_limit); + return true; + } + } + CUOPT_LOG_DEBUG("%sCPUFJ binary fast path declined: %s (row %d, var %d)", + climber.log_prefix.c_str(), + fj_binary_reject_name(scan.reject), + scan.bad_row, + scan.bad_var); + return false; + } + + auto run = [&](auto& engine) -> bool { + if (!fj_bin_narrow(climber, scan, engine.pb)) { + CUOPT_LOG_DEBUG("%sCPUFJ binary fast path declined: %s", + climber.log_prefix.c_str(), + fj_binary_reject_name(fj_binary_reject_t::narrow_check_failed)); + return false; + } + CUOPT_LOG_DEBUG( + "%sCPUFJ binary fast path enabled: int%d coefficients, %d rows after one-sided split", + climber.log_prefix.c_str(), + scan.coefficient_bits, + scan.n_split_constraints); + engine.solve(climber, time_limit, work_unit_limit); + return true; + }; + + if (scan.coefficient_bits == 8) { + fj_bin_engine_t engine; + return run(engine); + } + fj_bin_engine_t engine; + return run(engine); +} + +#if MIP_INSTANTIATE_FLOAT +template bool try_cpufj_binary_solve(fj_cpu_climber_t& climber, + float time_limit, + double work_unit_limit); +#endif + +#if MIP_INSTANTIATE_DOUBLE +template bool try_cpufj_binary_solve(fj_cpu_climber_t& climber, + double time_limit, + double work_unit_limit); +#endif + +} // namespace cuopt::mathematical_optimization::mip diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh new file mode 100644 index 0000000000..08005817aa --- /dev/null +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary.cuh @@ -0,0 +1,131 @@ +/* clang-format off */ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +/* clang-format on */ + +#pragma once + +#include + +// The fast path applies to instances whose variables are all binary and whose rows carry integer +// coefficients within int8 or int16 range. On those it runs a SIMD integer engine: exact feasibility +// against a single row bound, a live per-variable score patched through stored per-nnz +// contributions, and a global argmax move selection. + +namespace cuopt::mathematical_optimization::mip { + +template +struct fj_cpu_climber_t; + +enum class fj_binary_reject_t : uint8_t { + none, + empty_problem, + non_binary_var, + fractional_coefficient, + coefficient_out_of_range, + fractional_row_bound, + row_bound_out_of_range, + lhs_headroom, + narrow_check_failed, +}; +const char* fj_binary_reject_name(fj_binary_reject_t reason); + +// Returns true if the fast path ran (eligible and narrowed); false if declined, in which case the caller should take the general path. +// TODO: worth revisiting if the same climber is solved repeatedly to cache the fastpath state +template +bool try_cpufj_binary_solve(fj_cpu_climber_t& climber, + f_t time_limit, + double work_unit_limit); + +// Packed staged score: one int64 holding base * K + bonus, encoding the general path's +// lexicographic (base, bonus) comparison as a single arithmetic one. +// +// The width is what makes the encoding faithful. Both fields aggregate over the rows a variable +// appears in, so each is bounded by max_var_degree * max_weight -- unbounded above at build time, +// since DDFW grows the weights. At 15 bits the bonus field overflowed into the base on real +// instances (chromaticindex1024-7 reaches an aggregate bonus of 122880 against 16384), which +// silently corrupts the ordering the argmax depends on. 32 bits leaves the base free to use the +// whole int32 range before the encoding can break. +constexpr int32_t fj_bin_score_shift = 32; +constexpr int64_t fj_bin_score_k = (int64_t)1 << fj_bin_score_shift; +constexpr int64_t fj_bin_score_invalid = INT64_MIN; + +// Change in one row's weighted score when one variable flips, from the row's signed slack before +// (os) and after (ns) that flip. base is the weighted change in satisfaction; bonus is the +// weighted change in strict slack. When both states are violated the improving direction earns +// half weight, matching excess_improvement_weight of 1/2. +// purpose: implements the scoring delta logic from feasibility_jump.cuh in a form easier to port to SIMD +static inline void fj_bin_score_delta_parts( + int32_t os, int32_t ns, int32_t weight, int32_t& base, int32_t& bonus) +{ + const int32_t osat = os >= 0, nsat = ns >= 0; + const int32_t ost = os > 0, nst = ns > 0; + const int32_t improving = (os < ns) - (ns < os); + base = weight * (nsat - osat) + (1 - osat) * (1 - nsat) * improving * (weight / 2); + bonus = weight * (nst - ost); +} + +static inline int64_t fj_bin_packed_score_delta(int32_t os, int32_t ns, int32_t weight) +{ + int32_t base = 0, bonus = 0; + fj_bin_score_delta_parts(os, ns, weight, base, bonus); + return (int64_t)base * fj_bin_score_k + bonus; +} + +// Padding margin to prevent faults on tail SIMD loads +constexpr int32_t fj_bin_simd_padding = 256; + +// Patch every variable of one row against the row's current signed slack. The +// move case passes the post-move slack and the flipped variable's index; the reweight case passes +// the unchanged slack and -1, which matches no variable index. +template +void fj_bin_patch_row(const int32_t* variables, + const coef_t* coefficients, + int32_t kb, + int32_t ke, + int64_t* var_score, + int64_t* nnz_score_delta, + const int32_t* assign_i32, + int32_t weight, + int32_t os_new, + int32_t skip_var); + +constexpr int32_t fj_bin_walk_tile = 256; + +// Advance every row incident to one flipped variable within apply_move, and report which of those visits the caller +// must finish by hand (e.g. if the row needs patching) +// +// For every incidence i in the range this applies +// row_slack[incident_row[i]] -= reverse_coefficients[i] * delta +// then writes to out_incidence, in increasing order, the subset of i whose row is not deeply +// satisfied on both sides of the flip and returns how many. +template +int32_t fj_bin_walk_rows(int32_t* row_slack, + const int32_t* incident_row, + const coef_t* reverse_coefficients, + const coef_t* incident_row_cmax, + int32_t incidence_begin, + int32_t incidence_end, + int32_t delta, + int32_t* out_incidence); + +// Argmax over var_score, scanning all n variables. Valid while the objective weight is zero, where +// the full score is exactly var_score. Yields best_var of -1 only if n is 0. +// Tabu is handled by "blocking" the scores corresponding to the tabu vars, and restoring them after the argmax +// affordable since max_tenure is small +void fj_bin_argmax(const int64_t* var_score, + int32_t n, + int32_t tile, + int32_t& best_var, + int64_t& best_score); + +// combined[v] = var_score[v] + obj_score[v] over n variables, which is the full score once the +// objective weight is nonzero. The three arrays must not overlap. +void fj_bin_add_scores(const int64_t* var_score, + const int64_t* obj_score, + int32_t n, + int64_t* combined); + +} // namespace cuopt::mathematical_optimization::mip diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp new file mode 100644 index 0000000000..fd8a93ad73 --- /dev/null +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp @@ -0,0 +1,635 @@ +/* clang-format off */ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +/* clang-format on */ + +// Hot kernels of the binary CPU FJ fast path, vectorized with Google Highway. foreach_target.h +// re-includes this file once per SIMD target; HWY_EXPORT builds the dispatch table and +// HWY_DYNAMIC_DISPATCH picks at runtime. Host-compiled rather than nvcc-compiled: nvcc's frontend +// rejects Highway's x86 headers, which reinterpret-cast intrinsic vectors to compiler-specific +// vector types. + +#include + +#include +#include + +#undef HWY_TARGET_INCLUDE +#define HWY_TARGET_INCLUDE "mip_heuristics/feasibility_jump/fj_cpu_binary_kernels.cpp" +#include "hwy/foreach_target.h" // must precede highway.h +#include "hwy/highway.h" + +HWY_BEFORE_NAMESPACE(); +namespace cuopt::mathematical_optimization::mip { +namespace HWY_NAMESPACE { + +namespace hn = hwy::HWY_NAMESPACE; + +// Whether the row remainder is masked into the vector body or peeled into a scalar tail. AVX-512 +// k-registers, SVE predicates and RVV masks make every operation maskable at no cost, so a row of +// three nonzeros is one masked iteration; peeling it would send most of the work to the tail, since +// row lengths are short and have nothing to do with the lane count. AVX2 and NEON have no mask +// registers: the mask becomes a vector, gather and scatter are emulated, and the tail is cheaper. +// Measured on AVX2, masking the remainder cost 6.8% on supportcase22 and 12.9% on bnatt400. +constexpr bool k_mask_remainder = + (HWY_TARGET <= HWY_AVX3) || HWY_TARGET_IS_SVE || (HWY_TARGET == HWY_RVV); + +// Whether the row walk below is worth vectorizing on this target. It needs a real gather to read the +// slacks and a real compress to emit the tail list; where either is emulated the emulation costs +// more than the scalar loop it replaces, since 85% of visits do nothing but subtract and compare. +// The scalar arm still returns the same list, so the caller needs no second code path -- it pays +// only one store per reported visit. +constexpr bool k_vector_walk = + (HWY_TARGET <= HWY_AVX3) || HWY_TARGET_IS_SVE || (HWY_TARGET == HWY_RVV); + +// One tile of a flipped variable's incidence range, vectorized. The caller tiles the range and runs +// each tile's tail before asking for the next; see fj_bin_walk_tile. +// +// Measured on supportcase22: 84.87% of row visits leave the row deeply satisfied on both sides of +// the flip, and those visits do nothing but update the slack. The remaining 15.13% need the row's +// weight, the flipped variable's own score delta, the violated-set transitions and usually a +// patch -- all indirect, all awkward in a vector. So this kernel does only the uniform part and +// hands back the indices of the visits that are not deep_sat, in increasing order, for the caller +// to finish scalar. +// +// The layout this assumes is what makes it worth doing. Storing the row's signed slack rather than +// its lhs collapses the update to +// +// new_slack = old_slack - coef * delta +// +// so bound and lhs never appear, and the coefficient is the only per-incidence constant. It and +// cmax are replicated per incidence, which makes them unit-stride loads. What remains irregular is +// the slack itself: one gather and one scatter per vector, against four gathers and a scatter for a +// literal SoA split of the row record. +// +// Trajectory is preserved exactly. The slack update is per row and order-independent; the caller's +// tail visits its indices in the same order the scalar loop did; and a deep_sat row is never read by +// the tail, so updating it early is not observable. +template +int32_t WalkRowsImpl(int32_t* HWY_RESTRICT row_slack, + const int32_t* HWY_RESTRICT incident_row, + const coef_t* HWY_RESTRICT reverse_coefficients, + const coef_t* HWY_RESTRICT incident_row_cmax, + int32_t incidence_begin, + int32_t incidence_end, + int32_t delta, + int32_t* HWY_RESTRICT out_incidence) +{ + int32_t n_out = 0; + int32_t ii = incidence_begin; + + if constexpr (k_vector_walk) { + const hn::ScalableTag d; + const hn::Rebind dc; // same lane count, narrower lanes + using V = hn::Vec; + const size_t N = hn::Lanes(d); + + const V vdelta = hn::Set(d, delta); + + // The unit-stride loads always run whole and read into the per-incidence padding; FirstN keeps + // the overhang out of the gather, the scatter and the compress. + for (; ii < incidence_end; ii += (int32_t)N) { + const auto active = hn::FirstN(d, (size_t)(incidence_end - ii)); + + const V rows = hn::LoadU(d, incident_row + ii); + const V skv = hn::PromoteTo(d, hn::LoadU(dc, reverse_coefficients + ii)); + const V cmax = hn::PromoteTo(d, hn::LoadU(dc, incident_row_cmax + ii)); + + const V os = hn::MaskedGatherIndex(active, d, row_slack, rows); + // os - skv * vdelta + const V ns = hn::NegMulAdd(skv, vdelta, os); + + // Only the satisfied side. deep_viol is the caller's business: it fires on 0.02% of visits but + // guards the widest rows in the matrix, so it belongs where the row length is already known. + const auto deep_sat = hn::And(hn::Gt(os, cmax), hn::Gt(ns, cmax)); + const auto to_tail = hn::AndNot(deep_sat, active); + +#if HWY_TARGET == HWY_AVX3_ZEN4 + // Same Zen 4 microcode argument as the score scatter in PatchRowBody: VPSCATTERDD is 89 uops + // at ~24 CPI, against two vector stores and N scalar stores here. Unlike that one this is a + // pure store with no read-modify-write, so it needs its own A/B before the arm is settled. + HWY_ALIGN int32_t row_lane[hn::MaxLanes(d)], slack_lane[hn::MaxLanes(d)]; + hn::Store(rows, d, row_lane); + hn::Store(ns, d, slack_lane); + const size_t lanes = HWY_MIN(N, (size_t)(incidence_end - ii)); + for (size_t i = 0; i < lanes; ++i) row_slack[row_lane[i]] = slack_lane[i]; +#else + hn::MaskedScatterIndex(ns, active, d, row_slack, rows); +#endif + + // A variable meets each row at most once, so no two lanes carry the same row and neither the + // scatter above nor the store loop needs conflict detection. + n_out += (int32_t)hn::CompressStore(hn::Iota(d, ii), to_tail, d, out_incidence + n_out); + } + return n_out; + } + + // Targets without a native gather or compress. Also the remainder is not reached here: the loop + // above runs to oe under FirstN, and this arm replaces it wholesale rather than tailing it. + for (; ii < incidence_end; ++ii) { + const int32_t row = incident_row[ii]; + const int32_t os = row_slack[row]; + const int32_t ns = os - (int32_t)reverse_coefficients[ii] * delta; + row_slack[row] = ns; + const int32_t cmax = (int32_t)incident_row_cmax[ii]; + if (!(os > cmax && ns > cmax)) out_incidence[n_out++] = ii; + } + return n_out; +} + +// Row remainder when it is peeled rather than masked, and the whole row on scalar targets. +template +void PatchRowScalar(const int32_t* HWY_RESTRICT variables, + const coef_t* HWY_RESTRICT coefficients, + int32_t kb, + int32_t ke, + int64_t* HWY_RESTRICT var_score, + int64_t* HWY_RESTRICT nnz_score_delta, + const int32_t* HWY_RESTRICT assign_i32, + int32_t weight, + int32_t os_new, + int32_t skip_var) +{ + for (int32_t k = kb; k < ke; ++k) { + const int32_t v = variables[k]; + if (v == skip_var) continue; + const int32_t flip = 1 - 2 * assign_i32[v]; + const int32_t ns = os_new - (int32_t)coefficients[k] * flip; + const int64_t nc = fj_bin_packed_score_delta(os_new, ns, weight); + var_score[v] += nc - nnz_score_delta[k]; + nnz_score_delta[k] = nc; + } +} + +// Templated on the vector tag so one body serves both the native-width kernel and the narrow one. +// Rows here average well under a native 512-bit vector, and a gather costs the same whether its +// lanes are used or discarded, so short rows are cheaper through a narrower vector. +template +static HWY_INLINE void PatchRowBody(D d, + const int32_t* HWY_RESTRICT variables, + const coef_t* HWY_RESTRICT coefficients, + int32_t kb, + int32_t ke, + int64_t* HWY_RESTRICT var_score, + int64_t* HWY_RESTRICT nnz_score_delta, + const int32_t* HWY_RESTRICT assign_i32, + int32_t weight, + int32_t os_new, + int32_t skip_var) +{ + const hn::Rebind dc; // same lane count, narrower lanes + const hn::Repartition dw; // half the lanes, twice as wide: the packed score + using V = hn::Vec; + using VW = hn::Vec; + const size_t N = hn::Lanes(d); + const size_t NW = hn::Lanes(dw); + + // When the remainder is peeled, a row below one vector never reaches the body, so it skips the + // ten broadcasts below as well. + if constexpr (!k_mask_remainder) { + if ((size_t)(ke - kb) < N) { + PatchRowScalar(variables, coefficients, kb, ke, var_score, nnz_score_delta, + assign_i32, weight, os_new, skip_var); + return; + } + } + + const V vone = hn::Set(d, 1), vzero = hn::Zero(d); + const V vskip = hn::Set(d, skip_var); + const V vos = hn::Set(d, os_new); + const V vw = hn::Set(d, weight), vw2 = hn::Set(d, weight / 2); + + // The row's own slack is uniform across lanes, so its flags are scalars. Broadcast negated to + // match the new-state flags below, which come from VecFromMask and are 0 or -1. + const int32_t osat = os_new >= 0, ost = os_new > 0; + const V vneg_osat = hn::Set(d, -osat), vneg_ost = hn::Set(d, -ost); + const V v_not_osat = hn::Set(d, 1 - osat); + + // The loads always run unmasked and read into the per-nnz padding; when the remainder is masked, + // FirstN keeps the overhang out of the gather, the scatter and the store. + const int32_t vec_end = k_mask_remainder ? ke : ke - (int32_t)N + 1; + int32_t k = kb; + for (; k < vec_end; k += (int32_t)N) { + const V v = hn::LoadU(d, variables + k); + auto active = hn::Ne(v, vskip); + if constexpr (k_mask_remainder) { + active = hn::And(active, hn::FirstN(d, (size_t)(ke - k))); + } + + // Gathered in hardware even on Zen 4, unlike the score update below. Doing this one by lane + // instead measured 8.2% slower: it must spill the index vector and reload it 4 bytes at a time, + // which cannot store-to-load forward, and that cost 959 interlocks per iteration against 72. + // The score update escapes this because it already needs the spill for its read-modify-write. + const V a01 = hn::MaskedGatherIndex(active, d, assign_i32, v); + const V flip = hn::Sub(vone, hn::ShiftLeft<1>(a01)); + const V coef = hn::PromoteTo(d, hn::LoadU(dc, coefficients + k)); + + // vos - coef * flip + const V ns = hn::NegMulAdd(coef, flip, vos); + + // -(ns >= 0) + const V nsat_neg = hn::VecFromMask(d, hn::Ge(ns, vzero)); + // -(ns > 0) + const V nst_neg = hn::VecFromMask(d, hn::Gt(ns, vzero)); + // (ns > vos) - (ns < vos) + const V improving = + hn::Sub(hn::VecFromMask(d, hn::Lt(ns, vos)), hn::VecFromMask(d, hn::Gt(ns, vos))); + + // (1 - osat) * (1 - nsat) + const V both_violated = hn::Mul(v_not_osat, hn::Add(vone, nsat_neg)); + // vw * (nsat - osat) + both_violated * improving * vw2 + const V base = + hn::MulAdd(vw, hn::Sub(vneg_osat, nsat_neg), hn::Mul(hn::Mul(both_violated, improving), vw2)); + // vw * (nst - ost) + const V bonus = hn::Mul(vw, hn::Sub(vneg_ost, nst_neg)); + + // The score is int64, so packing it costs two vectors where the fields took one. Both fields + // are per-row here and fit int32, so they are computed at full lane count above and widened + // only for the pack. Everything below stays in the vector: the pack, the old value, the + // difference and the store back. What reaches the scalar loop is one add per nonzero, which is + // what it was before the score widened -- that loop is 38% of all cycles, so work belongs + // anywhere but there. + const VW base_lo = hn::PromoteLowerTo(dw, base); + const VW base_hi = hn::PromoteUpperTo(dw, base); + const VW bonus_lo = hn::PromoteLowerTo(dw, bonus); + const VW bonus_hi = hn::PromoteUpperTo(dw, bonus); + + const VW packed_lo = hn::Add(hn::ShiftLeft(base_lo), bonus_lo); + const VW packed_hi = hn::Add(hn::ShiftLeft(base_hi), bonus_hi); + + const VW delta_lo = hn::Sub(packed_lo, hn::LoadU(dw, nnz_score_delta + k)); + const VW delta_hi = hn::Sub(packed_hi, hn::LoadU(dw, nnz_score_delta + k + NW)); + + // The store mask is rebuilt at int64 width rather than narrowed from `active`: the same two + // conditions, on the promoted indices. FirstN is applied on every target because where the + // remainder is peeled the body never runs short, so it is all-true there anyway. + const size_t rem = (size_t)(ke - k); + const VW v_lo = hn::PromoteLowerTo(dw, v); + const VW v_hi = hn::PromoteUpperTo(dw, v); + const VW vskip_w = hn::Set(dw, skip_var); + const auto act_lo = hn::And(hn::Ne(v_lo, vskip_w), hn::FirstN(dw, rem)); + const auto act_hi = hn::And(hn::Ne(v_hi, vskip_w), hn::FirstN(dw, rem > NW ? rem - NW : 0)); + hn::BlendedStore(packed_lo, act_lo, dw, nnz_score_delta + k); + hn::BlendedStore(packed_hi, act_hi, dw, nnz_score_delta + k + NW); + +#if HWY_TARGET == HWY_AVX3_ZEN4 + // zmm VSIB is microcode on Zen 4: VPGATHERDD ~76-80 uops / ~21 CPI and VPSCATTERDD 89 / 24, + // against ~5 / ~10 and ~19 / ~11 on SPR-class Intel (Agner Fog, uops.info). So read-modify-write + // by lane here; measured +5.8% over the arm below on an EPYC 9554 (supportcase22, 16 climbers). + HWY_ALIGN int32_t idx[hn::MaxLanes(d)]; + HWY_ALIGN int64_t dl[hn::MaxLanes(d)]; + hn::Store(v, d, idx); + hn::Store(delta_lo, dw, dl); + hn::Store(delta_hi, dw, dl + NW); + // Bounded by the row, not the vector: the lanes past it hold padding, whose zero index would + // otherwise be applied to variable 0. + const size_t lanes = HWY_MIN(N, (size_t)(ke - k)); + for (size_t i = 0; i < lanes; ++i) { + if (idx[i] != skip_var) var_score[idx[i]] += dl[i]; + } +#else + // The score is int64, so the gather and scatter run at the promoted width against the promoted + // indices, in the two halves the pack already produced. + const VW cur_lo = hn::MaskedGatherIndex(act_lo, dw, var_score, v_lo); + const VW cur_hi = hn::MaskedGatherIndex(act_hi, dw, var_score, v_hi); + hn::MaskedScatterIndex(hn::Add(cur_lo, delta_lo), act_lo, dw, var_score, v_lo); + hn::MaskedScatterIndex(hn::Add(cur_hi, delta_hi), act_hi, dw, var_score, v_hi); +#endif + } + + if constexpr (!k_mask_remainder) { + PatchRowScalar(variables, coefficients, k, ke, var_score, nnz_score_delta, assign_i32, + weight, os_new, skip_var); + } +} + +// Native width, and the 8-lane variant for rows that would leave most of a native vector idle. +template +void PatchRowImpl(const int32_t* HWY_RESTRICT variables, + const coef_t* HWY_RESTRICT coefficients, + int32_t kb, + int32_t ke, + int64_t* HWY_RESTRICT var_score, + int64_t* HWY_RESTRICT nnz_score_delta, + const int32_t* HWY_RESTRICT assign_i32, + int32_t weight, + int32_t os_new, + int32_t skip_var) +{ + PatchRowBody(hn::ScalableTag(), variables, coefficients, kb, ke, var_score, + nnz_score_delta, assign_i32, weight, os_new, skip_var); +} + +template +void PatchRowNarrow8Impl(const int32_t* HWY_RESTRICT variables, + const coef_t* HWY_RESTRICT coefficients, + int32_t kb, + int32_t ke, + int64_t* HWY_RESTRICT var_score, + int64_t* HWY_RESTRICT nnz_score_delta, + const int32_t* HWY_RESTRICT assign_i32, + int32_t weight, + int32_t os_new, + int32_t skip_var) +{ + PatchRowBody(hn::CappedTagIfFixed(), variables, coefficients, kb, ke, + var_score, nnz_score_delta, assign_i32, weight, os_new, skip_var); +} + +template +void PatchRowNarrow4Impl(const int32_t* HWY_RESTRICT variables, + const coef_t* HWY_RESTRICT coefficients, + int32_t kb, + int32_t ke, + int64_t* HWY_RESTRICT var_score, + int64_t* HWY_RESTRICT nnz_score_delta, + const int32_t* HWY_RESTRICT assign_i32, + int32_t weight, + int32_t os_new, + int32_t skip_var) +{ + PatchRowBody(hn::CappedTagIfFixed(), variables, coefficients, kb, ke, + var_score, nnz_score_delta, assign_i32, weight, os_new, skip_var); +} + +// Longest row worth sending to each narrower kernel, or 0 where that width is not worth having. +// A gather costs the same whether its lanes carry data or are masked off, so a row that fills only +// part of a native vector is cheaper through a narrower one; past the crossover the extra vector +// and its extra full gather cost more than the wasted lanes. From the Zen 4 microcode ratio +// (VPGATHERDD ~78 uops at 512 bits, 48 at 256, 24 at 128) the crossovers land at 4 and 8. +// +// A width is offered only when it is strictly narrower than the native vector, so no target ever +// dispatches to a kernel identical to its own. Scalable targets opt out entirely: Highway notes +// that clamping Lanes() on RVV/SVE can cost more than the capping saves, which is why +// CappedTagIfFixed leaves them at native width above. +// +// These are per-target constants, so the width choice belongs here rather than at the call seam: a +// caller outside this file can only reach them through a dispatch pointer, which turns two +// immediates into two loads of runtime globals and puts an unpredictable branch directly in front +// of the indirect jump that follows it. Measured on supportcase22, that seam cost 2.4%. +constexpr size_t k_native_lanes = HWY_MAX_LANES_D(hn::ScalableTag); +constexpr int32_t k_narrow4_max = HWY_HAVE_SCALABLE ? 0 : (k_native_lanes > 4 ? 4 : 0); +constexpr int32_t k_narrow8_max = HWY_HAVE_SCALABLE ? 0 : (k_native_lanes > 8 ? 8 : 0); + +// Single entry point the seam dispatches to. On a scalable target both bounds are 0, so both +// compares fold away and the narrow arms are stripped. +template +void PatchRowDispatchImpl(const int32_t* HWY_RESTRICT variables, + const coef_t* HWY_RESTRICT coefficients, + int32_t kb, + int32_t ke, + int64_t* HWY_RESTRICT var_score, + int64_t* HWY_RESTRICT nnz_score_delta, + const int32_t* HWY_RESTRICT assign_i32, + int32_t weight, + int32_t os_new, + int32_t skip_var) +{ + const int32_t row_len = ke - kb; + if (row_len <= k_narrow4_max) { + PatchRowNarrow4Impl(variables, coefficients, kb, ke, var_score, nnz_score_delta, + assign_i32, weight, os_new, skip_var); + } else if (row_len <= k_narrow8_max) { + PatchRowNarrow8Impl(variables, coefficients, kb, ke, var_score, nnz_score_delta, + assign_i32, weight, os_new, skip_var); + } else { + PatchRowImpl(variables, coefficients, kb, ke, var_score, nnz_score_delta, assign_i32, + weight, os_new, skip_var); + } +} + +// Tiled sweep carrying a running maximum. The index re-scan fires only on a tile that raises it, +// and that tile is still cache-hot. The tabu window is uint16 against int32 scores, so the mask +// crosses a 2:1 width boundary through PromoteMaskTo. +void ArgmaxImpl(const int64_t* HWY_RESTRICT var_score, + int32_t n, + int32_t tile, + int32_t* best_var, + int64_t* best_score) +{ + const hn::ScalableTag d; + using V = hn::Vec; + + const int32_t step = (int32_t)hn::Lanes(d); + const V vmin = hn::Set(d, fj_bin_score_invalid); + + // Whole vectors only; the remainder is scanned scalar below. + const int32_t nblk = n - (n % step); + int32_t tile_step = tile - (tile % step); + if (tile_step < step) tile_step = step; + + int32_t bv = -1; + int64_t bs = fj_bin_score_invalid; + + for (int32_t t0 = 0; t0 < nblk; t0 += tile_step) { + const int32_t t1 = (t0 + tile_step < nblk) ? t0 + tile_step : nblk; + + V tile_max = vmin; + for (int32_t v = t0; v < t1; v += step) { + tile_max = hn::Max(tile_max, hn::LoadU(d, var_score + v)); + } + + const int64_t peak = hn::ReduceMax(d, tile_max); + if (peak > bs) { + const V vpeak = hn::Set(d, peak); + for (int32_t v = t0; v < t1; v += step) { + const intptr_t lane = hn::FindFirstTrue(d, hn::Eq(hn::LoadU(d, var_score + v), vpeak)); + if (lane >= 0) { + bv = v + (int32_t)lane; + break; + } + } + bs = peak; + } + } + + for (int32_t v = nblk; v < n; ++v) { + if (var_score[v] > bs) { + bs = var_score[v]; + bv = v; + } + } + + *best_var = bv; + *best_score = bs; +} + +// combined[v] = var_score[v] + obj_score[v] over all n variables. Materialized rather than fused +// into the argmax because block_tabu writes sentinels into the result and restores them afterwards, +// so the array has to outlive the scan. None of the three has SIMD padding, hence the scalar tail. +void AddScoresImpl(const int64_t* HWY_RESTRICT var_score, + const int64_t* HWY_RESTRICT obj_score, + int32_t n, + int64_t* HWY_RESTRICT combined) +{ + const hn::ScalableTag d; + const int32_t step = (int32_t)hn::Lanes(d); + const int32_t nblk = n - (n % step); + + for (int32_t v = 0; v < nblk; v += step) { + hn::StoreU(hn::Add(hn::LoadU(d, var_score + v), hn::LoadU(d, obj_score + v)), d, combined + v); + } + for (int32_t v = nblk; v < n; ++v) { + combined[v] = var_score[v] + obj_score[v]; + } +} + +} // namespace HWY_NAMESPACE +} // namespace cuopt::mathematical_optimization::mip +HWY_AFTER_NAMESPACE(); + +#if HWY_ONCE +namespace cuopt::mathematical_optimization::mip { + +// One dispatch table per (coefficient width, vector width). HWY_EXPORT_T names the table +// separately from the function, which lets the function be a template-id: only the table name goes +// through token pasting, so no hand-written non-template wrapper is needed. The template argument +// must stay comma-free, which is why the three tag-binding wrappers above take only coef_t. +HWY_EXPORT_T(PatchRowI8, PatchRowDispatchImpl); +HWY_EXPORT_T(PatchRowI16, PatchRowDispatchImpl); +HWY_EXPORT_T(WalkRowsI8, WalkRowsImpl); +HWY_EXPORT_T(WalkRowsI16, WalkRowsImpl); +HWY_EXPORT(ArgmaxImpl); +HWY_EXPORT(AddScoresImpl); + +// HWY_DYNAMIC_DISPATCH resolves the target on every call, and the hwy::GetChosenTarget() call it +// expands to is a real out-of-line call: it clobbers the argument registers, so the compiler spills +// all eleven parameters to the stack and reloads them around it. These run once per row per move, +// so the pointers are resolved once instead. +// +// Entry 0 of a dispatch table is a trampoline that chooses the target and re-dispatches, and an +// unchosen target makes GetIndex() return 0. Caching then would pin that extra indirection for the +// process lifetime, so the target is chosen first. File scope rather than function scope keeps the +// guard variable of a magic static out of the call: its cold path can call __cxa_guard_acquire, so +// the compiler must preserve the arguments across it and cannot leave a bare tail jump. Nothing in +// cuOpt reaches feasibility jump during static initialization. +static void fj_bin_choose_target() +{ + if (!hwy::GetChosenTarget().IsInitialized()) { + hwy::GetChosenTarget().Update(hwy::SupportedTargets()); + } +} + +template +using fj_bin_patch_fn_t = void (*)(const int32_t*, + const coef_t*, + int32_t, + int32_t, + int64_t*, + int64_t*, + const int32_t*, + int32_t, + int32_t, + int32_t); + +// The vector width is chosen inside the target (see PatchRowDispatchImpl), so the seam carries one +// pointer per coefficient width and nothing else. +static const auto fj_bin_patch_i8 = + (fj_bin_choose_target(), (fj_bin_patch_fn_t)HWY_DYNAMIC_POINTER_T(PatchRowI8)); +static const auto fj_bin_patch_i16 = + (fj_bin_choose_target(), (fj_bin_patch_fn_t)HWY_DYNAMIC_POINTER_T(PatchRowI16)); + +// Overloaded rather than specialized, matching fj_bin_walk_fn below. +static fj_bin_patch_fn_t fj_bin_patch_fn(int8_t) { return fj_bin_patch_i8; } +static fj_bin_patch_fn_t fj_bin_patch_fn(int16_t) { return fj_bin_patch_i16; } + +template +using fj_bin_walk_fn_t = int32_t (*)( + int32_t*, const int32_t*, const coef_t*, const coef_t*, int32_t, int32_t, int32_t, int32_t*); + +static const auto fj_bin_walk_i8 = + (fj_bin_choose_target(), (fj_bin_walk_fn_t)HWY_DYNAMIC_POINTER_T(WalkRowsI8)); +static const auto fj_bin_walk_i16 = + (fj_bin_choose_target(), (fj_bin_walk_fn_t)HWY_DYNAMIC_POINTER_T(WalkRowsI16)); + +static fj_bin_walk_fn_t fj_bin_walk_fn(int8_t) { return fj_bin_walk_i8; } +static fj_bin_walk_fn_t fj_bin_walk_fn(int16_t) { return fj_bin_walk_i16; } + +static const auto fj_bin_argmax_fn = (fj_bin_choose_target(), HWY_DYNAMIC_POINTER(ArgmaxImpl)); +static const auto fj_bin_add_scores_fn = + (fj_bin_choose_target(), HWY_DYNAMIC_POINTER(AddScoresImpl)); + +template +int32_t fj_bin_walk_rows(int32_t* row_slack, + const int32_t* incident_row, + const coef_t* reverse_coefficients, + const coef_t* incident_row_cmax, + int32_t incidence_begin, + int32_t incidence_end, + int32_t delta, + int32_t* out_incidence) +{ + return fj_bin_walk_fn(coef_t{})(row_slack, + incident_row, + reverse_coefficients, + incident_row_cmax, + incidence_begin, + incidence_end, + delta, + out_incidence); +} + +template int32_t fj_bin_walk_rows( + int32_t*, const int32_t*, const int8_t*, const int8_t*, int32_t, int32_t, int32_t, int32_t*); +template int32_t fj_bin_walk_rows( + int32_t*, const int32_t*, const int16_t*, const int16_t*, int32_t, int32_t, int32_t, int32_t*); + +template +void fj_bin_patch_row(const int32_t* variables, + const coef_t* coefficients, + int32_t kb, + int32_t ke, + int64_t* var_score, + int64_t* nnz_score_delta, + const int32_t* assign_i32, + int32_t weight, + int32_t os_new, + int32_t skip_var) +{ + fj_bin_patch_fn(coef_t{})(variables, coefficients, kb, ke, var_score, nnz_score_delta, assign_i32, + weight, os_new, skip_var); +} + +template void fj_bin_patch_row(const int32_t*, + const int8_t*, + int32_t, + int32_t, + int64_t*, + int64_t*, + const int32_t*, + int32_t, + int32_t, + int32_t); + +template void fj_bin_patch_row(const int32_t*, + const int16_t*, + int32_t, + int32_t, + int64_t*, + int64_t*, + const int32_t*, + int32_t, + int32_t, + int32_t); + +void fj_bin_argmax(const int64_t* var_score, + int32_t n, + int32_t tile, + int32_t& best_var, + int64_t& best_score) +{ + fj_bin_argmax_fn(var_score, n, tile, &best_var, &best_score); +} + +void fj_bin_add_scores(const int64_t* var_score, + const int64_t* obj_score, + int32_t n, + int64_t* combined) +{ + fj_bin_add_scores_fn(var_score, obj_score, n, combined); +} + +} // namespace cuopt::mathematical_optimization::mip +#endif // HWY_ONCE diff --git a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_worker.cuh b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_worker.cuh index bb2c69f81c..b30081059b 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_worker.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/fj_cpu_worker.cuh @@ -24,6 +24,13 @@ namespace cuopt::mathematical_optimization::mip { template struct fj_cpu_climber_t; +template +struct fj_cpu_shared_incumbent_t; + +// Defined in fj_cpu.cu, where the type is complete. +template +std::shared_ptr> make_fj_cpu_shared_incumbent(); + template struct fj_cpu_worker_t { // Custom deleter to avoid pulling the entire fj_cpu_climber_t class here. @@ -35,19 +42,26 @@ struct fj_cpu_worker_t { std::atomic preemption_flag{false}; std::unique_ptr, fj_cpu_deleter_t> fj_cpu; std::function&, double)> improvement_callback; + // Set before create_worker to join a portfolio; left null when the climber runs alone. + std::shared_ptr> shared_incumbent; ~fj_cpu_worker_t() { stop(); } + // `n_structural` is where `problem`'s slack block starts; those columns fold into two-sided row + // bounds, so the climber and the assignment it reports span only the ones below. -1 keeps them. // `seed` selects the FJ RNG seed: pass a non-negative value for a deterministic seed, // or -1 to draw from the global cuopt::seed_generator (the historical behavior). // In deterministic mode the caller MUST pass an explicit seed, otherwise the underlying // seed_generator::get_seed() racing with concurrent callers breaks reproducibility. + // `lane` >= 0 applies that lane's persona from the portfolio diversification ladder. void create_worker(const simplex::lp_problem_t& problem, const std::vector& variable_types, + i_t n_structural, const std::vector& seed_assignment, const simplex::simplex_solver_settings_t& settings, std::string log_prefix, - int64_t seed = -1); + int64_t seed = -1, + int lane = -1); // Run the worker asynchronously (i.e., launch an openmp task and then continue the // execution). Call `stop()` for stopping the worker diff --git a/cpp/src/mip_heuristics/local_search/local_search.cu b/cpp/src/mip_heuristics/local_search/local_search.cu index 23edf555cd..c1d80fcda7 100644 --- a/cpp/src/mip_heuristics/local_search/local_search.cu +++ b/cpp/src/mip_heuristics/local_search/local_search.cu @@ -79,6 +79,7 @@ void local_search_t::start_cpufj_scratch_threads(population_timprovement_callback = [this, &population, problem_ptr = context.problem_ptr]( f_t obj, const std::vector& h_vec, double /*work_units*/) { + context.solution_publication.publish_if_better(problem_ptr, h_vec, obj); population.add_external_solution(h_vec, obj, solution_origin_t::CPUFJ); (void)problem_ptr; if (obj < this->local_search_best_obj) { @@ -127,6 +128,7 @@ void local_search_t::start_cpufj_lptopt_scratch_threads( scratch_cpu_fj_on_lp_opt->log_prefix = "******* scratch on LP optimal: "; scratch_cpu_fj_on_lp_opt->improvement_callback = [this, &population](f_t obj, const std::vector& h_vec, double /*work_units*/) { + context.solution_publication.publish_if_better(context.problem_ptr, h_vec, obj); population.add_external_solution(h_vec, obj, solution_origin_t::CPUFJ); if (obj < this->local_search_best_obj) { CUOPT_LOG_DEBUG("******* New local search best obj %g, best overall %g", diff --git a/cpp/src/mip_heuristics/mip_constants.hpp b/cpp/src/mip_heuristics/mip_constants.hpp index f3fb68343a..e5c85a65fa 100644 --- a/cpp/src/mip_heuristics/mip_constants.hpp +++ b/cpp/src/mip_heuristics/mip_constants.hpp @@ -21,6 +21,14 @@ #define CUOPT_MIP_BATCH_PDLP_REQUIRED_THREAD_COUNT 3 #define CUOPT_MIP_CLIQUE_CUTS_REQUIRED_THREAD_COUNT 3 +/* @brief Threads the early CPUFJ portfolio leaves to the rest of the team. Every lane holds its + * own host copy of the problem and occupies an OMP task for the whole of presolve. */ +#define CUOPT_MIP_EARLY_CPUFJ_RESERVED_THREADS 4 + +/* @brief Upper bound on the persistent root CPUFJ lane set. Every lane holds its own host copy of + * the root LP and occupies an OMP task for the whole of the cut loop. */ +#define CUOPT_MIP_ROOT_CPUFJ_MAX_LANES 4 + // MIP-only gate: skip the concurrent barrier when fewer threads are available than this // (1 PDLP + 1 dual simplex + 1 barrier). Stand-alone LP always runs all three. #define CUOPT_CONCURRENT_LP_BARRIER_REQUIRED_THREAD_COUNT 3 diff --git a/cpp/src/mip_heuristics/presolve/semi_continuous.cu b/cpp/src/mip_heuristics/presolve/semi_continuous.cu index 33b7efff0e..51d2552746 100644 --- a/cpp/src/mip_heuristics/presolve/semi_continuous.cu +++ b/cpp/src/mip_heuristics/presolve/semi_continuous.cu @@ -115,6 +115,8 @@ bool reformulate_semi_continuous(optimization_problem_t& op_problem, std::vector* used_fallback_big_m, std::vector* semi_continuous_binary_to_original_indices) { + if (!op_problem.has_semi_continuous_variables()) { return false; } + // 1. Identify semi-continuous variables auto var_types = op_problem.get_variable_types_host(); auto var_lb = op_problem.get_variable_lower_bounds_host(); diff --git a/cpp/src/mip_heuristics/problem/presolve_data.cu b/cpp/src/mip_heuristics/problem/presolve_data.cu index 3c621bc2cd..ae78c3778c 100644 --- a/cpp/src/mip_heuristics/problem/presolve_data.cu +++ b/cpp/src/mip_heuristics/problem/presolve_data.cu @@ -257,8 +257,8 @@ void presolve_data_t::set_papilo_presolve_data( } template -void presolve_data_t::papilo_uncrush_assignment( - problem_t& problem, rmm::device_uvector& assignment) const +void presolve_data_t::papilo_uncrush_assignment(rmm::device_uvector& assignment, + rmm::cuda_stream_view stream) const { if (papilo_presolve_ptr == nullptr) { CUOPT_LOG_INFO("Papilo presolve data not set, skipping uncrushing assignment"); @@ -266,15 +266,12 @@ void presolve_data_t::papilo_uncrush_assignment( } cuopt_assert(assignment.size() == papilo_reduced_to_original_map.size(), "Papilo uncrush assignment size mismatch"); - auto h_assignment = cuopt::host_copy(assignment, problem.handle_ptr->get_stream()); + auto h_assignment = cuopt::host_copy(assignment, stream); std::vector full_assignment; papilo_presolve_ptr->uncrush_primal_solution(h_assignment, full_assignment); - assignment.resize(full_assignment.size(), problem.handle_ptr->get_stream()); - raft::copy(assignment.data(), - full_assignment.data(), - full_assignment.size(), - problem.handle_ptr->get_stream()); - problem.handle_ptr->sync_stream(); + assignment.resize(full_assignment.size(), stream); + raft::copy(assignment.data(), full_assignment.data(), full_assignment.size(), stream); + stream.synchronize(); } #if MIP_INSTANTIATE_FLOAT || PDLP_INSTANTIATE_FLOAT diff --git a/cpp/src/mip_heuristics/problem/presolve_data.cuh b/cpp/src/mip_heuristics/problem/presolve_data.cuh index 713bb24c0d..65b492a7e6 100644 --- a/cpp/src/mip_heuristics/problem/presolve_data.cuh +++ b/cpp/src/mip_heuristics/problem/presolve_data.cuh @@ -122,8 +122,8 @@ class presolve_data_t { i_t original_num_variables); bool has_papilo_presolve_data() const { return papilo_presolve_ptr != nullptr; } i_t get_papilo_original_num_variables() const { return papilo_original_num_variables; } - void papilo_uncrush_assignment(problem_t& problem, - rmm::device_uvector& assignment) const; + void papilo_uncrush_assignment(rmm::device_uvector& assignment, + rmm::cuda_stream_view stream) const; presolve_data_t(presolve_data_t&&) = default; presolve_data_t& operator=(presolve_data_t&&) = default; diff --git a/cpp/src/mip_heuristics/problem/problem.cu b/cpp/src/mip_heuristics/problem/problem.cu index 17d55b7cc2..b84206e08f 100644 --- a/cpp/src/mip_heuristics/problem/problem.cu +++ b/cpp/src/mip_heuristics/problem/problem.cu @@ -2192,9 +2192,10 @@ void problem_t::set_papilo_presolve_data( } template -void problem_t::papilo_uncrush_assignment(rmm::device_uvector& assignment) const +void problem_t::papilo_uncrush_assignment(rmm::device_uvector& assignment, + rmm::cuda_stream_view stream) const { - presolve_data.papilo_uncrush_assignment(const_cast(*this), assignment); + presolve_data.papilo_uncrush_assignment(assignment, stream); } template diff --git a/cpp/src/mip_heuristics/problem/problem.cuh b/cpp/src/mip_heuristics/problem/problem.cuh index 3ea3973d1d..fd38117b50 100644 --- a/cpp/src/mip_heuristics/problem/problem.cuh +++ b/cpp/src/mip_heuristics/problem/problem.cuh @@ -119,7 +119,12 @@ class problem_t { { return presolve_data.get_papilo_original_num_variables(); } - void papilo_uncrush_assignment(rmm::device_uvector& assignment) const; + void papilo_uncrush_assignment(rmm::device_uvector& assignment, + rmm::cuda_stream_view stream) const; + void papilo_uncrush_assignment(rmm::device_uvector& assignment) const + { + papilo_uncrush_assignment(assignment, handle_ptr->get_stream()); + } void compute_transpose_of_problem(); f_t get_user_obj_from_solver_obj(f_t solver_obj) const; f_t get_solver_obj_from_user_obj(f_t user_obj) const; diff --git a/cpp/src/mip_heuristics/root_heuristics.hpp b/cpp/src/mip_heuristics/root_heuristics.hpp index 1f29b25eef..17a7a73e51 100644 --- a/cpp/src/mip_heuristics/root_heuristics.hpp +++ b/cpp/src/mip_heuristics/root_heuristics.hpp @@ -9,8 +9,15 @@ #include #include +#include #include "feasibility_jump/fj_cpu_worker.cuh" +#include +#include +#include +#include +#include + namespace cuopt::mathematical_optimization::mip { template @@ -91,19 +98,63 @@ struct root_heuristics_t { std::shared_ptr> worker_count_; i_t max_workers_; + // CPU FJ lanes that outlive a single cut pass. + std::vector>> persistent_lanes_; + // Shared by every CPU FJ lane of the root phase, persistent and per-cut-pass alike. + std::shared_ptr> shared_incumbent_; + root_heuristics_t(i_t max_workers) - : worker_count_(std::make_shared>(0)), max_workers_(max_workers) + : worker_count_(std::make_shared>(0)), + max_workers_(max_workers), + shared_incumbent_(make_fj_cpu_shared_incumbent()) { } ~root_heuristics_t() { stop_and_sync(); } + // Must be called from the same task region as stop_and_sync: run_async's task dependence is + // matched only by a taskwait in the encountering region. + void start_persistent_lanes(const simplex::lp_problem_t& lp, + const std::vector& var_types, + i_t n_structural, + const std::vector& seed_assignment, + const simplex::simplex_solver_settings_t& settings, + i_t n_lanes, + f_t time_limit, + int64_t base_seed, + std::function&, double)> callback) + { + persistent_lanes_.reserve(n_lanes); + for (i_t k = 0; k < n_lanes; ++k) { + auto lane = std::make_unique>(); + lane->improvement_callback = callback; + lane->shared_incumbent = shared_incumbent_; + lane->create_worker(lp, + var_types, + n_structural, + seed_assignment, + settings, + "[Root FJ lane " + std::to_string(k) + "] ", + base_seed + k, + k); + lane->run_async(time_limit); + persistent_lanes_.push_back(std::move(lane)); + } + } + void stop_and_sync() { + for (auto& lane : persistent_lanes_) { + lane->send_stop_signal(); + } for (auto& heuristic : cut_passes_heuristics_) { heuristic->send_stop_signal(); } + for (auto& lane : persistent_lanes_) { + lane->stop(); + } + persistent_lanes_.clear(); for (auto& heuristic : cut_passes_heuristics_) { heuristic->stop_and_sync(); } @@ -127,8 +178,12 @@ struct root_heuristics_t { cut_passes_heuristics_.erase(cut_passes_heuristics_.begin()); } - return cut_passes_heuristics_.emplace_back(std::make_shared>( - Arow, var_types, root_solution, root_edge_norm)); + auto& heuristic = cut_passes_heuristics_.emplace_back( + std::make_shared>( + Arow, var_types, root_solution, root_edge_norm)); + // Read by create_worker, so it has to be in place before the caller builds the climber. + heuristic->fj_cpu_worker_.shared_incumbent = shared_incumbent_; + return heuristic; } }; diff --git a/cpp/src/mip_heuristics/solution/solution.cu b/cpp/src/mip_heuristics/solution/solution.cu index 3b00fca7a8..197db0627c 100644 --- a/cpp/src/mip_heuristics/solution/solution.cu +++ b/cpp/src/mip_heuristics/solution/solution.cu @@ -5,6 +5,7 @@ */ /* clang-format on */ +#include #include "feasibility_test.cuh" #include "solution.cuh" #include "solution_kernels.cuh" @@ -652,4 +653,29 @@ template class solution_t; template class solution_t; #endif +template +void build_start_assignment(problem_t& problem, + solution_t& solution, + const raft::handle_t* handle_ptr) +{ + // Default: zero, projected into the variable bounds. Deliberately the simplest + // thing that works -- the seeding strategy is what this hook exists to change. + thrust::fill(handle_ptr->get_thrust_policy(), + solution.assignment.begin(), + solution.assignment.end(), + f_t{0}); + clamp_within_var_bounds(solution.assignment, &problem, handle_ptr); + handle_ptr->sync_stream(); +} + +#if MIP_INSTANTIATE_FLOAT +template void build_start_assignment( + problem_t&, solution_t&, const raft::handle_t*); +#endif + +#if MIP_INSTANTIATE_DOUBLE +template void build_start_assignment( + problem_t&, solution_t&, const raft::handle_t*); +#endif + } // namespace cuopt::mathematical_optimization::mip diff --git a/cpp/src/mip_heuristics/solution/solution.cuh b/cpp/src/mip_heuristics/solution/solution.cuh index f243937d3e..4839ba1fb8 100644 --- a/cpp/src/mip_heuristics/solution/solution.cuh +++ b/cpp/src/mip_heuristics/solution/solution.cuh @@ -153,4 +153,13 @@ class solution_t { void test_variable_bounds(bool check_integer = true, i_t* is_feasible = nullptr); }; +// Builds the start assignment every climber is derived from. Defined in +// solution.cu, so editing it recompiles one translation unit rather than every +// file that includes this header. Runs inside the measured window: a better start +// has to be worth what it costs to build. +template +void build_start_assignment(problem_t& problem, + solution_t& solution, + const raft::handle_t* handle_ptr); + } // namespace cuopt::mathematical_optimization::mip diff --git a/cpp/src/mip_heuristics/solution_publication.cuh b/cpp/src/mip_heuristics/solution_publication.cuh new file mode 100644 index 0000000000..0e5c1d92ea --- /dev/null +++ b/cpp/src/mip_heuristics/solution_publication.cuh @@ -0,0 +1,141 @@ +/* clang-format off */ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +/* clang-format on */ + +#pragma once + +#include +#include +#include + +#include +#include + +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +namespace cuopt::mathematical_optimization::mip { + +// Single point at which MIP incumbents are reported to the user get-solution callbacks. +// The heuristic thread (through the population) and the branch-and-bound thread both publish +// here, so the guard on the last published objective is shared and every incumbent is reported +// once, at the moment it is found rather than when the heuristic thread next drains its queue. +template +class solution_publication_t { + public: + solution_publication_t(const mip_solver_settings_t& settings, + const solver_stats_t& stats) + : settings_(settings), stats_(stats) + { + if (has_get_solution_callback()) { + RAFT_CUDA_TRY(cudaGetDevice(&device_id_)); + handle_ = std::make_unique(); + } + } + + // Whether any get-solution callback is registered. Callers can use this to skip assembling + // the host assignment that publish_if_better would otherwise discard. + bool enabled() const { return handle_ != nullptr; } + + // `assignment` and `solver_objective` are in problem_ptr's solver space, which is always + // oriented as a minimization. Returns whether the incumbent was published. + // + // Post-processing runs on a private stream, so this is safe to call from the branch-and-bound + // thread while the heuristic thread owns problem_ptr->handle_ptr's stream. + bool publish_if_better(problem_t* problem_ptr, + const std::vector& assignment, + f_t solver_objective) + { + if (handle_ == nullptr) { return false; } + cuopt_assert(problem_ptr != nullptr, "Publication problem pointer must not be null"); + cuopt_assert(std::isfinite(solver_objective), "Published objective must be finite"); + + std::lock_guard lock(mutex_); + if (!(solver_objective < best_published_objective_)) { return false; } + best_published_objective_ = solver_objective; + + const auto user_assignment = build_user_assignment(problem_ptr, assignment); + const f_t user_objective = problem_ptr->get_user_obj_from_solver_obj(solver_objective); + const f_t user_bound = stats_.get_solution_bound(); + CUOPT_LOG_DEBUG("Publishing incumbent: objective %g, %lu variables", + user_objective, + user_assignment.size()); + + for (auto callback : settings_.get_mip_callbacks()) { + if (callback == nullptr || + callback->get_type() != internals::base_solution_callback_type::GET_SOLUTION) { + continue; + } + // Each callback gets its own copies: the interface hands out mutable pointers. + std::vector callback_assignment(user_assignment); + std::vector callback_objective(1, user_objective); + std::vector callback_bound(1, user_bound); + auto get_sol_callback = static_cast(callback); + get_sol_callback->get_solution(callback_assignment.data(), + callback_objective.data(), + callback_bound.data(), + get_sol_callback->get_user_data()); + } + return true; + } + + private: + // Lifts a solver-space assignment into the space the callbacks were set up for. + std::vector build_user_assignment(problem_t* problem_ptr, + const std::vector& assignment) + { + // The B&B thread may never have selected a device of its own. + RAFT_CUDA_TRY(cudaSetDevice(device_id_)); + auto stream = handle_->get_stream(); + rmm::device_uvector d_assignment(assignment.size(), stream); + raft::copy(d_assignment.data(), assignment.data(), assignment.size(), stream); + // post_process_assignment writes through problem_ptr->presolve_data.fixed_var_assignment, + // which both publishing threads share: the caller's lock is what keeps them apart. + problem_ptr->post_process_assignment(d_assignment, true, stream); + if (problem_ptr->has_papilo_presolve_data()) { + problem_ptr->papilo_uncrush_assignment(d_assignment, stream); + } + auto user_assignment = cuopt::host_copy(d_assignment, stream); + if (mip_solver_settings_accessor::has_semi_continuous_callback_translation( + settings_)) { + strip_semi_continuous_auxiliaries_from_assignment( + user_assignment, + mip_solver_settings_accessor::get_semi_continuous_original_num_variables( + settings_)); + } + return user_assignment; + } + + bool has_get_solution_callback() const + { + for (auto callback : settings_.get_mip_callbacks()) { + if (callback != nullptr && + callback->get_type() == internals::base_solution_callback_type::GET_SOLUTION) { + return true; + } + } + return false; + } + + const mip_solver_settings_t& settings_; + const solver_stats_t& stats_; + int device_id_{0}; + // Null when no get-solution callback is registered, which also disables publication. + std::unique_ptr handle_; + std::mutex mutex_; + f_t best_published_objective_{std::numeric_limits::max()}; +}; + +} // namespace cuopt::mathematical_optimization::mip diff --git a/cpp/src/mip_heuristics/solve.cu b/cpp/src/mip_heuristics/solve.cu index 162a5ba291..c42bac4365 100644 --- a/cpp/src/mip_heuristics/solve.cu +++ b/cpp/src/mip_heuristics/solve.cu @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -254,7 +255,6 @@ mip_solution_t run_mip_solver( settings.determinism_mode != CUOPT_MODE_DETERMINISTIC && problem.original_problem_ptr->get_n_integers() > 0; if (run_early_cpufj) { - auto early_fj_start = std::chrono::steady_clock::now(); auto* presolver_ptr = problem.presolve_data.papilo_presolve_ptr; auto mip_callbacks = settings.get_mip_callbacks(); f_t no_bound = problem.presolve_data.objective_scaling_factor >= 0 ? (f_t)-1e20 : (f_t)1e20; @@ -269,22 +269,19 @@ mip_solution_t run_mip_solver( mip_solver_settings_accessor::get_semi_continuous_original_num_variables( settings), ctx_ptr = &solver.context, - early_fj_start](f_t solver_obj, - f_t user_obj, - const std::vector& assignment, - const char* heuristic_name) { + &timer](f_t solver_obj, + f_t user_obj, + const std::vector& assignment, + const char* heuristic_name) { std::vector user_assignment; presolver_ptr->uncrush_primal_solution(assignment, user_assignment); ctx_ptr->initial_incumbent_assignment = user_assignment; ctx_ptr->initial_upper_bound = user_obj; - double elapsed = - std::chrono::duration(std::chrono::steady_clock::now() - early_fj_start) - .count(); CUOPT_LOG_INFO( - "New solution from early primal heuristics (%s). Objective %+.6e. Time %.2f", + "New solution from early primal heuristics (%s). Objective %+.6e. Time %.3f", heuristic_name, user_obj, - elapsed); + timer.elapsed_time()); invoke_solution_callbacks(mip_callbacks, has_semi_continuous_callback_translation, semi_continuous_original_num_variables, @@ -299,9 +296,11 @@ mip_solution_t run_mip_solver( if (std::isfinite(initial_upper_bound)) { early_cpufj->set_best_objective(problem.get_solver_obj_from_user_obj(initial_upper_bound)); } - early_cpufj->start(); + early_cpufj->start(omp_get_num_threads() - CUOPT_MIP_EARLY_CPUFJ_RESERVED_THREADS); solver.context.early_cpufj_ptr = early_cpufj.get(); - CUOPT_LOG_DEBUG("Started early CPUFJ on papilo-presolved problem during cuOpt presolve"); + CUOPT_LOG_DEBUG( + "Started early CPUFJ on papilo-presolved problem during cuOpt presolve with %d lanes", + early_cpufj->lane_count()); } auto presolved_sol = solver.run_solver(); @@ -376,16 +375,6 @@ mip_solution_t solve_mip_helper(optimization_problem_t& op_p raft::common::nvtx::range fun_scope("Running solver"); auto timer = timer_t(time_limit); - problem_checking_t::check_problem_representation(op_problem); - problem_checking_t::check_initial_solution_representation(op_problem, settings); - - CUOPT_LOG_INFO( - "Solving a problem with %d constraints, %d variables (%d integers), and %d nonzeros", - op_problem.get_n_constraints(), - op_problem.get_n_variables(), - op_problem.get_n_integers(), - op_problem.get_nnz()); - // Reformulate semi-continuous variables (x = 0 OR L <= x <= U) before Papilo presolve. // Uses deterministic CPU bounds strengthening to derive tight upper bounds for SC vars with // infinite UB. @@ -407,15 +396,6 @@ mip_solution_t solve_mip_helper(optimization_problem_t& op_p settings, n_orig_before_sc, semi_continuous_binary_to_original_indices); } - op_problem.print_scaling_information(); - - // Check for crossing bounds. Return infeasible if there are any - if (problem_checking_t::has_crossing_bounds(op_problem)) { - return mip_solution_t(mip_termination_status_t::Infeasible, - solver_stats_t{}, - op_problem.get_handle_ptr()->get_stream()); - } - for (auto callback : settings.get_mip_callbacks()) { auto callback_num_variables = op_problem.get_n_variables(); if (mip_solver_settings_accessor::has_semi_continuous_callback_translation( @@ -444,16 +424,6 @@ mip_solution_t solve_mip_helper(optimization_problem_t& op_p } #endif - if (settings.mip_scaling != CUOPT_MIP_SCALING_OFF) { - mip::mip_scaling_strategy_t scaling(op_problem); - scaling.scale_problem(settings.mip_scaling != CUOPT_MIP_SCALING_NO_OBJECTIVE); - } - double presolve_time = 0.0; - std::unique_ptr> presolver; - std::optional> presolve_result_opt; - mip::problem_t problem( - op_problem, settings.get_tolerances(), settings.determinism_mode == CUOPT_MODE_DETERMINISTIC); - auto run_presolve = settings.presolver != presolver_t::None; bool has_set_solution_callback = false; for (auto callback : settings.get_mip_callbacks()) { @@ -481,8 +451,8 @@ mip_solution_t solve_mip_helper(optimization_problem_t& op_p std::vector early_incumbent_pool; // Track best incumbent found during presolve (shared across CPU and GPU FJ). - // early_best_objective is in the original problem's solver-space (always minimization), - // used for fast comparison in the callback. + // The CPU and GPU heuristics can use differently scaled solver spaces, so compare their + // objectives in a common minimization-oriented user space. // early_best_user_obj is the corresponding user-space objective, // passed to run_mip for correct cross-space conversion. // We attempt to crush early-heuristics solutions into the presolved space. @@ -491,7 +461,7 @@ mip_solution_t solve_mip_helper(optimization_problem_t& op_p // but is dropped due to these dual reductions, and we lose a good solution. // This is why we still keep the solution around in original-space // and later extract it at the end of the solve. - std::atomic early_best_objective{std::numeric_limits::infinity()}; + std::atomic early_best_user_score{std::numeric_limits::infinity()}; f_t early_best_user_obj{std::numeric_limits::infinity()}; std::vector early_best_user_assignment; std::mutex early_callback_mutex; @@ -500,57 +470,93 @@ mip_solution_t solve_mip_helper(optimization_problem_t& op_p std::unique_ptr> early_gpufj; bool run_early_fj = run_presolve && settings.determinism_mode != CUOPT_MODE_DETERMINISTIC && - op_problem.get_n_integers() > 0 && op_problem.get_n_constraints() > 0; - f_t no_bound = problem.presolve_data.objective_scaling_factor >= 0 ? (f_t)-1e20 : (f_t)1e20; - if (run_early_fj) { - auto early_fj_start = std::chrono::steady_clock::now(); - auto early_fj_callback = - [&early_best_objective, - &early_best_user_obj, - &early_best_user_assignment, - &early_incumbent_pool, - &early_callback_mutex, - early_fj_start, - mip_callbacks = settings.get_mip_callbacks(), - has_semi_continuous_callback_translation = - mip_solver_settings_accessor::has_semi_continuous_callback_translation( - settings), - semi_continuous_original_num_variables = - mip_solver_settings_accessor::get_semi_continuous_original_num_variables( - settings), - no_bound](f_t solver_obj, - f_t user_obj, - const std::vector& assignment, - const char* heuristic_name) { - std::lock_guard lock(early_callback_mutex); - if (solver_obj >= early_best_objective.load()) { return; } - early_best_objective.store(solver_obj); - early_best_user_obj = user_obj; - early_best_user_assignment = assignment; - early_incumbent_pool.push_back({user_obj, assignment}); - double elapsed = - std::chrono::duration(std::chrono::steady_clock::now() - early_fj_start) - .count(); - CUOPT_LOG_INFO( - "New solution from early primal heuristics (%s). Objective %+.6e. Time %.2f", - heuristic_name, - user_obj, - elapsed); - auto user_assignment = assignment; - invoke_solution_callbacks(mip_callbacks, - has_semi_continuous_callback_translation, - semi_continuous_original_num_variables, - user_obj, - user_assignment, - no_bound); - }; + op_problem.get_problem_category() != problem_category_t::LP && + op_problem.get_n_constraints() > 0; + const f_t objective_sense = op_problem.get_sense() ? f_t{-1} : f_t{1}; + f_t no_bound = objective_sense > f_t{0} ? (f_t)-1e20 : (f_t)1e20; + auto early_fj_callback = + [&early_best_user_score, + &early_best_user_obj, + &early_best_user_assignment, + &early_incumbent_pool, + &early_callback_mutex, + &timer, + objective_sense, + mip_callbacks = settings.get_mip_callbacks(), + has_semi_continuous_callback_translation = + mip_solver_settings_accessor::has_semi_continuous_callback_translation(settings), + semi_continuous_original_num_variables = + mip_solver_settings_accessor::get_semi_continuous_original_num_variables( + settings), + no_bound]( + f_t, f_t user_obj, const std::vector& assignment, const char* heuristic_name) { + std::lock_guard lock(early_callback_mutex); + const f_t objective = objective_sense * user_obj; + if (objective >= early_best_user_score.load()) { return; } + early_best_user_score.store(objective); + early_best_user_obj = user_obj; + early_best_user_assignment = assignment; + early_incumbent_pool.push_back({user_obj, assignment}); + CUOPT_LOG_INFO("New solution from early primal heuristics (%s). Objective %+.6e. Time %.3f", + heuristic_name, + user_obj, + timer.elapsed_time()); + auto user_assignment = assignment; + invoke_solution_callbacks(mip_callbacks, + has_semi_continuous_callback_translation, + semi_continuous_original_num_variables, + user_obj, + user_assignment, + no_bound); + }; + if (run_early_fj) { // Start early CPUFJ on original problem (will restart on presolved problem after Papilo) early_cpufj = std::make_unique>( op_problem, settings.get_tolerances(), early_fj_callback); - early_cpufj->start(); - CUOPT_LOG_DEBUG("Started early CPUFJ on original problem"); + // Papilo runs on its own threads, so the team is otherwise idle here. + early_cpufj->start(omp_get_num_threads() - CUOPT_MIP_EARLY_CPUFJ_RESERVED_THREADS); + CUOPT_LOG_DEBUG("Started early CPUFJ on original problem with %d lanes", + early_cpufj->lane_count()); + } + + auto early_cpufj_guard = cuopt::scope_guard([&]() { + if (early_cpufj) { + early_cpufj->stop(); + early_cpufj.reset(); + } + }); + + problem_checking_t::check_problem_representation(op_problem); + problem_checking_t::check_initial_solution_representation(op_problem, settings); + + CUOPT_LOG_INFO( + "Solving a problem with %d constraints, %d variables (%d integers), and %d nonzeros", + op_problem.get_n_constraints(), + op_problem.get_n_variables(), + op_problem.get_n_integers(), + op_problem.get_nnz()); + op_problem.print_scaling_information(); + + // Check for crossing bounds. Return infeasible if there are any + if (problem_checking_t::has_crossing_bounds(op_problem)) { + return mip_solution_t(mip_termination_status_t::Infeasible, + solver_stats_t{}, + op_problem.get_handle_ptr()->get_stream()); + } + + if (settings.mip_scaling != CUOPT_MIP_SCALING_OFF) { + mip::mip_scaling_strategy_t scaling(op_problem); + scaling.scale_problem(settings.mip_scaling != CUOPT_MIP_SCALING_NO_OBJECTIVE); + } + double presolve_time = 0.0; + std::unique_ptr> presolver; + std::optional> presolve_result_opt; + mip::problem_t problem( + op_problem, settings.get_tolerances(), settings.determinism_mode == CUOPT_MODE_DETERMINISTIC); + + if (run_early_fj) { // Start early GPU FJ (uses GPU while CPU is busy with Papilo) early_gpufj = std::make_unique>(op_problem, settings, early_fj_callback); diff --git a/cpp/src/mip_heuristics/solver.cu b/cpp/src/mip_heuristics/solver.cu index f8eac0c4d8..f0a5a3c5aa 100644 --- a/cpp/src/mip_heuristics/solver.cu +++ b/cpp/src/mip_heuristics/solver.cu @@ -69,6 +69,10 @@ struct branch_and_bound_solution_helper_t { void solution_callback(std::vector& solution, f_t objective) { + if (dm->context.settings.determinism_mode == CUOPT_MODE_OPPORTUNISTIC) { + dm->context.solution_publication.publish_if_better( + dm->context.problem_ptr, solution, objective); + } dm->population.add_external_solution(solution, objective, solution_origin_t::BRANCH_AND_BOUND); } @@ -197,12 +201,8 @@ solution_t mip_solver_t::run_solver() if (context.problem_ptr->empty) { CUOPT_LOG_INFO("Problem fully reduced in presolve"); sol.set_problem_fully_reduced(); - for (auto callback : context.settings.get_mip_callbacks()) { - if (callback->get_type() == internals::base_solution_callback_type::GET_SOLUTION) { - auto get_sol_callback = static_cast(callback); - dm.population.invoke_get_solution_callback(sol, get_sol_callback); - } - } + context.solution_publication.publish_if_better( + context.problem_ptr, sol.get_host_assignment(), sol.get_objective()); context.problem_ptr->post_process_solution(sol); return sol; } @@ -237,12 +237,8 @@ solution_t mip_solver_t::run_solver() if (run_presolve && context.problem_ptr->empty) { CUOPT_LOG_INFO("Problem full reduced in presolve"); sol.set_problem_fully_reduced(); - for (auto callback : context.settings.get_mip_callbacks()) { - if (callback->get_type() == internals::base_solution_callback_type::GET_SOLUTION) { - auto get_sol_callback = static_cast(callback); - dm.population.invoke_get_solution_callback(sol, get_sol_callback); - } - } + context.solution_publication.publish_if_better( + context.problem_ptr, sol.get_host_assignment(), sol.get_objective()); context.problem_ptr->post_process_solution(sol); return sol; } @@ -273,12 +269,8 @@ solution_t mip_solver_t::run_solver() sol.set_problem_fully_reduced(); } if (opt_sol.get_termination_status() == pdlp_termination_status_t::Optimal) { - for (auto callback : context.settings.get_mip_callbacks()) { - if (callback->get_type() == internals::base_solution_callback_type::GET_SOLUTION) { - auto get_sol_callback = static_cast(callback); - dm.population.invoke_get_solution_callback(sol, get_sol_callback); - } - } + context.solution_publication.publish_if_better( + context.problem_ptr, sol.get_host_assignment(), sol.get_objective()); } context.problem_ptr->post_process_solution(sol); return sol; @@ -445,10 +437,10 @@ solution_t mip_solver_t::run_solver() branch_and_bound->set_concurrent_lp_root_solve(true); context.problem_ptr->branch_and_bound_callback = - std::bind(&mip::branch_and_bound_t::set_solution_from_heuristics, - branch_and_bound.get(), - std::placeholders::_1, - std::placeholders::_2); + [bb = branch_and_bound.get()](const std::vector& solution, + heuristics_origin_t origin) { + return bb->set_solution_from_heuristics(solution, origin); + }; } else if (context.settings.determinism_mode == CUOPT_MODE_DETERMINISTIC) { branch_and_bound->set_concurrent_lp_root_solve(false); // TODO once deterministic GPU heuristics are integrated diff --git a/cpp/src/mip_heuristics/solver_context.cuh b/cpp/src/mip_heuristics/solver_context.cuh index f98386cbaf..344d4e8d86 100644 --- a/cpp/src/mip_heuristics/solver_context.cuh +++ b/cpp/src/mip_heuristics/solver_context.cuh @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -58,6 +59,8 @@ struct mip_solver_context_t { std::atomic preempt_heuristic_solver_ = false; const mip_solver_settings_t settings; solver_stats_t stats; + // Every incumbent reported to the user goes through here, from whichever thread found it. + solution_publication_t solution_publication{settings, stats}; // Work limit context for tracking work units in deterministic mode (shared across all timers in // GPU heuristic loop) work_limit_context_t gpu_heur_loop{"GPUHeur"}; diff --git a/cpp/src/pdlp/cpu_optimization_problem.cpp b/cpp/src/pdlp/cpu_optimization_problem.cpp index 4b970eb6ec..93e86b7da6 100644 --- a/cpp/src/pdlp/cpu_optimization_problem.cpp +++ b/cpp/src/pdlp/cpu_optimization_problem.cpp @@ -29,20 +29,36 @@ namespace cuopt::mathematical_optimization { namespace { -// Classify a problem as LP / MIP / IP from its (enum) variable types. Single source of truth -// shared by set_variable_types() and adopt_from_mps_data_model() so the detection rule lives in -// one place. Empty types (no variables declared) classify as LP, matching the populate path where -// set_variable_types() is skipped and the category keeps its LP default. -problem_category_t problem_category_from_variable_types(const std::vector& variable_types) -{ - if (variable_types.empty()) { return problem_category_t::LP; } - const std::size_t n_discrete = static_cast( - std::count_if(variable_types.begin(), variable_types.end(), [](var_t v) { - return v == var_t::INTEGER || v == var_t::SEMI_CONTINUOUS; - })); - if (n_discrete == variable_types.size()) { return problem_category_t::IP; } - if (n_discrete > 0) { return problem_category_t::MIP; } - return problem_category_t::LP; +// Classify a problem as LP / MIP / IP from its (enum) variable types, and whether any +// SEMI_CONTINUOUS vars are present. Single source of truth shared by set_variable_types() and +// adopt_from_mps_data_model() so the detection rule lives in one place. Empty types (no variables +// declared) classify as LP with no SC, matching the populate path where set_variable_types() is +// skipped and the category keeps its LP default. +struct variable_type_summary_t { + problem_category_t category; + bool has_semi_continuous; +}; + +variable_type_summary_t summarize_variable_types(const std::vector& variable_types) +{ + if (variable_types.empty()) { + return {problem_category_t::LP, false}; + } + size_t n_discrete = 0; + bool has_semi_continuous = false; + for (var_t v : variable_types) { + if (v == var_t::SEMI_CONTINUOUS) { + has_semi_continuous = true; + ++n_discrete; + } else if (v == var_t::INTEGER) { + ++n_discrete; + } + } + if (n_discrete == variable_types.size()) { + return {problem_category_t::IP, has_semi_continuous}; + } + if (n_discrete > 0) { return {problem_category_t::MIP, has_semi_continuous}; } + return {problem_category_t::LP, false}; } } // namespace @@ -232,7 +248,9 @@ void cpu_optimization_problem_t::set_variable_types(const var_t* varia variable_types_.resize(size); std::copy(variable_types, variable_types + size, variable_types_.begin()); - problem_category_ = problem_category_from_variable_types(variable_types_); + const auto summary = summarize_variable_types(variable_types_); + problem_category_ = summary.category; + has_semi_continuous_variables_ = summary.has_semi_continuous; } template @@ -513,6 +531,12 @@ problem_category_t cpu_optimization_problem_t::get_problem_category() return problem_category_; } +template +bool cpu_optimization_problem_t::has_semi_continuous_variables() const noexcept +{ + return has_semi_continuous_variables_; +} + template const std::vector& cpu_optimization_problem_t::get_variable_names() const { @@ -1171,7 +1195,9 @@ void cpu_optimization_problem_t::adopt_from_mps_data_model( for (size_t i = 0; i < model.var_types_.size(); ++i) { variable_types_[i] = char_to_var_type(model.var_types_[i]); } - problem_category_ = problem_category_from_variable_types(variable_types_); + const auto summary = summarize_variable_types(variable_types_); + problem_category_ = summary.category; + has_semi_continuous_variables_ = summary.has_semi_continuous; if (model.has_quadratic_constraints()) { move_quadratic_constraints_from_model(*this, model.quadratic_constraints_); diff --git a/cpp/src/pdlp/optimization_problem.cu b/cpp/src/pdlp/optimization_problem.cu index 95457e2556..d4f669a118 100644 --- a/cpp/src/pdlp/optimization_problem.cu +++ b/cpp/src/pdlp/optimization_problem.cu @@ -54,6 +54,8 @@ namespace cuopt::mathematical_optimization { +constexpr size_t host_variable_type_summary_limit = 50'000; + template optimization_problem_t::optimization_problem_t(raft::handle_t const* handle_ptr) : handle_ptr_(handle_ptr), @@ -101,6 +103,7 @@ optimization_problem_t::optimization_problem_t( objective_name_{other.get_objective_name()}, problem_name_{other.get_problem_name()}, problem_category_{other.get_problem_category()}, + has_semi_continuous_variables_{other.has_semi_continuous_variables()}, var_names_{other.get_variable_names()}, row_names_{other.get_row_names()}, quadratic_constraints_{other.get_quadratic_constraints()} @@ -285,14 +288,40 @@ void optimization_problem_t::set_variable_types(const var_t* variable_ variable_types_.resize(size, stream_view_); raft::copy(variable_types_.data(), variable_types, size, stream_view_); - // Auto-detect problem category based on variable types. + // Auto-detect problem category and cache presence of SEMI_CONTINUOUS vars. // SEMI_CONTINUOUS vars will be reformulated into binary + continuous before solving, // so a problem with only SC vars is treated as MIP. - i_t n_discrete = thrust::count_if( - handle_ptr_->get_thrust_policy(), - variable_types_.begin(), - variable_types_.end(), - [] __device__(auto val) { return val == var_t::INTEGER || val == var_t::SEMI_CONTINUOUS; }); + // Prefer host-side for small instances to reduce latency between launch and first-feasible. + i_t n_discrete = 0; + bool has_semi_continuous_variables = false; + if ((size_t)size < host_variable_type_summary_limit) { + const auto h_variable_types = cuopt::host_copy(variable_types_, stream_view_); + for (const var_t val : h_variable_types) { + if (val == var_t::SEMI_CONTINUOUS) { + has_semi_continuous_variables = true; + ++n_discrete; + } else if (val == var_t::INTEGER) { + ++n_discrete; + } + } + } else { + auto is_discrete = [] __host__ __device__(var_t val) { + return val == var_t::INTEGER || val == var_t::SEMI_CONTINUOUS; + }; + auto is_semi_continuous = [] __host__ __device__(var_t val) { + return val == var_t::SEMI_CONTINUOUS; + }; + n_discrete = thrust::count_if(handle_ptr_->get_thrust_policy(), + variable_types_.begin(), + variable_types_.end(), + is_discrete); + has_semi_continuous_variables = + thrust::count_if(handle_ptr_->get_thrust_policy(), + variable_types_.begin(), + variable_types_.end(), + is_semi_continuous) > 0; + } + has_semi_continuous_variables_ = has_semi_continuous_variables; if (n_discrete == size) { problem_category_ = problem_category_t::IP; } else if (n_discrete > 0) { @@ -580,6 +609,12 @@ problem_category_t optimization_problem_t::get_problem_category() cons return problem_category_; } +template +bool optimization_problem_t::has_semi_continuous_variables() const noexcept +{ + return has_semi_continuous_variables_; +} + template const std::vector& optimization_problem_t::get_variable_names() const { diff --git a/cpp/src/utilities/macros.cuh b/cpp/src/utilities/macros.cuh index d36832015a..380851627f 100644 --- a/cpp/src/utilities/macros.cuh +++ b/cpp/src/utilities/macros.cuh @@ -14,7 +14,7 @@ // 3) heavy #ifdef ASSERT_MODE #include -#define cuopt_assert(val, msg) assert(val&& msg) +#define cuopt_assert(val, msg) assert((val) && msg) #define cuopt_func_call(func) func; #else #define cuopt_assert(val, msg) diff --git a/cpp/src/utilities/version_info.cpp b/cpp/src/utilities/version_info.cpp index 71dfc20c22..3fe1074f87 100644 --- a/cpp/src/utilities/version_info.cpp +++ b/cpp/src/utilities/version_info.cpp @@ -12,135 +12,224 @@ #include #include -#include -#include -#include -#include -#include -#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include namespace cuopt { -static int get_physical_cores() +// Reads up to buf_size-1 bytes, NUL-terminates, strips trailing whitespace/NULs. +// Returns bytes kept (excluding the terminator), or -1 on failure. +static ssize_t read_file_buf(const char* path, char* buf, size_t buf_size) { - std::ifstream cpuinfo("/proc/cpuinfo"); - if (!cpuinfo.is_open()) return 0; - - std::string line; - int physical_id = -1, core_id = -1; - std::set> cores; - - while (std::getline(cpuinfo, line)) { - if (line.find("physical id") != std::string::npos) { - physical_id = std::stoi(line.substr(line.find(":") + 1)); - } else if (line.find("core id") != std::string::npos) { - core_id = std::stoi(line.substr(line.find(":") + 1)); + if (buf_size == 0) return -1; + const int fd = open(path, O_RDONLY); + if (fd < 0) return -1; + const ssize_t n = read(fd, buf, buf_size - 1); + close(fd); + if (n < 0) return -1; + buf[n] = '\0'; + + // Device-tree properties are often NUL-terminated without a trailing newline. + size_t len = 0; + while (len < (size_t)n && buf[len] != '\0') { + ++len; + } + buf[len] = '\0'; + while (len > 0 && + (buf[len - 1] == '\n' || buf[len - 1] == '\r' || buf[len - 1] == ' ' || + buf[len - 1] == '\t')) { + buf[--len] = '\0'; + } + return (ssize_t)len; +} + +// Parses a kernel CPU list ("0-3,8,10-11") into cpus[0..max_cpus). Returns count written. +static int parse_cpu_list(const char* list, int* cpus, int max_cpus) +{ + int count = 0; + const char* p = list; + while (*p && count < max_cpus) { + while (*p == ',' || *p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') { + ++p; } + if (*p == '\0') break; + + char* end = nullptr; + const long lo = std::strtol(p, &end, 10); + if (end == p) break; + p = end; - if (physical_id != -1 && core_id != -1) { - cores.insert({physical_id, core_id}); - physical_id = -1; - core_id = -1; + if (*p == '-') { + ++p; + const long hi = std::strtol(p, &end, 10); + if (end == p) break; + p = end; + for (long cpu = lo; cpu <= hi && count < max_cpus; ++cpu) { + cpus[count++] = (int)cpu; + } + } else { + cpus[count++] = (int)lo; } } + return count; +} - if (cores.empty()) { - cpuinfo.clear(); - cpuinfo.seekg(0); - while (std::getline(cpuinfo, line)) { - if (line.find("cpu cores") != std::string::npos) { - return std::stoi(line.substr(line.find(":") + 1)); +static void mark_cpus_from_list(const char* list, char visited[CPU_SETSIZE]) +{ + const char* p = list; + while (*p) { + while (*p == ',' || *p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') { + ++p; + } + if (*p == '\0') break; + + char* end = nullptr; + const long lo = std::strtol(p, &end, 10); + if (end == p) break; + p = end; + + if (*p == '-') { + ++p; + const long hi = std::strtol(p, &end, 10); + if (end == p) break; + p = end; + for (long cpu = lo; cpu <= hi; ++cpu) { + if (cpu >= 0 && cpu < CPU_SETSIZE) { visited[cpu] = 1; } } + } else if (lo >= 0 && lo < CPU_SETSIZE) { + visited[lo] = 1; + } + } +} + +// CPUs this process may run on (respects Slurm/cgroup cpusets, taskset, etc.). +static int get_allowed_cpus(int* cpus, int max_cpus) +{ + cpu_set_t set; + CPU_ZERO(&set); + int count = 0; + if (sched_getaffinity(0, sizeof(set), &set) == 0) { + for (int cpu = 0; cpu < CPU_SETSIZE && count < max_cpus; ++cpu) { + if (CPU_ISSET(cpu, &set)) { cpus[count++] = cpu; } + } + } + if (count > 0) return count; + + char buf[256]; + if (read_file_buf("/sys/devices/system/cpu/online", buf, sizeof(buf)) < 0) return 0; + return parse_cpu_list(buf, cpus, max_cpus); +} + +static int get_physical_cores(const int* allowed_cpus, int allowed_count) +{ + if (allowed_count <= 0) return 0; + + char visited[CPU_SETSIZE]; + std::memset(visited, 0, sizeof(visited)); + int cores = 0; + + for (int i = 0; i < allowed_count; ++i) { + const int cpu = allowed_cpus[i]; + if (cpu < 0 || cpu >= CPU_SETSIZE || visited[cpu]) continue; + + char path[128]; + char buf[256]; + snprintf(path, + sizeof(path), + "/sys/devices/system/cpu/cpu%d/topology/core_cpus_list", + cpu); + ssize_t n = read_file_buf(path, buf, sizeof(buf)); + if (n < 0) { + snprintf(path, + sizeof(path), + "/sys/devices/system/cpu/cpu%d/topology/thread_siblings_list", + cpu); + n = read_file_buf(path, buf, sizeof(buf)); } - return 1; + + if (n >= 0) { + mark_cpus_from_list(buf, visited); + } + visited[cpu] = 1; + ++cores; } - return cores.size(); + + return cores > 0 ? cores : allowed_count; } -static std::string get_cpu_model_from_proc() +static bool copy_stripped(char* dst, size_t dst_size, const char* src) { - std::ifstream cpuinfo("/proc/cpuinfo"); - if (!cpuinfo.is_open()) return ""; - - std::string line; - while (std::getline(cpuinfo, line)) { - std::size_t pos = line.find("model name"); - if (pos == std::string::npos) pos = line.find("Processor"); - if (pos != std::string::npos) { - std::size_t colon = line.find(':', pos); - if (colon != std::string::npos) return line.substr(colon + 2); // Skip ": " + if (dst_size == 0) return false; + size_t len = std::strlen(src); + while (len > 0 && (src[len - 1] == '\n' || src[len - 1] == '\r' || src[len - 1] == ' ')) { + --len; + } + if (len >= dst_size) len = dst_size - 1; + std::memcpy(dst, src, len); + dst[len] = '\0'; + return len > 0; +} + +static bool get_cpu_model_from_proc(char* out, size_t out_size) +{ + FILE* cpuinfo = fopen("/proc/cpuinfo", "r"); + if (cpuinfo == nullptr) return false; + + char line[512]; + while (fgets(line, sizeof(line), cpuinfo) != nullptr) { + const char* field = std::strstr(line, "model name"); + if (field == nullptr) field = std::strstr(line, "Processor"); + if (field == nullptr) continue; + + const char* colon = std::strchr(field, ':'); + if (colon == nullptr) continue; + ++colon; + while (*colon == ' ' || *colon == '\t') { + ++colon; } + const bool ok = copy_stripped(out, out_size, colon); + fclose(cpuinfo); + return ok; } - return ""; + fclose(cpuinfo); + return false; } -// From https://gcc.gnu.org/onlinedocs/gcc/x86-Built-in-Functions.html -// Also supported by clang -static std::string get_cpu_model_builtin() +static void get_cpu_model(char* out, size_t out_size) { -#if (defined(__x86_64__) || defined(__i386__)) && (defined(__GNUC__) || defined(__clang__)) - __builtin_cpu_init(); - return __builtin_cpu_is("amd") ? "AMD CPU" - : __builtin_cpu_is("intel") ? "Intel CPU" - : __builtin_cpu_is("atom") ? "Intel Atom CPU" - : __builtin_cpu_is("slm") ? "Intel Silvermont CPU" - : __builtin_cpu_is("core2") ? "Intel Core 2 CPU" - : __builtin_cpu_is("corei7") ? "Intel Core i7 CPU" - : __builtin_cpu_is("nehalem") ? "Intel Core i7 Nehalem CPU" - : __builtin_cpu_is("westmere") ? "Intel Core i7 Westmere CPU" - : __builtin_cpu_is("sandybridge") ? "Intel Core i7 Sandy Bridge CPU" - : __builtin_cpu_is("ivybridge") ? "Intel Core i7 Ivy Bridge CPU" - : __builtin_cpu_is("haswell") ? "Intel Core i7 Haswell CPU" - : __builtin_cpu_is("broadwell") ? "Intel Core i7 Broadwell CPU" - : __builtin_cpu_is("skylake") ? "Intel Core i7 Skylake CPU" - : __builtin_cpu_is("skylake-avx512") ? "Intel Core i7 Skylake AVX512 CPU" - : __builtin_cpu_is("cannonlake") ? "Intel Core i7 Cannon Lake CPU" - : __builtin_cpu_is("icelake-client") ? "Intel Core i7 Ice Lake Client CPU" - : __builtin_cpu_is("icelake-server") ? "Intel Core i7 Ice Lake Server CPU" - : __builtin_cpu_is("cascadelake") ? "Intel Core i7 Cascadelake CPU" - : __builtin_cpu_is("tigerlake") ? "Intel Core i7 Tigerlake CPU" - : __builtin_cpu_is("cooperlake") ? "Intel Core i7 Cooperlake CPU" - : __builtin_cpu_is("sapphirerapids") ? "Intel Core i7 sapphirerapids CPU" - : __builtin_cpu_is("alderlake") ? "Intel Core i7 Alderlake CPU" - : __builtin_cpu_is("rocketlake") ? "Intel Core i7 Rocketlake CPU" - : __builtin_cpu_is("graniterapids") ? "Intel Core i7 graniterapids CPU" - : __builtin_cpu_is("graniterapids-d") ? "Intel Core i7 graniterapids D CPU" - : __builtin_cpu_is("bonnell") ? "Intel Atom Bonnell CPU" - : __builtin_cpu_is("silvermont") ? "Intel Atom Silvermont CPU" - : __builtin_cpu_is("goldmont") ? "Intel Atom Goldmont CPU" - : __builtin_cpu_is("goldmont-plus") ? "Intel Atom Goldmont Plus CPU" - : __builtin_cpu_is("tremont") ? "Intel Atom Tremont CPU" - : __builtin_cpu_is("sierraforest") ? "Intel Atom Sierra Forest CPU" - : __builtin_cpu_is("grandridge") ? "Intel Atom Grand Ridge CPU" - : __builtin_cpu_is("amdfam10h") ? "AMD Family 10h CPU" - : __builtin_cpu_is("barcelona") ? "AMD Family 10h Barcelona CPU" - : __builtin_cpu_is("shanghai") ? "AMD Family 10h Shanghai CPU" - : __builtin_cpu_is("istanbul") ? "AMD Family 10h Istanbul CPU" - : __builtin_cpu_is("btver1") ? "AMD Family 14h CPU" - : __builtin_cpu_is("amdfam15h") ? "AMD Family 15h CPU" - : __builtin_cpu_is("bdver1") ? "AMD Family 15h Bulldozer version 1" - : __builtin_cpu_is("bdver2") ? "AMD Family 15h Bulldozer version 2" - : __builtin_cpu_is("bdver3") ? "AMD Family 15h Bulldozer version 3" - : __builtin_cpu_is("bdver4") ? "AMD Family 15h Bulldozer version 4" - : __builtin_cpu_is("btver2") ? "AMD Family 16h CPU" - : __builtin_cpu_is("amdfam17h") ? "AMD Family 17h CPU" - : __builtin_cpu_is("znver1") ? "AMD Family 17h Zen version 1" - : __builtin_cpu_is("znver2") ? "AMD Family 17h Zen version 2" - : __builtin_cpu_is("amdfam19h") ? "AMD Family 19h CPU" - : "Unknown"; -#else - return "Unknown"; -#endif + if (get_cpu_model_from_proc(out, out_size)) return; + + char buf[256]; + if (read_file_buf("/sys/firmware/devicetree/base/model", buf, sizeof(buf)) >= 0 || + read_file_buf("/proc/device-tree/model", buf, sizeof(buf)) >= 0) { + if (copy_stripped(out, out_size, buf)) return; + } + if (read_file_buf("/sys/devices/virtual/dmi/id/product_name", buf, sizeof(buf)) >= 0) { + if (copy_stripped(out, out_size, buf)) return; + } + std::snprintf(out, out_size, "Unknown"); } -static std::string get_cpu_model() +static const char* get_simd_target() { - if (auto model_from_proc = get_cpu_model_from_proc(); !model_from_proc.empty()) { - return model_from_proc; - } else if (auto model_from_builtin = get_cpu_model_builtin(); !model_from_builtin.empty()) { - return model_from_builtin; + const int64_t target = hwy::DispatchedTarget(); + switch (target) { + case HWY_AVX3: + case HWY_AVX3_DL: + case HWY_AVX3_ZEN4: + case HWY_AVX3_SPR: + case HWY_AVX10_2: return "AVX-512"; + default: return hwy::TargetName(target); } - return "Unknown"; } struct host_memory_info_t { @@ -150,26 +239,28 @@ struct host_memory_info_t { static host_memory_info_t get_host_memory_info() { - std::ifstream meminfo("/proc/meminfo"); - if (!meminfo.is_open()) return {}; + FILE* meminfo = fopen("/proc/meminfo", "r"); + if (meminfo == nullptr) return {}; - std::string line; + char line[256]; long total_kb = 0; long available_kb = 0; long free_kb = 0; - while (std::getline(meminfo, line)) { - std::istringstream fields(line); - std::string key; + int found = 0; + while (found < 3 && fgets(line, sizeof(line), meminfo) != nullptr) { long value_kb = 0; - fields >> key >> value_kb; - if (key == "MemTotal:") { + if (std::sscanf(line, "MemTotal: %ld", &value_kb) == 1) { total_kb = value_kb; - } else if (key == "MemAvailable:") { + ++found; + } else if (std::sscanf(line, "MemAvailable: %ld", &value_kb) == 1) { available_kb = value_kb; - } else if (key == "MemFree:") { + ++found; + } else if (std::sscanf(line, "MemFree: %ld", &value_kb) == 1) { free_kb = value_kb; + ++found; } } + fclose(meminfo); if (available_kb == 0) { available_kb = free_kb; } constexpr double kb_per_gib = 1024.0 * 1024.0; @@ -193,14 +284,19 @@ void print_version_info(int num_devices) CUOPT_GIT_COMMIT_HASH, CUOPT_CPU_ARCHITECTURE, CUOPT_CUDA_ARCHITECTURES); + const auto memory = get_host_memory_info(); - CUOPT_LOG_INFO( - "CPU: %s, threads (physical/logical): %d/%d, RAM (available/total): %.2f / %.2f GiB", - get_cpu_model().c_str(), - get_physical_cores(), - std::thread::hardware_concurrency(), - memory.available_gb, - memory.total_gb); + int allowed_cpus[CPU_SETSIZE]; + const int allowed_count = get_allowed_cpus(allowed_cpus, CPU_SETSIZE); + char cpu_model[256]; + get_cpu_model(cpu_model, sizeof(cpu_model)); + CUOPT_LOG_INFO("CPU: %s, threads: %dC/%dT, RAM usage: %.2f/%.2fGiB", + cpu_model, + get_physical_cores(allowed_cpus, allowed_count), + allowed_count, + std::max(0.0, memory.total_gb - memory.available_gb), + memory.total_gb); + CUOPT_LOG_INFO("CPU SIMD target: %s", get_simd_target()); for (int device_id = 0; device_id < num_devices; ++device_id) { cudaDeviceProp device_prop{}; diff --git a/skills/cuopt-developer/references/conventions.md b/skills/cuopt-developer/references/conventions.md index 1bef2bbe3a..74fac403bf 100644 --- a/skills/cuopt-developer/references/conventions.md +++ b/skills/cuopt-developer/references/conventions.md @@ -197,6 +197,24 @@ rmm::device_uvector data(100, stream); Read existing code in `cpp/src/` for real examples of RMM allocation, stream-ordering, RAFT utilities, and kernel launch patterns. +### Bypassing `ins_vector`: credit the bytes back to the wrapper + +The instrumented accessors record a load per element read, and the counter lives in the +wrapper while the data lives in the vector's buffer. The compiler cannot prove those do not +alias, so the counter round-trips through memory every iteration and serializes the loop. +That cost is measurable in the innermost scoring loops. + +Two instrumentation-free paths to the same buffers already exist: `data()` on the wrapper +returns the raw pointer without recording, and the spans published on `fj_cpu.view` alias the +same allocations. + +When you take either path, add the skipped bytes back into the wrapper you bypassed — +`byte_loads` and `byte_stores` are public `mutable size_t` on +`memory_instrumentation_base_t`, so one `+= n * sizeof(element)` above the loop replaces N +per-element records. Do not route them into a separate counter: the byte totals feed the +deterministic work-unit proxy, and crediting the wrapper keeps both `collect()` and +`collect_per_wrapper()` correct and leaves the work-unit calibration untouched. + ## Test Impact Check **Before any behavioral change, ask:** diff --git a/thirdparty/THIRD_PARTY_LICENSES b/thirdparty/THIRD_PARTY_LICENSES index 7424a65232..15d5a08e00 100644 --- a/thirdparty/THIRD_PARTY_LICENSES +++ b/thirdparty/THIRD_PARTY_LICENSES @@ -597,3 +597,217 @@ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +----------------------------------------------------------------------------------------- + +== highway Apache-2.0 + +Files: cpp/build/_deps/highway-src + +Copyright (c) The Highway Project Authors. All rights reserved. + +Highway is dual-licensed under the Apache License 2.0 or the BSD 3-Clause License; +cuOpt elects the Apache License 2.0, reproduced below. + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License.