Merge pull request #2098 from tweag/cb/fix-2097

Fix toolchain libraries creation when using `ghcWithPackages`

Author: mergify[bot]

.bazelrc.common

@@ -143,8 +143,8 @@ coverage --build_tag_filters "coverage-compatible" --test_tag_filters "coverage-
 
 # To update these lines, execute
 # `bazel run @contrib_rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples,examples/arm,examples/basic_modules,examples/cat_hs,examples/cat_hs/exec/cat_hs,examples/cat_hs/lib/args,examples/cat_hs/lib/cat,examples/primitive,examples/rts,examples/transformers,examples/vector,tests/c2hs/repo,tests/haskell_module/repl/haskell_module_repl_cross_library_deps_test/package-a,tests/haskell_module/repl/haskell_module_repl_cross_library_deps_test/package-b,tests/haskell_module/repl/haskell_module_repl_test,tests/library-external-workspace/repo,tests/recompilation/recompilation_workspace,tests/repl-targets/hs_bin_repl_test,tests/repl-targets/hs_lib_repl_test,tests/stack-snapshot-deps/hs_override_stack_test,tutorial,tutorial/lib,tutorial/main,tutorial/tools/build_rules
-query --deleted_packages=examples,examples/arm,examples/basic_modules,examples/cat_hs,examples/cat_hs/exec/cat_hs,examples/cat_hs/lib/args,examples/cat_hs/lib/cat,examples/primitive,examples/rts,examples/transformers,examples/vector,tests/c2hs/repo,tests/haskell_module/repl/haskell_module_repl_cross_library_deps_test/package-a,tests/haskell_module/repl/haskell_module_repl_cross_library_deps_test/package-b,tests/haskell_module/repl/haskell_module_repl_test,tests/library-external-workspace/repo,tests/recompilation/recompilation_workspace,tests/repl-targets/hs_bin_repl_test,tests/repl-targets/hs_lib_repl_test,tests/stack-snapshot-deps/hs_override_stack_test,tutorial,tutorial/lib,tutorial/main,tutorial/tools/build_rules
+build --deleted_packages=examples,examples/arm,examples/basic_modules,examples/cat_hs,examples/cat_hs/exec/cat_hs,examples/cat_hs/lib/args,examples/cat_hs/lib/cat,examples/primitive,examples/rts,examples/transformers,examples/vector,tests/c2hs/repo,tests/haskell_module/repl/haskell_module_repl_cross_library_deps_test/package-a,tests/haskell_module/repl/haskell_module_repl_cross_library_deps_test/package-b,tests/haskell_module/repl/haskell_module_repl_test,tests/library-external-workspace/repo,tests/recompilation/recompilation_workspace,tests/repl-targets/hs_bin_repl_test,tests/repl-targets/hs_lib_repl_test,tests/stack-snapshot-deps/hs_override_stack_test,tutorial,tutorial/lib,tutorial/main,tutorial/tools/build_rules,tests/ghcWithPackages_2097/test
+query --deleted_packages=examples,examples/arm,examples/basic_modules,examples/cat_hs,examples/cat_hs/exec/cat_hs,examples/cat_hs/lib/args,examples/cat_hs/lib/cat,examples/primitive,examples/rts,examples/transformers,examples/vector,tests/c2hs/repo,tests/haskell_module/repl/haskell_module_repl_cross_library_deps_test/package-a,tests/haskell_module/repl/haskell_module_repl_cross_library_deps_test/package-b,tests/haskell_module/repl/haskell_module_repl_test,tests/library-external-workspace/repo,tests/recompilation/recompilation_workspace,tests/repl-targets/hs_bin_repl_test,tests/repl-targets/hs_lib_repl_test,tests/stack-snapshot-deps/hs_override_stack_test,tutorial,tutorial/lib,tutorial/main,tutorial/tools/build_rules,tests/ghcWithPackages_2097/test
 
 # Needed to build @rules_jvm_external//private/tools/java/com/github/bazelbuild/rules_jvm_external/zip (used by //tests/java_classpath)
 build:macos-nixpkgs --tool_java_language_version=11

haskell/private/pkgdb_to_bzl.py

@@ -15,9 +15,13 @@
 import textwrap
 import types
 import json
+from pathlib import Path
 
 import package_configuration
 
+def match_glob(root_dir, pattern):
+    return sorted([p.relative_to(root_dir).as_posix() for p in Path(root_dir).glob(pattern)])
+
 if len(sys.argv) == 3:
     repo_dir = "external/" + sys.argv[1]
     topdir = sys.argv[2]
@@ -46,7 +50,9 @@ def resolve(path, pkgroot):
     else:
         return path
 
-def path_to_label(path, pkgroot):
+symlinks = {}
+
+def path_to_label(path, pkgroot, output=None):
     """Substitute one pkgroot for another relative one to obtain a label."""
     if path.find("${pkgroot}") != -1:
         # determine if the given path is inside the repository root
@@ -58,9 +64,21 @@ def path_to_label(path, pkgroot):
         return None if relative_path.startswith('..') else relative_path.replace('\\', '/')
 
     topdir_relative_path = path.replace(os.path.realpath(pkgroot), "$topdir")
-    if topdir_relative_path.find("$topdir") != -1:
+    if topdir_relative_path.startswith("$topdir"):
         return os.path.normpath(topdir_relative_path.replace("$topdir", topdir)).replace('\\', '/')
 
+    if not output is None:
+        if os.path.isabs(path) and os.path.exists(path):
+            global symlinks
+            lnk = symlinks.get(path)
+            if not lnk:
+                lnk = "lnk_{}".format(len(symlinks))
+                symlinks[path] = lnk
+                output.append("#SYMLINK: {} {}".format(path.replace('\\', '/'), lnk))
+            return lnk
+        else:
+            print("WARN: could not handle", path, file=sys.stderr)
+
 def hs_library_pattern(name, mode = "static", profiling = False):
     """Convert hs-libraries entry to glob patterns.
 
@@ -108,16 +126,15 @@ def hs_library_pattern(name, mode = "static", profiling = False):
 # Accumulate package id to package name mappings.
 pkg_id_map = []
 
-# pkgroot is not part of .conf files. It's a computed value. It is
-# defined to be the directory enclosing the package database
-# directory.
-pkgroot = os.path.dirname(package_conf_dir)
-
-
 for conf in glob.glob(os.path.join(package_conf_dir, '*.conf')):
     with open(conf, 'r') as f:
         pkg = package_configuration.parse_package_configuration(f)
 
+    # pkgroot is not part of .conf files. It's a computed value. It is
+    # defined to be the directory enclosing the package database
+    # directory.
+    pkgroot = os.path.dirname(os.path.dirname(conf))
+
     pkg_id_map.append((pkg.name, pkg.id))
 
     # Haddock handling
@@ -201,37 +218,36 @@ def hs_library_pattern(name, mode = "static", profiling = False):
                 name = pkg.name,
                 id = pkg.id,
                 version = pkg.version,
-                hdrs = "glob({}, allow_empty = True)".format([
-                    path_to_label("{}/**/*.h".format(include_dir), pkgroot)
+                hdrs = [
+                    "/".join([path_to_label(include_dir, pkgroot, output), header])
                     for include_dir in pkg.include_dirs
-                    if path_to_label(include_dir, pkgroot)
-                ]),
+                    for header in match_glob(resolve(include_dir, pkgroot), "**/*.h")
+                ],
                 includes = [
-                    "/".join([repo_dir, path_to_label(include_dir, pkgroot)])
+                    "/".join([repo_dir, path_to_label(include_dir, pkgroot, output)])
                     for include_dir in pkg.include_dirs
-                    if path_to_label(include_dir, pkgroot)
                 ],
-                static_libraries = "glob({}, allow_empty = True)".format([
-                    path_to_label("{}/{}".format(library_dir, pattern), pkgroot)
+                static_libraries = [
+                    "/".join([path_to_label(library_dir, pkgroot, output), library])
                     for hs_library in pkg.hs_libraries
                     for pattern in hs_library_pattern(hs_library, mode = "static", profiling = False)
                     for library_dir in pkg.library_dirs
-                    if path_to_label(library_dir, pkgroot)
-                ]),
-                static_profiling_libraries = "glob({}, allow_empty = True)".format([
-                    path_to_label("{}/{}".format(library_dir, pattern), pkgroot)
+                    for library in match_glob(resolve(library_dir, pkgroot), pattern)
+                ],
+                static_profiling_libraries = [
+                    "/".join([path_to_label(library_dir, pkgroot, output), library])
                     for hs_library in pkg.hs_libraries
                     for pattern in hs_library_pattern(hs_library, mode = "static", profiling = True)
                     for library_dir in pkg.library_dirs
-                    if path_to_label(library_dir, pkgroot)
-                ]),
-                shared_libraries = "glob({}, allow_empty = True)".format([
-                    path_to_label("{}/{}".format(dynamic_library_dir, pattern), pkgroot)
+                    for library in match_glob(resolve(library_dir, pkgroot), pattern)
+                ],
+                shared_libraries = [
+                    "/".join([path_to_label(dynamic_library_dir, pkgroot, output), library])
                     for hs_library in pkg.hs_libraries
                     for pattern in hs_library_pattern(hs_library, mode = "dynamic", profiling = False)
-                    for dynamic_library_dir in pkg.dynamic_library_dirs + pkg.library_dirs
-                    if path_to_label(dynamic_library_dir, pkgroot)
-                ]),
+                    for dynamic_library_dir in set(pkg.dynamic_library_dirs + pkg.library_dirs)
+                    for library in match_glob(resolve(dynamic_library_dir, pkgroot), pattern)
+                ],
                 haddock_html = repr(haddock_html),
                 haddock_interfaces = repr(haddock_interfaces),
                 deps = pkg.depends,

rules_haskell_tests/tests/ghcWithPackages_2097/BUILD.bazel

@@ -0,0 +1,14 @@
+load("//tests/integration_testing:rules_haskell_integration_test.bzl", "rules_haskell_integration_test")
+
+rules_haskell_integration_test(
+    name = "ghc_with_packages_test",
+    srcs = ["Test.hs"],
+    workspace_path = "test",
+)
+
+filegroup(
+    name = "all_files",
+    testonly = True,
+    srcs = glob(["**"]),
+    visibility = ["//visibility:public"],
+)

rules_haskell_tests/tests/ghcWithPackages_2097/Test.hs

@@ -0,0 +1,11 @@
+{-# OPTIONS -Wall #-}
+
+import Test.Hspec (hspec, it)
+import IntegrationTesting
+
+main :: IO ()
+main =  hspec $ do
+  it "bazel run ghcWithPackages" $ do
+    bazel <- setupTestBazel
+    assertSuccess (bazel ["run", "//:add-one"])
+    -- assertSuccess (bazel ["cquery", "--output", "build", "@rules_haskell_ghc_nixpkgs_haskell_toolchain//:rts"])

rules_haskell_tests/tests/ghcWithPackages_2097/test/BUILD.bazel

@@ -0,0 +1,35 @@
+load("@rules_cc//cc:defs.bzl", "cc_test")
+load(
+    "@rules_haskell//haskell:defs.bzl",
+    "haskell_library",
+    "haskell_toolchain_library",
+)
+
+haskell_toolchain_library(name = "base")
+
+haskell_toolchain_library(name = "rts")
+
+haskell_library(
+    name = "add-one-hs",
+    srcs = ["One.hs"],
+    deps = [":base"],
+)
+
+cc_test(
+    name = "add-one",
+    srcs = [
+        "main.c",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":add-one-hs",
+        ":rts",
+    ],
+)
+
+filegroup(
+    name = "all_files",
+    testonly = True,
+    srcs = glob(["**"]),
+    visibility = ["//visibility:public"],
+)

rules_haskell_tests/tests/ghcWithPackages_2097/test/One.hs

@@ -0,0 +1,6 @@
+module One () where
+
+add_one_hs :: Int -> Int
+add_one_hs x = x + 1
+
+foreign export ccall add_one_hs :: Int -> Int

rules_haskell_tests/tests/ghcWithPackages_2097/test/WORKSPACE

@@ -0,0 +1,36 @@
+local_repository(
+    name = "rules_haskell",
+    path = "%RULES_HASKELL_PATH%",
+)
+
+load("@rules_haskell//haskell:repositories.bzl", "rules_haskell_dependencies")
+
+rules_haskell_dependencies()
+
+load("@rules_haskell//haskell:nixpkgs.bzl", "haskell_register_ghc_nixpkgs")
+
+haskell_register_ghc_nixpkgs(
+    attribute_path = "",
+    nix_file_content = "with import <nixpkgs> {}; haskell.packages.ghc948.ghcWithPackages (p:[p.async])",
+    repository = "@rules_haskell//nixpkgs:default.nix",
+    version = "9.4.8",
+)
+
+load("@rules_haskell//haskell:toolchain.bzl", "rules_haskell_toolchains")
+
+rules_haskell_toolchains(version = "9.4.8")
+
+load(
+    "@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl",
+    "nixpkgs_cc_configure",
+    "nixpkgs_python_configure",
+)
+
+nixpkgs_cc_configure(
+    name = "nixpkgs_config_cc",
+    repository = "@rules_haskell//nixpkgs:default.nix",
+)
+
+nixpkgs_python_configure(
+    repository = "@rules_haskell//nixpkgs:default.nix",
+)

rules_haskell_tests/tests/ghcWithPackages_2097/test/main.c

@@ -0,0 +1,11 @@
+#include <stdio.h>
+#include "HsFFI.h"
+
+extern HsInt add_one_hs(HsInt a0);
+
+int main(int argc, char *argv[]) {
+  hs_init(&argc, &argv);
+  printf("Adding one to 5 through Haskell is %lld\n", add_one_hs(5));
+  hs_exit();
+  return 0;
+}