From d23c7ea5ae254194bc845728b7b445d2e65b8666 Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Fri, 10 Nov 2023 12:56:51 +0100 Subject: [PATCH 01/26] learning how to build a project --- Makefile | 15 +++++++++++++++ docker/Containerfile | 0 docker/Makefile | 14 +++++++++++--- linux/Dockerfile | 2 +- 4 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 Makefile create mode 100644 docker/Containerfile diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..9c1bd131 --- /dev/null +++ b/Makefile @@ -0,0 +1,15 @@ +.PHONY: docker-image +docker-image: + DOCKER_BUILDKIT=1 docker build -f Dockerfile -t opentracing-contrib/nginx-opentracing --target final . + +.PHONY: docker-image-alpine +docker-image-alpine: + DOCKER_BUILDKIT=1 docker build -f Dockerfile -t opentracing-contrib/nginx-opentracing --target final --build-arg BUILD_OS=alpine . + +.PHONY: test +test: + ./ci/system_testing.sh + +.PHONY: clean +clean: + rm -fr test-log diff --git a/docker/Containerfile b/docker/Containerfile new file mode 100644 index 00000000..e69de29b diff --git a/docker/Makefile b/docker/Makefile index 5a1bb7a7..ce9c7927 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -33,9 +33,9 @@ ubuntu64: $(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-ubuntu-x86_64.tar.gz ubuntu64: $(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-ubuntu-x86_64-build.tar.gz .PHONY: alpine -alpine: ## Build alpine images -alpine: $(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-alpine.tar.gz -alpine: $(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-alpine-build.tar.gz +alpine: ## Build alpine x86_64 images +alpine: $(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-alpine-x86_64.tar.gz +alpine: $(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-alpine-x86_64-build.tar.gz $(BUILD_CONTEXT)/ubuntu64: ubuntu.Dockerfile $(BUILD_CONTEXT)/ubuntu64/crystal.tar.gz cp ubuntu.Dockerfile $@/Dockerfile @@ -55,6 +55,14 @@ $(BUILD_CONTEXT)/alpine: alpine.Dockerfile $(BUILD_CONTEXT)/alpine/crystal.tar.g $(OUTPUT_DIR): mkdir -p $(OUTPUT_DIR) +# .PHONY: docker-image +# docker-image: +# DOCKER_BUILDKIT=1 docker build -f Dockerfile -t opentracing-contrib/nginx-opentracing --target final . +# +# .PHONY: docker-image-alpine +# docker-image-alpine: +# DOCKER_BUILDKIT=1 docker build -f Dockerfile -t opentracing-contrib/nginx-opentracing --target final --build-arg BUILD_OS=alpine . + # Ubuntu x86_64 $(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-ubuntu-x86_64.tar.gz: $(BUILD_CONTEXT)/ubuntu64 $(OUTPUT_DIR) docker build -t $(DOCKER_TAG_UBUNTU) --target runtime $(BUILD_ARGS_UBUNTU64) diff --git a/linux/Dockerfile b/linux/Dockerfile index 7ae24e60..1d8c0fe2 100644 --- a/linux/Dockerfile +++ b/linux/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:11 AS debian +FROM --platform=$BUILDPLATFORM debian:11 AS debian RUN apt-get update \ && apt-get install -y build-essential libevent-dev libpcre2-dev automake libtool pkg-config git curl llvm-13 clang-13 \ From d9be203b351e2e62ff14e92545bde86d5133e9d3 Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Fri, 10 Nov 2023 13:08:10 +0100 Subject: [PATCH 02/26] Update ruby in omnibus --- .gitignore | 1 + Makefile | 14 ++++++++++---- omnibus/omnibus.rb | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index a33d539b..baf72a9e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ omnibus/bin omnibus/crystal-darwin-* omnibus/shards-darwin-* omnibus/vendor +omnibus/local docs/build/ diff --git a/Makefile b/Makefile index 9c1bd131..ab5fd253 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,8 @@ .PHONY: docker-image docker-image: + make -C linux DOCKER_BUILDKIT=1 docker build -f Dockerfile -t opentracing-contrib/nginx-opentracing --target final . -.PHONY: docker-image-alpine -docker-image-alpine: - DOCKER_BUILDKIT=1 docker build -f Dockerfile -t opentracing-contrib/nginx-opentracing --target final --build-arg BUILD_OS=alpine . - .PHONY: test test: ./ci/system_testing.sh @@ -13,3 +10,12 @@ test: .PHONY: clean clean: rm -fr test-log + +.PHONY: setup +omnibus_setup: + cd omnibus && bundle install + + +.PHONY: darwin +darwin: omnibus_setup + make -C darwin CRYSTAL_VERSION=1.10.1 PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ=https://github.com/crystal-lang/crystal/releases/download/1.10.1/crystal-1.10.1-1-darwin-universal.tar.gz diff --git a/omnibus/omnibus.rb b/omnibus/omnibus.rb index 47a9fad4..0531241f 100644 --- a/omnibus/omnibus.rb +++ b/omnibus/omnibus.rb @@ -17,7 +17,7 @@ # # Uncomment this line to change the default base directory to "local" # ------------------------------------------------------------------- -# base_dir './local' +base_dir './local' # # Alternatively you can tune the individual values # ------------------------------------------------ From 61d28c47411865e6851a9abd50149ecde7de3f1e Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Fri, 10 Nov 2023 18:50:00 +0100 Subject: [PATCH 03/26] wip --- Makefile | 2 +- omnibus/config/projects/crystal.rb | 2 +- omnibus/config/projects/llvm.rb | 2 +- omnibus/config/software/crystal.rb | 4 +++- omnibus/omnibus.rb | 4 +++- 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index ab5fd253..77199e9c 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ clean: .PHONY: setup omnibus_setup: - cd omnibus && bundle install + cd omnibus && bundle install --binstubs --path vendor/bundler .PHONY: darwin diff --git a/omnibus/config/projects/crystal.rb b/omnibus/config/projects/crystal.rb index a2b1e1eb..7b8eee04 100644 --- a/omnibus/config/projects/crystal.rb +++ b/omnibus/config/projects/crystal.rb @@ -2,7 +2,7 @@ maintainer 'Juan Wajnerman' homepage 'http://crystal-lang.org/' -install_dir '/opt/crystal' +install_dir '/Users/miry/tmp/crystal' build_version do source :version, from_dependency: 'crystal' end diff --git a/omnibus/config/projects/llvm.rb b/omnibus/config/projects/llvm.rb index e0c0032b..9e07faf0 100644 --- a/omnibus/config/projects/llvm.rb +++ b/omnibus/config/projects/llvm.rb @@ -2,7 +2,7 @@ maintainer 'Juan Wajnerman' homepage 'http://llvm.org/' -install_dir '/opt/llvm' +install_dir '/Users/miry/tmp/llvm' build_version do source :version, from_dependency: 'llvm' end diff --git a/omnibus/config/software/crystal.rb b/omnibus/config/software/crystal.rb index 4a31e3cb..d1b16537 100644 --- a/omnibus/config/software/crystal.rb +++ b/omnibus/config/software/crystal.rb @@ -39,8 +39,10 @@ env["PATH"] = "#{llvm_bin.project_dir}/bin:#{project_dir}/deps:#{env["PATH"]}" end +# raise Omnibus::Config.cache_dir.inspect + if macos? || mac_os_x? - env["CRYSTAL_PATH"] = "lib:/private/var/cache/omnibus/src/crystal/src" + env["CRYSTAL_PATH"] = "lib:#{Omnibus::Config.cache_dir}:/private/var/cache/omnibus/src/crystal/src" else env["CRYSTAL_PATH"] = "lib:#{project_dir}/src" end diff --git a/omnibus/omnibus.rb b/omnibus/omnibus.rb index 0531241f..35a75449 100644 --- a/omnibus/omnibus.rb +++ b/omnibus/omnibus.rb @@ -22,7 +22,7 @@ # Alternatively you can tune the individual values # ------------------------------------------------ # cache_dir './local/omnibus/cache' -# git_cache_dir './local/omnibus/cache/install_path' +# git_cache_dir './local/omnibus/cache/' # source_dir './local/omnibus/src' # build_dir './local/omnibus/build' # package_dir './local/omnibus/pkg' @@ -50,3 +50,5 @@ # local_software_dirs ['/path/to/local/software'] append_timestamp false + +use_git_caching true From ed2a31e88f56410d34f43c7a1c346d6b7eb0a422 Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Fri, 10 Nov 2023 22:13:34 +0100 Subject: [PATCH 04/26] Specify explicitly target for Intel Darwin build ``` Environment: CFLAGS="-I/Users/miry/tmp/crystal/embedded/include -O3 -D_FORTIFY_SOURCE=2 -fstack-protector -fPIC -arch arm64 -arch x86_64 -target arm64-apple-darwin" CPPFLAGS="-I/Users/miry/tmp/crystal/embedded/include -O3 -D_FORTIFY_SOURCE=2 -fstack-protector -fPIC " CRYSTAL_LIBRARY_PATH="/Users/miry/tmp/crystal/embedded/lib" CRYSTAL_PATH="lib:/private/var/cache/omnibus/src/crystal/src:/Users/miry/src/crystal/distribution-scripts/omnibus/local/src/crystal/src" CXXFLAGS="-I/Users/miry/tmp/crystal/embedded/include -O3 -D_FORTIFY_SOURCE=2 -fstack-protector -fPIC -arch arm64 -arch x86_64 -target arm64-apple-darwin" LDFLAGS="-Wl,-rpath,/Users/miry/tmp/crystal/embedded/lib -L/Users/miry/tmp/crystal/embedded/lib" LD_RUN_PATH="/Users/miry/tmp/crystal/embedded/lib" LIBRARY_PATH="/Users/miry/tmp/crystal/embedded/lib:/opt/homebrew/lib" OMNIBUS_INSTALL_DIR="/Users/miry/tmp/crystal" $ llvm-config --host-target arm64-apple-darwin23.0.0 $ make crystal stats=true release=true FLAGS="--no-debug" CRYSTAL_CONFIG_LIBRARY_PATH= O=/Users/miry/tmp/crystal/embedded/bin Using /usr/local/bin/llvm-config [version= 15.0.7] CRYSTAL_CONFIG_BUILD_COMMIT="1f592eca6" CRYSTAL_CONFIG_PATH='$ORIGIN/../share/crystal/src' SOURCE_DATE_EPOCH="1699639716" CRYSTAL_CONFIG_LIBRARY_PATH= ./bin/crystal build --no-debug -D strict_multi_assign -D preview_overload_order --release --stats -Dwithout_interpreter -o /Users/miry/tmp/crystal/embedded/bin/crystal src/compiler/crystal.cr -D without_openssl -D without_zlib -D use_pcre2 $ file /Users/miry/tmp/crystal/embedded/bin/crystal /Users/miry/tmp/crystal/embedded/bin/crystal: Mach-O 64-bit executable arm64 ``` The target builds the same as for arm target, because it uses default target for host. --- Makefile | 1 + omnibus/config/software/crystal.rb | 27 +++++++++++++-------------- omnibus/omnibus.rb | 2 -- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 77199e9c..cb914a57 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,7 @@ test: clean: rm -fr test-log +# https://github.com/chef/omnibus .PHONY: setup omnibus_setup: cd omnibus && bundle install --binstubs --path vendor/bundler diff --git a/omnibus/config/software/crystal.rb b/omnibus/config/software/crystal.rb index d1b16537..a27b88eb 100644 --- a/omnibus/config/software/crystal.rb +++ b/omnibus/config/software/crystal.rb @@ -1,18 +1,12 @@ CRYSTAL_VERSION = ENV['CRYSTAL_VERSION'] CRYSTAL_SHA1 = ENV['CRYSTAL_SHA1'] FIRST_RUN = ENV["FIRST_RUN"] -CRYSTAL_SRC = (ENV['CRYSTAL_SRC'] || "").strip +CRYSTAL_SRC = ENV.fetch('CRYSTAL_SRC', 'https://github.com/crystal-lang/crystal') name "crystal" default_version CRYSTAL_VERSION skip_transitive_dependency_licensing true -if CRYSTAL_SRC.empty? - source git: "https://github.com/crystal-lang/crystal" -else - source git: CRYSTAL_SRC -end - dependency "pcre2" dependency "bdw-gc" dependency "llvm_bin" unless FIRST_RUN @@ -20,7 +14,7 @@ dependency "libffi" env = with_standard_compiler_flags(with_embedded_path( - "LIBRARY_PATH" => "#{install_dir}/embedded/lib", + "LIBRARY_PATH" => "#{install_dir}/embedded/lib:/opt/homebrew/lib", "CRYSTAL_LIBRARY_PATH" => "#{install_dir}/embedded/lib", )) env["CFLAGS"] << " -fPIC -arch arm64 -arch x86_64" @@ -33,16 +27,16 @@ output_path = "#{install_dir}/embedded/bin" output_bin = "#{output_path}/crystal" -if FIRST_RUN - env["PATH"] = "#{project_dir}/deps:#{env["PATH"]}" -else - env["PATH"] = "#{llvm_bin.project_dir}/bin:#{project_dir}/deps:#{env["PATH"]}" +path_env = "#{project_dir}/deps:#{env["PATH"]}" +unless FIRST_RUN + path_env = "#{llvm_bin.project_dir}/bin:#{path_env}" end +env["PATH"] = path_env # raise Omnibus::Config.cache_dir.inspect if macos? || mac_os_x? - env["CRYSTAL_PATH"] = "lib:#{Omnibus::Config.cache_dir}:/private/var/cache/omnibus/src/crystal/src" + env["CRYSTAL_PATH"] = "lib:/private/var/cache/omnibus/src/crystal/src:/Users/miry/src/crystal/distribution-scripts/omnibus/local/src/crystal/src" else env["CRYSTAL_PATH"] = "lib:#{project_dir}/src" end @@ -60,8 +54,11 @@ copy "#{Dir.pwd}/crystal-#{ohai['os']}-#{ohai['kernel']['machine']}/embedded/bin/crystal", ".build/crystal" # Compile for Intel - command "make crystal stats=true release=true FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env + command "make crystal target=x86_64-apple-darwin stats=true release=true FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env move output_bin, "#{output_bin}_x86_64" + block {raise "Could not build crystal x86_64" unless File.exist?("#{output_bin}_x86_64")} + command "file #{output_bin}_x86_64", env: env + # Clean up make "clean_cache clean", env: env @@ -78,6 +75,8 @@ make "crystal stats=true release=true target=aarch64-apple-darwin FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_TARGET=aarch64-apple-darwin CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env command "clang #{output_path}/crystal.o -o #{output_bin}_arm64 -target arm64-apple-darwin src/llvm/ext/llvm_ext.o `llvm-config --libs --system-libs --ldflags 2>/dev/null` -lstdc++ -lpcre2-8 -lgc -lpthread -levent -liconv -ldl -v", env: env + block {raise "Could not build crystal arm64" unless File.exist?("#{output_bin}_arm64")} + command "file #{output_bin}_arm64", env: env delete "#{output_path}/crystal.o" # Lipo them up diff --git a/omnibus/omnibus.rb b/omnibus/omnibus.rb index 35a75449..685d767d 100644 --- a/omnibus/omnibus.rb +++ b/omnibus/omnibus.rb @@ -50,5 +50,3 @@ # local_software_dirs ['/path/to/local/software'] append_timestamp false - -use_git_caching true From 8daafd62638d6b32acec0d39efc8e74c6400f499 Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Fri, 10 Nov 2023 23:44:14 +0100 Subject: [PATCH 05/26] Update cxxflag for crystal --- omnibus/config/software/crystal.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/omnibus/config/software/crystal.rb b/omnibus/config/software/crystal.rb index a27b88eb..ddac5e69 100644 --- a/omnibus/config/software/crystal.rb +++ b/omnibus/config/software/crystal.rb @@ -44,6 +44,9 @@ build do command "git checkout #{CRYSTAL_SHA1}", cwd: project_dir + original_CXXFLAGS_env = env["CXXFLAGS"].dup + env["CXXFLAGS"] = original_CXXFLAGS_env + " -target x86_64-apple-darwin" + mkdir "#{project_dir}/deps" make "deps", env: env mkdir ".build" @@ -69,7 +72,7 @@ # Compile for ARM64. Apple's clang only understands arm64, LLVM uses aarch64, # so we need to sub out aarch64 in our calls to Apple tools - env["CXXFLAGS"] << " -target arm64-apple-darwin" + env["CXXFLAGS"] = original_CXXFLAGS_env + " -target arm64-apple-darwin" make "deps", env: env make "crystal stats=true release=true target=aarch64-apple-darwin FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_TARGET=aarch64-apple-darwin CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env From 276f2dc2773ac375926b3165cea39b6060d77687 Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Sat, 11 Nov 2023 22:16:15 +0100 Subject: [PATCH 06/26] Use darwin crystal universal --- Makefile | 3 +++ darwin/Makefile | 16 ++++++++-------- omnibus/config/software/crystal.rb | 18 +++++++++--------- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index cb914a57..560fed3d 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,9 @@ test: .PHONY: clean clean: rm -fr test-log + rm -fr omnibus/local + rm -fr omnibus/crystal-* + rm -fr ~/tmp # https://github.com/chef/omnibus .PHONY: setup diff --git a/darwin/Makefile b/darwin/Makefile index 584a93ef..0c6096f3 100644 --- a/darwin/Makefile +++ b/darwin/Makefile @@ -9,7 +9,7 @@ CRYSTAL_SHA1 ?= $(CRYSTAL_VERSION) ## Git tag/branch/sha1 to checkout and build PACKAGE_ITERATION ?= 1 FORCE_GIT_TAGGED ?= 1 ## Require build to be based on git tag/branch -PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ ?= ## url to crystal-{version}-{package}-darwin-x86_64.tar.gz +PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ ?= ## url to crystal-{version}-{package}-darwin-universal.tar.gz OUTPUT_DIR = build @@ -40,15 +40,15 @@ help: ## Show this help awk 'BEGIN {FS = "## "}; /^## [a-zA-Z_-]/ {printf " \033[36m%s\033[0m\n", $$2}; /^## / {printf " %s\n", $$2}' .PHONY: darwin-previous -darwin-previous: $(CURDIR)/../omnibus/crystal-darwin-x86_64/embedded/bin/crystal ## download previous crystal darwin release +darwin-previous: $(CURDIR)/../omnibus/crystal-darwin/embedded/bin/crystal ## download previous crystal darwin release # Once there are prior builds for arm64, this can use DARWIN_ARCH -$(CURDIR)/../omnibus/crystal-darwin-x86_64/embedded/bin/crystal: - curl -L -o /tmp/crystal-darwin-x86_64.tar.gz $(PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ) \ - && mkdir -p $(CURDIR)/../omnibus/crystal-darwin-x86_64 \ - && tar xfz /tmp/crystal-darwin-x86_64.tar.gz -C $(CURDIR)/../omnibus/crystal-darwin-x86_64 --strip-components=1 \ - && rm /tmp/crystal-darwin-x86_64.tar.gz \ - && chmod +x $(CURDIR)/../omnibus/crystal-darwin-x86_64/embedded/bin/crystal +$(CURDIR)/../omnibus/crystal-darwin/embedded/bin/crystal: + curl -L -o /tmp/crystal-darwin.tar.gz $(PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ) \ + && mkdir -p $(CURDIR)/../omnibus/crystal-darwin \ + && tar xfz /tmp/crystal-darwin.tar.gz -C $(CURDIR)/../omnibus/crystal-darwin --strip-components=1 \ + && rm /tmp/crystal-darwin.tar.gz \ + && chmod +x $(CURDIR)/../omnibus/crystal-darwin/embedded/bin/crystal $(OUTPUT_DIR)/$(DARWIN_NAME) $(OUTPUT_DIR)/$(DARWIN_PKG_NAME): ## Build omnibus crystal project ifeq ($(FORCE_GIT_TAGGED), 0) diff --git a/omnibus/config/software/crystal.rb b/omnibus/config/software/crystal.rb index ddac5e69..d0a0574d 100644 --- a/omnibus/config/software/crystal.rb +++ b/omnibus/config/software/crystal.rb @@ -44,22 +44,22 @@ build do command "git checkout #{CRYSTAL_SHA1}", cwd: project_dir - original_CXXFLAGS_env = env["CXXFLAGS"].dup - env["CXXFLAGS"] = original_CXXFLAGS_env + " -target x86_64-apple-darwin" - mkdir "#{project_dir}/deps" make "deps", env: env mkdir ".build" - command "echo #{Dir.pwd}", env: env - crflags = "--no-debug" + original_CXXFLAGS_env = env["CXXFLAGS"].dup + env["CXXFLAGS"] = original_CXXFLAGS_env + " -target x86_64-apple-darwin" + command "echo #{Dir.pwd}", env: env.dup + crflags = "--no-debug" copy "#{Dir.pwd}/crystal-#{ohai['os']}-#{ohai['kernel']['machine']}/embedded/bin/crystal", ".build/crystal" # Compile for Intel - command "make crystal target=x86_64-apple-darwin stats=true release=true FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env + env["CXXFLAGS"] = original_CXXFLAGS_env + " -target x86_64-apple-darwin" + command "make crystal target=x86_64-apple-darwin stats=true release=true FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env.dup move output_bin, "#{output_bin}_x86_64" - block {raise "Could not build crystal x86_64" unless File.exist?("#{output_bin}_x86_64")} + block { raise "Could not build crystal x86_64" unless File.exist?("#{output_bin}_x86_64") } command "file #{output_bin}_x86_64", env: env @@ -73,12 +73,12 @@ # Compile for ARM64. Apple's clang only understands arm64, LLVM uses aarch64, # so we need to sub out aarch64 in our calls to Apple tools env["CXXFLAGS"] = original_CXXFLAGS_env + " -target arm64-apple-darwin" - make "deps", env: env + make "deps", env: env.dup make "crystal stats=true release=true target=aarch64-apple-darwin FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_TARGET=aarch64-apple-darwin CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env command "clang #{output_path}/crystal.o -o #{output_bin}_arm64 -target arm64-apple-darwin src/llvm/ext/llvm_ext.o `llvm-config --libs --system-libs --ldflags 2>/dev/null` -lstdc++ -lpcre2-8 -lgc -lpthread -levent -liconv -ldl -v", env: env - block {raise "Could not build crystal arm64" unless File.exist?("#{output_bin}_arm64")} + block { raise "Could not build crystal arm64" unless File.exist?("#{output_bin}_arm64") } command "file #{output_bin}_arm64", env: env delete "#{output_path}/crystal.o" From 443c7d6b8502e279f649128d1cf6a7ed65d447af Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Sat, 11 Nov 2023 23:02:52 +0100 Subject: [PATCH 07/26] oops --- Makefile | 3 ++- darwin/Makefile | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 560fed3d..e646ae3b 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,7 @@ clean: rm -fr omnibus/local rm -fr omnibus/crystal-* rm -fr ~/tmp + make -C darwin clean # https://github.com/chef/omnibus .PHONY: setup @@ -22,4 +23,4 @@ omnibus_setup: .PHONY: darwin darwin: omnibus_setup - make -C darwin CRYSTAL_VERSION=1.10.1 PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ=https://github.com/crystal-lang/crystal/releases/download/1.10.1/crystal-1.10.1-1-darwin-universal.tar.gz + make -C darwin CRYSTAL_SRC=../crystal/.git CRYSTAL_VERSION=1.10.1 PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ=https://github.com/crystal-lang/crystal/releases/download/1.10.1/crystal-1.10.1-1-darwin-universal.tar.gz diff --git a/darwin/Makefile b/darwin/Makefile index 0c6096f3..4c6a8ed5 100644 --- a/darwin/Makefile +++ b/darwin/Makefile @@ -75,6 +75,6 @@ clean: ## Clean up build directory rm -Rf $(CURDIR)/tmp rm -Rf $(CURDIR)/../omnibus/pkg/crystal-* rm -Rf $(CURDIR)/../omnibus/pkg/version-* - rm -Rf $(CURDIR)/../omnibus/crystal-darwin-* + rm -Rf $(CURDIR)/../omnibus/crystal-darwin* rm -Rf /var/cache/omnibus/* rm -Rf /opt/crystal/* From aed0a033efa4402e4abd2f78f1c0a90bde3656d6 Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Sat, 11 Nov 2023 23:09:55 +0100 Subject: [PATCH 08/26] Ignore crystal embeded --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index baf72a9e..87474010 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ omnibus/pkg/ omnibus/.bundle omnibus/bin -omnibus/crystal-darwin-* +omnibus/crystal-darwin* omnibus/shards-darwin-* omnibus/vendor omnibus/local From e95b21236c68deb5b15e8022583b97e403fa1b78 Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Sat, 11 Nov 2023 23:33:20 +0100 Subject: [PATCH 09/26] llvm is universal --- omnibus/config/software/llvm_bin.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/omnibus/config/software/llvm_bin.rb b/omnibus/config/software/llvm_bin.rb index eaa62b1f..24078da1 100644 --- a/omnibus/config/software/llvm_bin.rb +++ b/omnibus/config/software/llvm_bin.rb @@ -18,7 +18,11 @@ raise "llvm_bin not supported" end -source url: "http://crystal-lang.s3.amazonaws.com/llvm/llvm-#{version}-#{ohai['os']}-#{ohai['kernel']['machine']}.tar.gz", +platform = ohai['os'] +arch = ohai['kernel']['machine'] +arch = "x86_64" + +source url: "http://crystal-lang.s3.amazonaws.com/llvm/llvm-#{version}-#{platform}-#{arch}.tar.gz", md5: source_md5 relative_path "llvm-#{version}" From 1dccf80a4765218cdf2e4c972eafbcf783f30a7a Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Sun, 12 Nov 2023 00:46:12 +0100 Subject: [PATCH 10/26] temp work --- omnibus/config/software/crystal.rb | 33 +++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/omnibus/config/software/crystal.rb b/omnibus/config/software/crystal.rb index d0a0574d..378e12dd 100644 --- a/omnibus/config/software/crystal.rb +++ b/omnibus/config/software/crystal.rb @@ -55,21 +55,34 @@ crflags = "--no-debug" copy "#{Dir.pwd}/crystal-#{ohai['os']}-#{ohai['kernel']['machine']}/embedded/bin/crystal", ".build/crystal" - # Compile for Intel - env["CXXFLAGS"] = original_CXXFLAGS_env + " -target x86_64-apple-darwin" - command "make crystal target=x86_64-apple-darwin stats=true release=true FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env.dup - move output_bin, "#{output_bin}_x86_64" - block { raise "Could not build crystal x86_64" unless File.exist?("#{output_bin}_x86_64") } - command "file #{output_bin}_x86_64", env: env - - - # Clean up - make "clean_cache clean", env: env +# # Compile for Intel +# env["CXXFLAGS"] = original_CXXFLAGS_env + " -target x86_64-apple-darwin" +# command "make crystal target=x86_64-apple-darwin stats=true release=true FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_TARGET=x86_64-apple-darwin CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env.dup +# move output_bin, "#{output_bin}_x86_64" +# # block { raise "Could not build crystal x86_64" unless File.exist?("#{output_bin}_x86_64") } +# command "file #{output_bin}_x86_64", env: env +# +# # Clean up +# make "clean_cache clean", env: env # Restore x86_64 compiler w/ cross-compile support mkdir ".build" copy "#{output_bin}_x86_64", ".build/crystal" + # Custom x86 + env["CXXFLAGS"] = original_CXXFLAGS_env + " -target x86_64-apple-darwin" + env["CC"] = "-v -target x86_64-apple-darwin" + make "deps", env: env.dup + + make "crystal stats=true release=true target=x86_64-apple-darwin FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_TARGET=x86_64-apple-darwin CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env + + command "clang #{output_path}/crystal.o -o #{output_bin}_x86_64 -target x86_64-apple-darwin src/llvm/ext/llvm_ext.o `llvm-config --libs --system-libs --ldflags 2>/dev/null` -lstdc++ -lpcre2-8 -lgc -lpthread -levent -liconv -ldl -v", env: env + block { raise "Could not build crystal x86_64" unless File.exist?("#{output_bin}_x86_64") } + command "file #{output_bin}_x86_64", env: env + delete "#{output_path}/crystal.o" + # Clean up + make "clean_cache clean", env: env + # Compile for ARM64. Apple's clang only understands arm64, LLVM uses aarch64, # so we need to sub out aarch64 in our calls to Apple tools env["CXXFLAGS"] = original_CXXFLAGS_env + " -target arm64-apple-darwin" From 806ad1421d1449e18b99b53ff60cffec75411020 Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Sun, 12 Nov 2023 12:59:00 +0100 Subject: [PATCH 11/26] Manage to build crystal universal --- Makefile | 5 ++- darwin/Makefile | 2 +- omnibus/config/software/crystal.rb | 71 +++++++++++++++++++----------- omnibus/omnibus.rb | 2 +- 4 files changed, 51 insertions(+), 29 deletions(-) diff --git a/Makefile b/Makefile index e646ae3b..9401a8a8 100644 --- a/Makefile +++ b/Makefile @@ -12,8 +12,9 @@ clean: rm -fr test-log rm -fr omnibus/local rm -fr omnibus/crystal-* - rm -fr ~/tmp make -C darwin clean + rm -fr ~/tmp + rm -fr ~/.cache/crystal # https://github.com/chef/omnibus .PHONY: setup @@ -23,4 +24,4 @@ omnibus_setup: .PHONY: darwin darwin: omnibus_setup - make -C darwin CRYSTAL_SRC=../crystal/.git CRYSTAL_VERSION=1.10.1 PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ=https://github.com/crystal-lang/crystal/releases/download/1.10.1/crystal-1.10.1-1-darwin-universal.tar.gz + CRYSTAL_SRC=/Users/miry/src/crystal/crystal/.git CRYSTAL_SHA1=master make -C darwin CRYSTAL_VERSION=1.10.1 PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ=https://github.com/crystal-lang/crystal/releases/download/1.10.1/crystal-1.10.1-1-darwin-universal.tar.gz diff --git a/darwin/Makefile b/darwin/Makefile index 4c6a8ed5..d7abc1a4 100644 --- a/darwin/Makefile +++ b/darwin/Makefile @@ -65,7 +65,7 @@ endif && export MACOSX_DEPLOYMENT_TARGET=11.0 \ && export SDKROOT=$(shell xcrun --sdk macosx --show-sdk-path) \ && bundle exec omnibus clean crystal shards \ - && bundle exec omnibus build crystal \ + && bundle exec omnibus build crystal --log-level debug \ && cp ./pkg/$(DARWIN_NAME) $(CURDIR)/$(OUTPUT_DIR)/$(subst x86_64,universal,$(DARWIN_NAME)) \ && cp ./pkg/$(DARWIN_PKG_NAME) $(CURDIR)/$(OUTPUT_DIR)/$(subst x86_64,universal,$(DARWIN_PKG_NAME)) diff --git a/omnibus/config/software/crystal.rb b/omnibus/config/software/crystal.rb index 378e12dd..f8dafe08 100644 --- a/omnibus/config/software/crystal.rb +++ b/omnibus/config/software/crystal.rb @@ -7,6 +7,8 @@ default_version CRYSTAL_VERSION skip_transitive_dependency_licensing true +source git: CRYSTAL_SRC + dependency "pcre2" dependency "bdw-gc" dependency "llvm_bin" unless FIRST_RUN @@ -14,7 +16,7 @@ dependency "libffi" env = with_standard_compiler_flags(with_embedded_path( - "LIBRARY_PATH" => "#{install_dir}/embedded/lib:/opt/homebrew/lib", + "LIBRARY_PATH" => "#{install_dir}/embedded/lib", "CRYSTAL_LIBRARY_PATH" => "#{install_dir}/embedded/lib", )) env["CFLAGS"] << " -fPIC -arch arm64 -arch x86_64" @@ -36,59 +38,77 @@ # raise Omnibus::Config.cache_dir.inspect if macos? || mac_os_x? - env["CRYSTAL_PATH"] = "lib:/private/var/cache/omnibus/src/crystal/src:/Users/miry/src/crystal/distribution-scripts/omnibus/local/src/crystal/src" + env["CRYSTAL_PATH"] = "lib:/private/var/cache/omnibus/src/crystal/src:#{project_dir}/src" else env["CRYSTAL_PATH"] = "lib:#{project_dir}/src" end build do - command "git checkout #{CRYSTAL_SHA1}", cwd: project_dir + block { puts "\n=== Starting build phase for crystal from #{Dir.pwd} ===\n\n" } + command "echo ' Sources are located in #{project_dir}'", env: env.dup + + command "git checkout '#{CRYSTAL_SHA1}'", cwd: project_dir + block { puts "\n=== Build crystal deps in #{project_dir}\n\n" } mkdir "#{project_dir}/deps" - make "deps", env: env + make "deps", env: env.dup mkdir ".build" - original_CXXFLAGS_env = env["CXXFLAGS"].dup - env["CXXFLAGS"] = original_CXXFLAGS_env + " -target x86_64-apple-darwin" - command "echo #{Dir.pwd}", env: env.dup + block { puts "\n=== Build native crystal bin with embedded universal crystal binary\n\n" } + copy "#{Dir.pwd}/crystal-#{ohai['os']}/embedded/bin/crystal", ".build/crystal" + command ".build/crystal --version", env: env.dup + command "file .build/crystal", env: env.dup + # Compile native crflags = "--no-debug" - copy "#{Dir.pwd}/crystal-#{ohai['os']}-#{ohai['kernel']['machine']}/embedded/bin/crystal", ".build/crystal" - -# # Compile for Intel -# env["CXXFLAGS"] = original_CXXFLAGS_env + " -target x86_64-apple-darwin" -# command "make crystal target=x86_64-apple-darwin stats=true release=true FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_TARGET=x86_64-apple-darwin CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env.dup -# move output_bin, "#{output_bin}_x86_64" -# # block { raise "Could not build crystal x86_64" unless File.exist?("#{output_bin}_x86_64") } -# command "file #{output_bin}_x86_64", env: env -# -# # Clean up -# make "clean_cache clean", env: env + command "make crystal stats=true release=true FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env.dup + block "Testing the result file" do + puts "===> Testing the result file #{output_bin}" + raise "Could not build native crystal: #{output_bin}" unless File.exist?("#{output_bin}") + end + command "file #{output_bin}", env: env.dup - # Restore x86_64 compiler w/ cross-compile support - mkdir ".build" - copy "#{output_bin}_x86_64", ".build/crystal" + block { puts "\n\n=== Restore compiler with cross-compile support ===\n\n" } + # Restore compiler w/ cross-compile support + move "#{output_bin}", ".build/crystal" + command ".build/crystal --version", env: env.dup + - # Custom x86 + block { puts "\n\n=== Building x86_64 version ===\n\n" } + + original_CXXFLAGS_env = env["CXXFLAGS"].dup + original_LDFLAGS_env = env["LDFLAGS"].dup + + # x86_64 build env["CXXFLAGS"] = original_CXXFLAGS_env + " -target x86_64-apple-darwin" - env["CC"] = "-v -target x86_64-apple-darwin" + env["LDFLAGS"] = original_LDFLAGS_env + " -v -target x86_64-apple-darwin" + env["LDLIBS"] = "-v -target x86_64-apple-darwin" make "deps", env: env.dup - make "crystal stats=true release=true target=x86_64-apple-darwin FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_TARGET=x86_64-apple-darwin CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env + make "crystal verbose=true stats=true release=true target=x86_64-apple-darwin FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_TARGET=x86_64-apple-darwin CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env command "clang #{output_path}/crystal.o -o #{output_bin}_x86_64 -target x86_64-apple-darwin src/llvm/ext/llvm_ext.o `llvm-config --libs --system-libs --ldflags 2>/dev/null` -lstdc++ -lpcre2-8 -lgc -lpthread -levent -liconv -ldl -v", env: env block { raise "Could not build crystal x86_64" unless File.exist?("#{output_bin}_x86_64") } command "file #{output_bin}_x86_64", env: env delete "#{output_path}/crystal.o" + + # Restore x86_64 compiler w/ cross-compile support + mkdir ".build" + copy "#{output_bin}_x86_64", ".build/crystal" + # Clean up make "clean_cache clean", env: env + + block { puts "\n\n=== Building arm64 version ===\n\n" } # Compile for ARM64. Apple's clang only understands arm64, LLVM uses aarch64, # so we need to sub out aarch64 in our calls to Apple tools env["CXXFLAGS"] = original_CXXFLAGS_env + " -target arm64-apple-darwin" + env["LDFLAGS"] = original_LDFLAGS_env + " -v -target arm64-apple-darwin" + env["LDLIBS"] = "-v -target x86_64-apple-darwin" make "deps", env: env.dup - make "crystal stats=true release=true target=aarch64-apple-darwin FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_TARGET=aarch64-apple-darwin CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env + make "crystal verbose=true stats=true release=true target=aarch64-apple-darwin FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_TARGET=aarch64-apple-darwin CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env command "clang #{output_path}/crystal.o -o #{output_bin}_arm64 -target arm64-apple-darwin src/llvm/ext/llvm_ext.o `llvm-config --libs --system-libs --ldflags 2>/dev/null` -lstdc++ -lpcre2-8 -lgc -lpthread -levent -liconv -ldl -v", env: env block { raise "Could not build crystal arm64" unless File.exist?("#{output_bin}_arm64") } @@ -96,6 +116,7 @@ delete "#{output_path}/crystal.o" # Lipo them up + block { puts "\n\n=== Combine x86_64 and arm64 binaries in single universal binary ===\n\n" } command "lipo -create -output #{output_bin} #{output_bin}_x86_64 #{output_bin}_arm64" delete "#{output_bin}_x86_64" delete "#{output_bin}_arm64" diff --git a/omnibus/omnibus.rb b/omnibus/omnibus.rb index 685d767d..9ed151c7 100644 --- a/omnibus/omnibus.rb +++ b/omnibus/omnibus.rb @@ -17,7 +17,7 @@ # # Uncomment this line to change the default base directory to "local" # ------------------------------------------------------------------- -base_dir './local' +base_dir '/Users/miry/tmp/local' # # Alternatively you can tune the individual values # ------------------------------------------------ From b71d9e98d829076ee338f8954c2ef8469da32e65 Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Sun, 12 Nov 2023 13:49:48 +0100 Subject: [PATCH 12/26] Remove local folder from ignore, as it is better to have test folder somewhere else, as there is git interactions --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 87474010..c4abf11c 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,6 @@ omnibus/bin omnibus/crystal-darwin* omnibus/shards-darwin-* omnibus/vendor -omnibus/local docs/build/ From 079f589f727859d2402018c64e3d7ae04e611445 Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Sun, 12 Nov 2023 15:39:49 +0100 Subject: [PATCH 13/26] Add a lot of verbosity --- .gitignore | 1 - omnibus/config/software/crystal.rb | 68 +++++++++++++++----------- omnibus/config/software/shards.rb | 48 +++++++++++++++--- omnibus/config/software/tgz_package.rb | 5 +- omnibus/omnibus.rb | 2 +- 5 files changed, 84 insertions(+), 40 deletions(-) diff --git a/.gitignore b/.gitignore index c4abf11c..e02e43c8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,6 @@ omnibus/pkg/ omnibus/.bundle omnibus/bin omnibus/crystal-darwin* -omnibus/shards-darwin-* omnibus/vendor docs/build/ diff --git a/omnibus/config/software/crystal.rb b/omnibus/config/software/crystal.rb index f8dafe08..bc9c8625 100644 --- a/omnibus/config/software/crystal.rb +++ b/omnibus/config/software/crystal.rb @@ -28,6 +28,8 @@ output_path = "#{install_dir}/embedded/bin" output_bin = "#{output_path}/crystal" +output_bin_x86_64 = "#{output_path}/crystal_x86_64" +output_bin_arm = "#{output_path}/crystal_arm" path_env = "#{project_dir}/deps:#{env["PATH"]}" unless FIRST_RUN @@ -44,17 +46,16 @@ end build do - block { puts "\n=== Starting build phase for crystal from #{Dir.pwd} ===\n\n" } + block { puts "\n===> Starting build phase for crystal from #{Dir.pwd} ===\n\n" } command "echo ' Sources are located in #{project_dir}'", env: env.dup command "git checkout '#{CRYSTAL_SHA1}'", cwd: project_dir - block { puts "\n=== Build crystal deps in #{project_dir}\n\n" } + block { puts "\n===== 1. Build native crystal bin with embedded universal crystal binary\n\n" } mkdir "#{project_dir}/deps" make "deps", env: env.dup mkdir ".build" - block { puts "\n=== Build native crystal bin with embedded universal crystal binary\n\n" } copy "#{Dir.pwd}/crystal-#{ohai['os']}/embedded/bin/crystal", ".build/crystal" command ".build/crystal --version", env: env.dup command "file .build/crystal", env: env.dup @@ -63,67 +64,74 @@ crflags = "--no-debug" command "make crystal stats=true release=true FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env.dup block "Testing the result file" do - puts "===> Testing the result file #{output_bin}" + puts "===== >>> Testing the result file #{output_bin}" raise "Could not build native crystal: #{output_bin}" unless File.exist?("#{output_bin}") end + # TODO: Add validation of command output command "file #{output_bin}", env: env.dup - block { puts "\n\n=== Restore compiler with cross-compile support ===\n\n" } + block { puts "\n===== 2. Restore compiler with cross-compile support\n\n" } # Restore compiler w/ cross-compile support move "#{output_bin}", ".build/crystal" command ".build/crystal --version", env: env.dup - - block { puts "\n\n=== Building x86_64 version ===\n\n" } + block { puts "\n===== 3. Building crystal x86_64 version\n\n" } original_CXXFLAGS_env = env["CXXFLAGS"].dup original_LDFLAGS_env = env["LDFLAGS"].dup - # x86_64 build + # Build for x86_64 env["CXXFLAGS"] = original_CXXFLAGS_env + " -target x86_64-apple-darwin" env["LDFLAGS"] = original_LDFLAGS_env + " -v -target x86_64-apple-darwin" env["LDLIBS"] = "-v -target x86_64-apple-darwin" make "deps", env: env.dup make "crystal verbose=true stats=true release=true target=x86_64-apple-darwin FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_TARGET=x86_64-apple-darwin CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env - - command "clang #{output_path}/crystal.o -o #{output_bin}_x86_64 -target x86_64-apple-darwin src/llvm/ext/llvm_ext.o `llvm-config --libs --system-libs --ldflags 2>/dev/null` -lstdc++ -lpcre2-8 -lgc -lpthread -levent -liconv -ldl -v", env: env - block { raise "Could not build crystal x86_64" unless File.exist?("#{output_bin}_x86_64") } - command "file #{output_bin}_x86_64", env: env - delete "#{output_path}/crystal.o" - - # Restore x86_64 compiler w/ cross-compile support - mkdir ".build" - copy "#{output_bin}_x86_64", ".build/crystal" + command "clang #{output_path}/crystal.o -o #{output_bin_x86_64} -target x86_64-apple-darwin src/llvm/ext/llvm_ext.o `llvm-config --libs --system-libs --ldflags 2>/dev/null` -lstdc++ -lpcre2-8 -lgc -lpthread -levent -liconv -ldl -v", env: env + + # Assertion + block { raise "Could not build #{output_bin_x86_64}" unless File.exist?(output_bin_x86_64) } + command "file #{output_bin_x86_64}", env: env # Clean up + delete "#{output_path}/crystal.o" make "clean_cache clean", env: env - block { puts "\n\n=== Building arm64 version ===\n\n" } - + # Build for arm64 + block { puts "\n===== 4. Building crystal arm64 version\n\n" } + # Compile for ARM64. Apple's clang only understands arm64, LLVM uses aarch64, # so we need to sub out aarch64 in our calls to Apple tools env["CXXFLAGS"] = original_CXXFLAGS_env + " -target arm64-apple-darwin" env["LDFLAGS"] = original_LDFLAGS_env + " -v -target arm64-apple-darwin" env["LDLIBS"] = "-v -target x86_64-apple-darwin" make "deps", env: env.dup - make "crystal verbose=true stats=true release=true target=aarch64-apple-darwin FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_TARGET=aarch64-apple-darwin CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env - - command "clang #{output_path}/crystal.o -o #{output_bin}_arm64 -target arm64-apple-darwin src/llvm/ext/llvm_ext.o `llvm-config --libs --system-libs --ldflags 2>/dev/null` -lstdc++ -lpcre2-8 -lgc -lpthread -levent -liconv -ldl -v", env: env - block { raise "Could not build crystal arm64" unless File.exist?("#{output_bin}_arm64") } - command "file #{output_bin}_arm64", env: env + command "clang #{output_path}/crystal.o -o #{output_bin_arm} -target arm64-apple-darwin src/llvm/ext/llvm_ext.o `llvm-config --libs --system-libs --ldflags 2>/dev/null` -lstdc++ -lpcre2-8 -lgc -lpthread -levent -liconv -ldl -v", env: env + + # Assertion + block { raise "Could not build #{output_bin_arm}" unless File.exist?(output_bin_arm) } + command "file #{output_bin_arm}", env: env + + # Clean up delete "#{output_path}/crystal.o" # Lipo them up - block { puts "\n\n=== Combine x86_64 and arm64 binaries in single universal binary ===\n\n" } - command "lipo -create -output #{output_bin} #{output_bin}_x86_64 #{output_bin}_arm64" - delete "#{output_bin}_x86_64" - delete "#{output_bin}_arm64" + block { puts "\n===== 5. Combine x86_64 and arm64 binaries in single crystal universal binary\n\n" } + command "lipo -create -output #{output_bin} #{output_bin_x86_64} #{output_bin_arm}" block do - raise "Could not build crystal" unless File.exist?(output_bin) + puts "===== >>> Testing the result file #{output_bin}" + raise "Could not build universal crystal #{output_bin}" unless File.exist?(output_bin) + end + # TODO: Add validation of command output + command "file #{output_bin}", env: env.dup + + # Clean up + delete output_bin_x86_64 + delete output_bin_arm + block do if macos? || mac_os_x? otool_libs = `otool -L #{output_bin}` if otool_libs.include?("/usr/local/lib") || otool_libs.include?('/opt/homebrew/lib') @@ -141,4 +149,6 @@ dest: "#{install_dir}/bin/crystal", mode: 0755, vars: { install_dir: install_dir } + + block { puts "\n===< Crystal successfully built\n\n" } end diff --git a/omnibus/config/software/shards.rb b/omnibus/config/software/shards.rb index 2b6448ad..77626945 100644 --- a/omnibus/config/software/shards.rb +++ b/omnibus/config/software/shards.rb @@ -82,22 +82,54 @@ env["CPPFLAGS"] = env["CPPFLAGS"].gsub("-arch arm64 -arch x86_64", "") build do + block { puts "\n=== Starting build phase for shards from #{Dir.pwd} ===\n\n" } + crflags = "--no-debug --release" - # Build for Intel - make "bin/shards SHARDS=false CRYSTAL=#{install_dir}/bin/crystal FLAGS='#{crflags}'", env: env - move "bin/shards", "bin/shards_x86_64" + # Build for x86_64 + block { puts "\n===== 1. Building shards x86_64 version\n\n" } + crflags_x86_64 = crflags + " --cross-compile --target x86_64-apple-darwin" + make "bin/shards SHARDS=false CRYSTAL=#{install_dir}/bin/crystal FLAGS='#{crflags_x86_64}'", env: env + + output_bin_x86_64 = 'bin/shards_x86_64' + target_x86_64 = 'x86_64-apple-darwin' + command "clang bin/shards.o -o #{output_bin_x86_64} -target #{target_x86_64} -L#{install_dir}/embedded/lib -lyaml -lpcre2-8 -lgc -lpthread -levent -liconv -ldl", env: env + block "Testing the result file" do + puts "===== >>> Testing the result file #{output_bin_x86_64}" + raise "Could not build #{output_bin_x86_64}" unless File.exist?("#{output_bin_x86_64}") + end + # TODO: Add a validation of the output to check archs + command "file #{output_bin_x86_64}", env: env.dup # Clean make "clean", env: env - # Build for ARM64 - crflags += " --cross-compile --target aarch64-apple-darwin" - make "bin/shards SHARDS=false CRYSTAL=#{install_dir}/bin/crystal FLAGS='#{crflags}'", env: env - command "clang bin/shards.o -o bin/shards_arm64 -target arm64-apple-darwin -L#{install_dir}/embedded/lib -lyaml -lpcre2-8 -lgc -lpthread -levent -liconv -ldl", env: env + # Build for arm64 + block { puts "\n===== 2. Building shards arm64 version\n\n" } + crflags_arm = crflags + " --cross-compile --target aarch64-apple-darwin" + make "bin/shards SHARDS=false CRYSTAL=#{install_dir}/bin/crystal FLAGS='#{crflags_arm}'", env: env + output_bin_arm64 = 'bin/shards_arm64' + target_arm64 = 'arm64-apple-darwin' + command "clang bin/shards.o -o #{output_bin_arm64} -target #{target_arm64} -L#{install_dir}/embedded/lib -lyaml -lpcre2-8 -lgc -lpthread -levent -liconv -ldl", env: env + block "Testing the result file" do + puts "===== >>> Testing the result file #{output_bin_arm64}" + raise "Could not build #{output_bin_arm64}" unless File.exist?("#{output_bin_arm64}") + end + # TODO: Add a validation of the output to check archs + command "file #{output_bin_arm64}", env: env.dup # Lipo them up - command "lipo -create -output bin/shards bin/shards_x86_64 bin/shards_arm64" + block { puts "\n===== 3. Combine x86_64 and arm64 binaries in a single universal binary\n\n" } + output_bin = 'bin/shards' + command "lipo -create -output #{output_bin} #{output_bin_x86_64} #{output_bin_arm64}" + block "Testing the result file" do + puts "===== >>> Testing the result file #{output_bin}" + raise "Could not build #{output_bin}: #{output_bin}" unless File.exist?("#{output_bin}") + end + # TODO: Add a validation of the output to check archs + command "file #{output_bin}", env: env.dup copy "bin/shards", "#{install_dir}/embedded/bin/shards" + + block { puts "\n===< Shards successfully built\n\n" } end diff --git a/omnibus/config/software/tgz_package.rb b/omnibus/config/software/tgz_package.rb index 200754f8..b550974c 100644 --- a/omnibus/config/software/tgz_package.rb +++ b/omnibus/config/software/tgz_package.rb @@ -7,7 +7,7 @@ destination = File.expand_path('pkg', Omnibus::Config.project_root) version = "#{project.build_version}-#{project.build_iteration}" version.gsub!("/", "-") - tgz_name = "#{project.name}-#{version}-#{ohai['os']}-#{ohai['kernel']['machine']}.tar.gz" + tgz_name = "#{project.name}-#{version}-#{ohai['os']}-universal.tar.gz" if macos? || mac_os_x? transform = "-s /./#{project.name}-#{version}/" else @@ -16,5 +16,8 @@ command "tar czf #{destination}/#{tgz_name} #{transform} -C #{install_dir} .", env: {"COPYFILE_DISABLE" => "1"} + + # NOTE: hack to bypass `git_caching` to modify anything in the working directory to create a commit + command "date > #{install_dir}/tgz_package.log" end end diff --git a/omnibus/omnibus.rb b/omnibus/omnibus.rb index 9ed151c7..dbb6cb1d 100644 --- a/omnibus/omnibus.rb +++ b/omnibus/omnibus.rb @@ -22,7 +22,7 @@ # Alternatively you can tune the individual values # ------------------------------------------------ # cache_dir './local/omnibus/cache' -# git_cache_dir './local/omnibus/cache/' +# git_cache_dir './local/omnibus/cache/{install_path}' # source_dir './local/omnibus/src' # build_dir './local/omnibus/build' # package_dir './local/omnibus/pkg' From 71e21c1cc5a6f9ace0f95cbd0cbef8e0857f5ae0 Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Sun, 12 Nov 2023 18:55:10 +0100 Subject: [PATCH 14/26] Make shards working and build result pkgs --- Makefile | 1 + omnibus/config/software/crystal.rb | 12 ++++++++---- omnibus/config/software/shards.rb | 27 ++++++++++++++------------ omnibus/config/software/tgz_package.rb | 2 +- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 9401a8a8..392ab3fd 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,7 @@ clean: make -C darwin clean rm -fr ~/tmp rm -fr ~/.cache/crystal + rm -fr ~/.cache/shards # https://github.com/chef/omnibus .PHONY: setup diff --git a/omnibus/config/software/crystal.rb b/omnibus/config/software/crystal.rb index bc9c8625..f5808872 100644 --- a/omnibus/config/software/crystal.rb +++ b/omnibus/config/software/crystal.rb @@ -75,6 +75,9 @@ move "#{output_bin}", ".build/crystal" command ".build/crystal --version", env: env.dup + # Clean up + make "clean_cache clean", env: env + block { puts "\n===== 3. Building crystal x86_64 version\n\n" } original_CXXFLAGS_env = env["CXXFLAGS"].dup @@ -88,7 +91,7 @@ make "crystal verbose=true stats=true release=true target=x86_64-apple-darwin FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_TARGET=x86_64-apple-darwin CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env command "clang #{output_path}/crystal.o -o #{output_bin_x86_64} -target x86_64-apple-darwin src/llvm/ext/llvm_ext.o `llvm-config --libs --system-libs --ldflags 2>/dev/null` -lstdc++ -lpcre2-8 -lgc -lpthread -levent -liconv -ldl -v", env: env - + # Assertion block { raise "Could not build #{output_bin_x86_64}" unless File.exist?(output_bin_x86_64) } command "file #{output_bin_x86_64}", env: env @@ -108,11 +111,11 @@ make "deps", env: env.dup make "crystal verbose=true stats=true release=true target=aarch64-apple-darwin FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_TARGET=aarch64-apple-darwin CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env command "clang #{output_path}/crystal.o -o #{output_bin_arm} -target arm64-apple-darwin src/llvm/ext/llvm_ext.o `llvm-config --libs --system-libs --ldflags 2>/dev/null` -lstdc++ -lpcre2-8 -lgc -lpthread -levent -liconv -ldl -v", env: env - + # Assertion block { raise "Could not build #{output_bin_arm}" unless File.exist?(output_bin_arm) } command "file #{output_bin_arm}", env: env - + # Clean up delete "#{output_path}/crystal.o" @@ -126,10 +129,11 @@ end # TODO: Add validation of command output command "file #{output_bin}", env: env.dup - + # Clean up delete output_bin_x86_64 delete output_bin_arm + make "clean_cache clean", env: env block do if macos? || mac_os_x? diff --git a/omnibus/config/software/shards.rb b/omnibus/config/software/shards.rb index 77626945..e1e17642 100644 --- a/omnibus/config/software/shards.rb +++ b/omnibus/config/software/shards.rb @@ -82,35 +82,39 @@ env["CPPFLAGS"] = env["CPPFLAGS"].gsub("-arch arm64 -arch x86_64", "") build do - block { puts "\n=== Starting build phase for shards from #{Dir.pwd} ===\n\n" } - - crflags = "--no-debug --release" + block { puts "\n=== Starting build phase for shards from #{project_dir} ===\n\n" } + + make "clean", env: env + command "#{install_dir}/bin/crystal clear_cache" # Build for x86_64 + crflags = "--no-debug --release" block { puts "\n===== 1. Building shards x86_64 version\n\n" } crflags_x86_64 = crflags + " --cross-compile --target x86_64-apple-darwin" make "bin/shards SHARDS=false CRYSTAL=#{install_dir}/bin/crystal FLAGS='#{crflags_x86_64}'", env: env - - output_bin_x86_64 = 'bin/shards_x86_64' + + output_bin = "#{project_dir}/bin/shards" + output_bin_x86_64 = "#{output_bin}_x86_64" target_x86_64 = 'x86_64-apple-darwin' - command "clang bin/shards.o -o #{output_bin_x86_64} -target #{target_x86_64} -L#{install_dir}/embedded/lib -lyaml -lpcre2-8 -lgc -lpthread -levent -liconv -ldl", env: env + command "clang bin/shards.o -o #{output_bin_x86_64} -v -target #{target_x86_64} -L#{install_dir}/embedded/lib -lyaml -lpcre2-8 -lgc -lpthread -levent -liconv -ldl", env: env block "Testing the result file" do - puts "===== >>> Testing the result file #{output_bin_x86_64}" - raise "Could not build #{output_bin_x86_64}" unless File.exist?("#{output_bin_x86_64}") + puts "===== >>> Testing the result file #{output_bin_x86_64} in #{project_dir}}" + raise "Could not build #{output_bin_x86_64}" unless File.exist?(output_bin_x86_64) end # TODO: Add a validation of the output to check archs command "file #{output_bin_x86_64}", env: env.dup # Clean make "clean", env: env + command "#{install_dir}/bin/crystal clear_cache" # Build for arm64 block { puts "\n===== 2. Building shards arm64 version\n\n" } crflags_arm = crflags + " --cross-compile --target aarch64-apple-darwin" make "bin/shards SHARDS=false CRYSTAL=#{install_dir}/bin/crystal FLAGS='#{crflags_arm}'", env: env - output_bin_arm64 = 'bin/shards_arm64' + output_bin_arm64 = "#{output_bin}_arm64" target_arm64 = 'arm64-apple-darwin' - command "clang bin/shards.o -o #{output_bin_arm64} -target #{target_arm64} -L#{install_dir}/embedded/lib -lyaml -lpcre2-8 -lgc -lpthread -levent -liconv -ldl", env: env + command "clang bin/shards.o -o #{output_bin_arm64} -v -target #{target_arm64} -L#{install_dir}/embedded/lib -lyaml -lpcre2-8 -lgc -lpthread -levent -liconv -ldl", env: env block "Testing the result file" do puts "===== >>> Testing the result file #{output_bin_arm64}" raise "Could not build #{output_bin_arm64}" unless File.exist?("#{output_bin_arm64}") @@ -120,7 +124,6 @@ # Lipo them up block { puts "\n===== 3. Combine x86_64 and arm64 binaries in a single universal binary\n\n" } - output_bin = 'bin/shards' command "lipo -create -output #{output_bin} #{output_bin_x86_64} #{output_bin_arm64}" block "Testing the result file" do puts "===== >>> Testing the result file #{output_bin}" @@ -130,6 +133,6 @@ command "file #{output_bin}", env: env.dup copy "bin/shards", "#{install_dir}/embedded/bin/shards" - + block { puts "\n===< Shards successfully built\n\n" } end diff --git a/omnibus/config/software/tgz_package.rb b/omnibus/config/software/tgz_package.rb index b550974c..2e672b1e 100644 --- a/omnibus/config/software/tgz_package.rb +++ b/omnibus/config/software/tgz_package.rb @@ -7,7 +7,7 @@ destination = File.expand_path('pkg', Omnibus::Config.project_root) version = "#{project.build_version}-#{project.build_iteration}" version.gsub!("/", "-") - tgz_name = "#{project.name}-#{version}-#{ohai['os']}-universal.tar.gz" + tgz_name = "#{project.name}-#{version}-#{ohai['os']}-#{ohai['kernel']['machine']}.tar.gz" if macos? || mac_os_x? transform = "-s /./#{project.name}-#{version}/" else From 62bd413510b5222a231b758da2779cf5aa8a92f5 Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Sun, 12 Nov 2023 19:09:41 +0100 Subject: [PATCH 15/26] reorgansie gitignore --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index e02e43c8..44df7663 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,8 @@ omnibus/pkg/ -omnibus/.bundle -omnibus/bin omnibus/crystal-darwin* +omnibus/.bundle omnibus/vendor +omnibus/bin docs/build/ From 59f48d049832c3beccff2051d59cafc882ad4ce8 Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Sun, 12 Nov 2023 19:26:40 +0100 Subject: [PATCH 16/26] make unversal filenames --- darwin/Makefile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/darwin/Makefile b/darwin/Makefile index d7abc1a4..d56a7f8c 100644 --- a/darwin/Makefile +++ b/darwin/Makefile @@ -17,7 +17,9 @@ OUTPUT_DIR = build DARWIN_ARCH = $(shell uname -m) DARWIN_PREFIX = crystal-$(CRYSTAL_VERSION)-$(PACKAGE_ITERATION) DARWIN_NAME = $(DARWIN_PREFIX)-darwin-$(DARWIN_ARCH).tar.gz +DARWIN_NAME_UNIVERSAL = $(DARWIN_PREFIX)-darwin-universal.tar.gz DARWIN_PKG_NAME = $(DARWIN_PREFIX).$(DARWIN_ARCH).pkg +DARWIN_PKG_NAME_UNIVERSAL = $(DARWIN_PREFIX).universal.pkg .PHONY: all all: darwin-previous $(OUTPUT_DIR)/$(DARWIN_NAME) $(OUTPUT_DIR)/$(DARWIN_PKG_NAME) ## Build compressed omnibus and distribution packages [default] @@ -42,7 +44,6 @@ awk 'BEGIN {FS = "## "}; /^## [a-zA-Z_-]/ {printf " \033[36m%s\033[0m\n", $$2}; .PHONY: darwin-previous darwin-previous: $(CURDIR)/../omnibus/crystal-darwin/embedded/bin/crystal ## download previous crystal darwin release -# Once there are prior builds for arm64, this can use DARWIN_ARCH $(CURDIR)/../omnibus/crystal-darwin/embedded/bin/crystal: curl -L -o /tmp/crystal-darwin.tar.gz $(PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ) \ && mkdir -p $(CURDIR)/../omnibus/crystal-darwin \ @@ -50,7 +51,7 @@ $(CURDIR)/../omnibus/crystal-darwin/embedded/bin/crystal: && rm /tmp/crystal-darwin.tar.gz \ && chmod +x $(CURDIR)/../omnibus/crystal-darwin/embedded/bin/crystal -$(OUTPUT_DIR)/$(DARWIN_NAME) $(OUTPUT_DIR)/$(DARWIN_PKG_NAME): ## Build omnibus crystal project +$(OUTPUT_DIR)/$(DARWIN_NAME_UNIVERSAL) $(OUTPUT_DIR)/$(DARWIN_PKG_NAME_UNIVERSAL): ## Build omnibus crystal project ifeq ($(FORCE_GIT_TAGGED), 0) rm -Rf $(CURDIR)/tmp && mkdir -p $(CURDIR)/tmp && cd $(CURDIR)/tmp \ && git clone "$(CRYSTAL_REPO)" \ @@ -66,8 +67,8 @@ endif && export SDKROOT=$(shell xcrun --sdk macosx --show-sdk-path) \ && bundle exec omnibus clean crystal shards \ && bundle exec omnibus build crystal --log-level debug \ - && cp ./pkg/$(DARWIN_NAME) $(CURDIR)/$(OUTPUT_DIR)/$(subst x86_64,universal,$(DARWIN_NAME)) \ - && cp ./pkg/$(DARWIN_PKG_NAME) $(CURDIR)/$(OUTPUT_DIR)/$(subst x86_64,universal,$(DARWIN_PKG_NAME)) + && cp ./pkg/$(DARWIN_NAME) $(CURDIR)/$(OUTPUT_DIR)/$(DARWIN_NAME_UNIVERSAL) \ + && cp ./pkg/$(DARWIN_PKG_NAME) $(CURDIR)/$(OUTPUT_DIR)/$(DARWIN_PKG_NAME_UNIVERSAL) .PHONY: clean clean: ## Clean up build directory From 31dcf573654c1ab2ca49ea49248b701e1e43b166 Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Sun, 12 Nov 2023 19:27:56 +0100 Subject: [PATCH 17/26] Use universal in tar gz --- darwin/Makefile | 2 +- omnibus/config/software/tgz_package.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/darwin/Makefile b/darwin/Makefile index d56a7f8c..bac131b3 100644 --- a/darwin/Makefile +++ b/darwin/Makefile @@ -67,7 +67,7 @@ endif && export SDKROOT=$(shell xcrun --sdk macosx --show-sdk-path) \ && bundle exec omnibus clean crystal shards \ && bundle exec omnibus build crystal --log-level debug \ - && cp ./pkg/$(DARWIN_NAME) $(CURDIR)/$(OUTPUT_DIR)/$(DARWIN_NAME_UNIVERSAL) \ + && cp ./pkg/$(DARWIN_NAME_UNIVERSAL) $(CURDIR)/$(OUTPUT_DIR)/$(DARWIN_NAME_UNIVERSAL) \ && cp ./pkg/$(DARWIN_PKG_NAME) $(CURDIR)/$(OUTPUT_DIR)/$(DARWIN_PKG_NAME_UNIVERSAL) .PHONY: clean diff --git a/omnibus/config/software/tgz_package.rb b/omnibus/config/software/tgz_package.rb index 2e672b1e..b550974c 100644 --- a/omnibus/config/software/tgz_package.rb +++ b/omnibus/config/software/tgz_package.rb @@ -7,7 +7,7 @@ destination = File.expand_path('pkg', Omnibus::Config.project_root) version = "#{project.build_version}-#{project.build_iteration}" version.gsub!("/", "-") - tgz_name = "#{project.name}-#{version}-#{ohai['os']}-#{ohai['kernel']['machine']}.tar.gz" + tgz_name = "#{project.name}-#{version}-#{ohai['os']}-universal.tar.gz" if macos? || mac_os_x? transform = "-s /./#{project.name}-#{version}/" else From 02c93e692ee66105f65fac8394af4e49a87a7ce8 Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Sun, 12 Nov 2023 19:39:58 +0100 Subject: [PATCH 18/26] universal tagz --- darwin/Makefile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/darwin/Makefile b/darwin/Makefile index bac131b3..a24b3972 100644 --- a/darwin/Makefile +++ b/darwin/Makefile @@ -16,8 +16,7 @@ OUTPUT_DIR = build # mimics the tgz_package.rb version mangling DARWIN_ARCH = $(shell uname -m) DARWIN_PREFIX = crystal-$(CRYSTAL_VERSION)-$(PACKAGE_ITERATION) -DARWIN_NAME = $(DARWIN_PREFIX)-darwin-$(DARWIN_ARCH).tar.gz -DARWIN_NAME_UNIVERSAL = $(DARWIN_PREFIX)-darwin-universal.tar.gz +DARWIN_NAME = $(DARWIN_PREFIX)-darwin-universal.tar.gz DARWIN_PKG_NAME = $(DARWIN_PREFIX).$(DARWIN_ARCH).pkg DARWIN_PKG_NAME_UNIVERSAL = $(DARWIN_PREFIX).universal.pkg @@ -51,7 +50,7 @@ $(CURDIR)/../omnibus/crystal-darwin/embedded/bin/crystal: && rm /tmp/crystal-darwin.tar.gz \ && chmod +x $(CURDIR)/../omnibus/crystal-darwin/embedded/bin/crystal -$(OUTPUT_DIR)/$(DARWIN_NAME_UNIVERSAL) $(OUTPUT_DIR)/$(DARWIN_PKG_NAME_UNIVERSAL): ## Build omnibus crystal project +$(OUTPUT_DIR)/$(DARWIN_NAME) $(OUTPUT_DIR)/$(DARWIN_PKG_NAME_UNIVERSAL): ## Build omnibus crystal project ifeq ($(FORCE_GIT_TAGGED), 0) rm -Rf $(CURDIR)/tmp && mkdir -p $(CURDIR)/tmp && cd $(CURDIR)/tmp \ && git clone "$(CRYSTAL_REPO)" \ @@ -67,7 +66,7 @@ endif && export SDKROOT=$(shell xcrun --sdk macosx --show-sdk-path) \ && bundle exec omnibus clean crystal shards \ && bundle exec omnibus build crystal --log-level debug \ - && cp ./pkg/$(DARWIN_NAME_UNIVERSAL) $(CURDIR)/$(OUTPUT_DIR)/$(DARWIN_NAME_UNIVERSAL) \ + && cp ./pkg/$(DARWIN_NAME) $(CURDIR)/$(OUTPUT_DIR)/$(DARWIN_NAME) \ && cp ./pkg/$(DARWIN_PKG_NAME) $(CURDIR)/$(OUTPUT_DIR)/$(DARWIN_PKG_NAME_UNIVERSAL) .PHONY: clean From 535986c16ed2ca3c3905f898151ad9939ea9135f Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Sun, 12 Nov 2023 20:44:09 +0100 Subject: [PATCH 19/26] Macos target 11.0 --- darwin/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/darwin/Makefile b/darwin/Makefile index a24b3972..f34d63ac 100644 --- a/darwin/Makefile +++ b/darwin/Makefile @@ -21,7 +21,7 @@ DARWIN_PKG_NAME = $(DARWIN_PREFIX).$(DARWIN_ARCH).pkg DARWIN_PKG_NAME_UNIVERSAL = $(DARWIN_PREFIX).universal.pkg .PHONY: all -all: darwin-previous $(OUTPUT_DIR)/$(DARWIN_NAME) $(OUTPUT_DIR)/$(DARWIN_PKG_NAME) ## Build compressed omnibus and distribution packages [default] +all: darwin-previous $(OUTPUT_DIR)/$(DARWIN_NAME) $(OUTPUT_DIR)/$(DARWIN_PKG_NAME_UNIVERSAL) ## Build compressed omnibus and distribution packages [default] .PHONY: help help: ## Show this help From d3334087585bf55c232da1856a3fd973cb3385ff Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Sun, 12 Nov 2023 21:36:20 +0100 Subject: [PATCH 20/26] env ldlibs is not required --- omnibus/config/software/crystal.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/omnibus/config/software/crystal.rb b/omnibus/config/software/crystal.rb index f5808872..16a51eb5 100644 --- a/omnibus/config/software/crystal.rb +++ b/omnibus/config/software/crystal.rb @@ -86,7 +86,6 @@ # Build for x86_64 env["CXXFLAGS"] = original_CXXFLAGS_env + " -target x86_64-apple-darwin" env["LDFLAGS"] = original_LDFLAGS_env + " -v -target x86_64-apple-darwin" - env["LDLIBS"] = "-v -target x86_64-apple-darwin" make "deps", env: env.dup make "crystal verbose=true stats=true release=true target=x86_64-apple-darwin FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_TARGET=x86_64-apple-darwin CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env @@ -107,7 +106,6 @@ # so we need to sub out aarch64 in our calls to Apple tools env["CXXFLAGS"] = original_CXXFLAGS_env + " -target arm64-apple-darwin" env["LDFLAGS"] = original_LDFLAGS_env + " -v -target arm64-apple-darwin" - env["LDLIBS"] = "-v -target x86_64-apple-darwin" make "deps", env: env.dup make "crystal verbose=true stats=true release=true target=aarch64-apple-darwin FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_TARGET=aarch64-apple-darwin CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env command "clang #{output_path}/crystal.o -o #{output_bin_arm} -target arm64-apple-darwin src/llvm/ext/llvm_ext.o `llvm-config --libs --system-libs --ldflags 2>/dev/null` -lstdc++ -lpcre2-8 -lgc -lpthread -levent -liconv -ldl -v", env: env From c1b974da0b6bceb0d3f416f9c4bb676abfae24fb Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Mon, 13 Nov 2023 17:25:33 +0100 Subject: [PATCH 21/26] Allow to specify custom CRYSTAL_SRC for forks --- Makefile | 3 +-- darwin/Makefile | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 392ab3fd..2681dac7 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,6 @@ clean: omnibus_setup: cd omnibus && bundle install --binstubs --path vendor/bundler - .PHONY: darwin darwin: omnibus_setup - CRYSTAL_SRC=/Users/miry/src/crystal/crystal/.git CRYSTAL_SHA1=master make -C darwin CRYSTAL_VERSION=1.10.1 PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ=https://github.com/crystal-lang/crystal/releases/download/1.10.1/crystal-1.10.1-1-darwin-universal.tar.gz + FORCE_GIT_TAGGED=0 CRYSTAL_SRC=https://github.com/crystal-lang/crystal CRYSTAL_SHA1=master make -C darwin CRYSTAL_VERSION=1.10.1 PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ=https://github.com/crystal-lang/crystal/releases/download/1.10.1/crystal-1.10.1-1-darwin-universal.tar.gz diff --git a/darwin/Makefile b/darwin/Makefile index f34d63ac..6ee2822d 100644 --- a/darwin/Makefile +++ b/darwin/Makefile @@ -55,7 +55,7 @@ ifeq ($(FORCE_GIT_TAGGED), 0) rm -Rf $(CURDIR)/tmp && mkdir -p $(CURDIR)/tmp && cd $(CURDIR)/tmp \ && git clone "$(CRYSTAL_REPO)" \ && cd crystal \ - && git checkout $(CRYSTAL_SHA1) \ + && git checkout "$(CRYSTAL_SHA1)" \ && git checkout -b $(CRYSTAL_VERSION) endif From ab86116bab0849342a0691b92452c7d1cf0f78ea Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Mon, 13 Nov 2023 17:27:55 +0100 Subject: [PATCH 22/26] Revert linux dockerfile --- linux/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linux/Dockerfile b/linux/Dockerfile index 1d8c0fe2..7ae24e60 100644 --- a/linux/Dockerfile +++ b/linux/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=$BUILDPLATFORM debian:11 AS debian +FROM debian:11 AS debian RUN apt-get update \ && apt-get install -y build-essential libevent-dev libpcre2-dev automake libtool pkg-config git curl llvm-13 clang-13 \ From c5f50d2f941fc365c5542025de6aa9dd097144d8 Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Mon, 13 Nov 2023 19:53:26 +0100 Subject: [PATCH 23/26] Use universal --- .gitignore | 2 +- darwin/Makefile | 12 ++++++------ omnibus/config/software/crystal.rb | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 44df7663..c6beae71 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ omnibus/pkg/ -omnibus/crystal-darwin* +omnibus/crystal-darwin-universal omnibus/.bundle omnibus/vendor omnibus/bin diff --git a/darwin/Makefile b/darwin/Makefile index 6ee2822d..18e89225 100644 --- a/darwin/Makefile +++ b/darwin/Makefile @@ -41,14 +41,14 @@ help: ## Show this help awk 'BEGIN {FS = "## "}; /^## [a-zA-Z_-]/ {printf " \033[36m%s\033[0m\n", $$2}; /^## / {printf " %s\n", $$2}' .PHONY: darwin-previous -darwin-previous: $(CURDIR)/../omnibus/crystal-darwin/embedded/bin/crystal ## download previous crystal darwin release +darwin-previous: $(CURDIR)/../omnibus/crystal-darwin-universal/embedded/bin/crystal ## download previous crystal darwin release -$(CURDIR)/../omnibus/crystal-darwin/embedded/bin/crystal: +$(CURDIR)/../omnibus/crystal-darwin-universal/embedded/bin/crystal: curl -L -o /tmp/crystal-darwin.tar.gz $(PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ) \ - && mkdir -p $(CURDIR)/../omnibus/crystal-darwin \ - && tar xfz /tmp/crystal-darwin.tar.gz -C $(CURDIR)/../omnibus/crystal-darwin --strip-components=1 \ + && mkdir -p $(CURDIR)/../omnibus/crystal-darwin-universal \ + && tar xfz /tmp/crystal-darwin.tar.gz -C $(CURDIR)/../omnibus/crystal-darwin-universal --strip-components=1 \ && rm /tmp/crystal-darwin.tar.gz \ - && chmod +x $(CURDIR)/../omnibus/crystal-darwin/embedded/bin/crystal + && chmod +x $(CURDIR)/../omnibus/crystal-darwin-universal/embedded/bin/crystal $(OUTPUT_DIR)/$(DARWIN_NAME) $(OUTPUT_DIR)/$(DARWIN_PKG_NAME_UNIVERSAL): ## Build omnibus crystal project ifeq ($(FORCE_GIT_TAGGED), 0) @@ -75,6 +75,6 @@ clean: ## Clean up build directory rm -Rf $(CURDIR)/tmp rm -Rf $(CURDIR)/../omnibus/pkg/crystal-* rm -Rf $(CURDIR)/../omnibus/pkg/version-* - rm -Rf $(CURDIR)/../omnibus/crystal-darwin* + rm -Rf $(CURDIR)/../omnibus/crystal-darwin-universal* rm -Rf /var/cache/omnibus/* rm -Rf /opt/crystal/* diff --git a/omnibus/config/software/crystal.rb b/omnibus/config/software/crystal.rb index 16a51eb5..f5d84a3c 100644 --- a/omnibus/config/software/crystal.rb +++ b/omnibus/config/software/crystal.rb @@ -56,7 +56,8 @@ make "deps", env: env.dup mkdir ".build" - copy "#{Dir.pwd}/crystal-#{ohai['os']}/embedded/bin/crystal", ".build/crystal" + block { raise "Could not find embedded/bin/crystal" unless File.exist?("#{Dir.pwd}/crystal-darwin-universal/embedded/bin/crystal") } + copy "#{Dir.pwd}/crystal-darwin-universal/embedded/bin/crystal", ".build/crystal" command ".build/crystal --version", env: env.dup command "file .build/crystal", env: env.dup From 8de3941605b0d05120eabcc0ca02eecaf2ed8b39 Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Mon, 13 Nov 2023 20:10:48 +0100 Subject: [PATCH 24/26] Revert changes for docker/makefile --- docker/Makefile | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/docker/Makefile b/docker/Makefile index ce9c7927..5a1bb7a7 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -33,9 +33,9 @@ ubuntu64: $(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-ubuntu-x86_64.tar.gz ubuntu64: $(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-ubuntu-x86_64-build.tar.gz .PHONY: alpine -alpine: ## Build alpine x86_64 images -alpine: $(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-alpine-x86_64.tar.gz -alpine: $(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-alpine-x86_64-build.tar.gz +alpine: ## Build alpine images +alpine: $(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-alpine.tar.gz +alpine: $(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-alpine-build.tar.gz $(BUILD_CONTEXT)/ubuntu64: ubuntu.Dockerfile $(BUILD_CONTEXT)/ubuntu64/crystal.tar.gz cp ubuntu.Dockerfile $@/Dockerfile @@ -55,14 +55,6 @@ $(BUILD_CONTEXT)/alpine: alpine.Dockerfile $(BUILD_CONTEXT)/alpine/crystal.tar.g $(OUTPUT_DIR): mkdir -p $(OUTPUT_DIR) -# .PHONY: docker-image -# docker-image: -# DOCKER_BUILDKIT=1 docker build -f Dockerfile -t opentracing-contrib/nginx-opentracing --target final . -# -# .PHONY: docker-image-alpine -# docker-image-alpine: -# DOCKER_BUILDKIT=1 docker build -f Dockerfile -t opentracing-contrib/nginx-opentracing --target final --build-arg BUILD_OS=alpine . - # Ubuntu x86_64 $(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-ubuntu-x86_64.tar.gz: $(BUILD_CONTEXT)/ubuntu64 $(OUTPUT_DIR) docker build -t $(DOCKER_TAG_UBUNTU) --target runtime $(BUILD_ARGS_UBUNTU64) From 4f082d25687bc2052d1373cd68bd8f02865d951f Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Mon, 13 Nov 2023 20:15:40 +0100 Subject: [PATCH 25/26] Use home folder isntead of static --- .gitignore | 1 + omnibus/config/projects/crystal.rb | 2 +- omnibus/config/projects/llvm.rb | 2 +- omnibus/omnibus.rb | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index c6beae71..a54b7129 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ omnibus/bin docs/build/ darwin/build/ +darwin/tmp/ snapcraft/snap/snapcraft.yaml snapcraft/*.snap diff --git a/omnibus/config/projects/crystal.rb b/omnibus/config/projects/crystal.rb index 7b8eee04..5f8f642e 100644 --- a/omnibus/config/projects/crystal.rb +++ b/omnibus/config/projects/crystal.rb @@ -2,7 +2,7 @@ maintainer 'Juan Wajnerman' homepage 'http://crystal-lang.org/' -install_dir '/Users/miry/tmp/crystal' +install_dir File.join(ENV.fetch('HOME'), '/tmp/crystal') build_version do source :version, from_dependency: 'crystal' end diff --git a/omnibus/config/projects/llvm.rb b/omnibus/config/projects/llvm.rb index 9e07faf0..92099ec3 100644 --- a/omnibus/config/projects/llvm.rb +++ b/omnibus/config/projects/llvm.rb @@ -2,7 +2,7 @@ maintainer 'Juan Wajnerman' homepage 'http://llvm.org/' -install_dir '/Users/miry/tmp/llvm' +install_dir File.join(ENV.fetch('HOME'), '/tmp/llvm') build_version do source :version, from_dependency: 'llvm' end diff --git a/omnibus/omnibus.rb b/omnibus/omnibus.rb index dbb6cb1d..2abb9f2d 100644 --- a/omnibus/omnibus.rb +++ b/omnibus/omnibus.rb @@ -17,7 +17,7 @@ # # Uncomment this line to change the default base directory to "local" # ------------------------------------------------------------------- -base_dir '/Users/miry/tmp/local' +base_dir File.join(ENV.fetch('HOME'), '/tmp/omnibus') # # Alternatively you can tune the individual values # ------------------------------------------------ From f6c86d8bfc2ba3fcb66675ce6b49e72cb51bd015 Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Thu, 4 Jan 2024 23:09:14 +0100 Subject: [PATCH 26/26] Darwin makefile uses space in env variable --- Makefile | 6 +++--- darwin/Makefile | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 2681dac7..d21bc490 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: docker-image docker-image: - make -C linux + make -C linux DOCKER_BUILDKIT=1 docker build -f Dockerfile -t opentracing-contrib/nginx-opentracing --target final . .PHONY: test @@ -21,7 +21,7 @@ clean: .PHONY: setup omnibus_setup: cd omnibus && bundle install --binstubs --path vendor/bundler - + .PHONY: darwin darwin: omnibus_setup - FORCE_GIT_TAGGED=0 CRYSTAL_SRC=https://github.com/crystal-lang/crystal CRYSTAL_SHA1=master make -C darwin CRYSTAL_VERSION=1.10.1 PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ=https://github.com/crystal-lang/crystal/releases/download/1.10.1/crystal-1.10.1-1-darwin-universal.tar.gz + FORCE_GIT_TAGGED=0 CRYSTAL_REPO=https://github.com/crystal-lang/crystal CRYSTAL_SRC=https://github.com/crystal-lang/crystal CRYSTAL_SHA1=master make -C darwin CRYSTAL_VERSION=1.10.1 PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ=https://github.com/crystal-lang/crystal/releases/download/1.10.1/crystal-1.10.1-1-darwin-universal.tar.gz diff --git a/darwin/Makefile b/darwin/Makefile index 18e89225..2eea1849 100644 --- a/darwin/Makefile +++ b/darwin/Makefile @@ -53,9 +53,9 @@ $(CURDIR)/../omnibus/crystal-darwin-universal/embedded/bin/crystal: $(OUTPUT_DIR)/$(DARWIN_NAME) $(OUTPUT_DIR)/$(DARWIN_PKG_NAME_UNIVERSAL): ## Build omnibus crystal project ifeq ($(FORCE_GIT_TAGGED), 0) rm -Rf $(CURDIR)/tmp && mkdir -p $(CURDIR)/tmp && cd $(CURDIR)/tmp \ - && git clone "$(CRYSTAL_REPO)" \ + && git clone $(CRYSTAL_REPO) \ && cd crystal \ - && git checkout "$(CRYSTAL_SHA1)" \ + && git checkout $(CRYSTAL_SHA1) \ && git checkout -b $(CRYSTAL_VERSION) endif