From 04082c08da1b29bcb42024d6cf886b6e6f6f8cc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Mon, 9 Mar 2026 23:23:28 +0100 Subject: [PATCH 1/2] Stream tarball downloads and file uploads to avoid memory buffering - Stream tarball downloads to disk via Store.get_to_file instead of loading full binary into memory - Stream file uploads via put_file! to avoid File.read! on every doc - Add TmpDir GenServer for process-based temp file cleanup - Add put_file! to Store behaviour and backends (GS, S3, Local) - Fix flaky debouncer test grace time - Fix unsafe paths test to assert on error logs --- lib/hexdocs/application.ex | 1 + lib/hexdocs/bucket.ex | 92 +++++++++++++++++++------------- lib/hexdocs/http.ex | 18 +++++++ lib/hexdocs/queue.ex | 93 +++++++++++++++++++++------------ lib/hexdocs/store/gs.ex | 14 ++++- lib/hexdocs/store/impl.ex | 10 ++++ lib/hexdocs/store/local.ex | 17 ++++++ lib/hexdocs/store/s3.ex | 7 +++ lib/hexdocs/store/store.ex | 4 ++ lib/hexdocs/tar.ex | 65 +++++++---------------- lib/hexdocs/tmp_dir.ex | 85 ++++++++++++++++++++++++++++++ mix.exs | 2 +- mix.lock | 2 +- test/hexdocs/bucket_test.exs | 87 +++++++++++++++++------------- test/hexdocs/debouncer_test.exs | 2 +- test/hexdocs/queue_test.exs | 17 +++--- test/hexdocs/tar_test.exs | 24 ++++++--- test/hexdocs/tmp_dir_test.exs | 79 ++++++++++++++++++++++++++++ 18 files changed, 450 insertions(+), 169 deletions(-) create mode 100644 lib/hexdocs/tmp_dir.ex create mode 100644 test/hexdocs/tmp_dir_test.exs diff --git a/lib/hexdocs/application.ex b/lib/hexdocs/application.ex index fdf0223..33fbcdf 100644 --- a/lib/hexdocs/application.ex +++ b/lib/hexdocs/application.ex @@ -12,6 +12,7 @@ defmodule Hexdocs.Application do Logger.info("Running Cowboy with #{inspect(cowboy_options)}") children = [ + Hexdocs.TmpDir, {Task.Supervisor, name: Hexdocs.Tasks}, {Hexdocs.Debouncer, name: Hexdocs.Debouncer}, goth_spec(), diff --git a/lib/hexdocs/bucket.ex b/lib/hexdocs/bucket.ex index 50c1386..63b9d7b 100644 --- a/lib/hexdocs/bucket.ex +++ b/lib/hexdocs/bucket.ex @@ -23,8 +23,6 @@ defmodule Hexdocs.Bucket do meta: [{"surrogate-key", key}] ] - Logger.info("Uploading docs_public_bucket #{path}") - case Hexdocs.Store.put(:docs_public_bucket, path, content, opts) do {:ok, 200, _headers, _body} -> :ok @@ -39,10 +37,10 @@ defmodule Hexdocs.Bucket do purge([key]) end - def upload(repository, package, version, all_versions, files) do + def upload(repository, package, version, all_versions, dir, files) do latest_version? = Hexdocs.Utils.latest_version?(package, version, all_versions) upload_type = upload_type(latest_version?) - upload_files = list_upload_files(repository, package, version, files, upload_type) + upload_files = list_upload_files(repository, package, version, dir, files, upload_type) paths = MapSet.new(upload_files, &elem(&1, 0)) upload_new_files(upload_files) @@ -53,7 +51,7 @@ defmodule Hexdocs.Bucket do {:docs_config, repository, package}, @gcs_put_debounce, fn -> - docs_config = build_docs_config(repository, package, version, all_versions, files) + docs_config = build_docs_config(repository, package, version, all_versions, dir, files) upload_new_files([docs_config]) end ) @@ -63,17 +61,24 @@ defmodule Hexdocs.Bucket do end # For Elixir and Hex we use the docs_config.js included in the tarball - defp build_docs_config(repository, package, _version, _all_versions, files) + defp build_docs_config(repository, package, _version, _all_versions, dir, files) when package in @special_package_names do path = "docs_config.js" unversioned_path = repository_path(repository, Path.join([package, path])) cdn_key = docs_config_cdn_key(repository, package) - {"docs_config.js", data} = List.keyfind(files, "docs_config.js", 0) + + data = + if "docs_config.js" in files do + File.read!(Path.join(dir, "docs_config.js")) + else + "" + end + {unversioned_path, cdn_key, data, public?(repository)} end # TODO: don't include retired versions? - defp build_docs_config(repository, package, version, all_versions, _files) do + defp build_docs_config(repository, package, version, all_versions, _dir, _files) do versions = if version in all_versions do all_versions @@ -135,22 +140,32 @@ defmodule Hexdocs.Bucket do cond do deleting_latest_version? && new_latest_version -> key = build_key(repository, package, new_latest_version) - body = Hexdocs.Store.get(:repo_bucket, key) - - case Hexdocs.Tar.unpack(body, repository: repository, package: package, version: version) do - {:ok, files} -> - upload_files = - list_upload_files(repository, package, new_latest_version, files, :both) - - paths = MapSet.new(upload_files, &elem(&1, 0)) - update_versions = [version, new_latest_version] - - upload_new_files(upload_files) - delete_old_docs(repository, package, update_versions, paths, :both) - purge_hexdocs_cache(repository, package, update_versions, :both) - - {:error, reason} -> - Logger.error("Failed unpack #{repository}/#{package} #{version}: #{reason}") + tarball_path = Hexdocs.TmpDir.tmp_file("docs-tarball") + + case Hexdocs.Store.get_to_file(:repo_bucket, key, tarball_path) do + :ok -> + case Hexdocs.Tar.unpack_to_dir({:file, tarball_path}, + repository: repository, + package: package, + version: version + ) do + {:ok, dir, files} -> + upload_files = + list_upload_files(repository, package, new_latest_version, dir, files, :both) + + paths = MapSet.new(upload_files, &elem(&1, 0)) + update_versions = [version, new_latest_version] + + upload_new_files(upload_files) + delete_old_docs(repository, package, update_versions, paths, :both) + purge_hexdocs_cache(repository, package, update_versions, :both) + + {:error, reason} -> + Logger.error("Failed unpack #{repository}/#{package} #{version}: #{reason}") + end + + nil -> + Logger.error("Failed to get tarball #{repository}/#{package} #{new_latest_version}") end deleting_latest_version? -> @@ -171,21 +186,23 @@ defmodule Hexdocs.Bucket do Path.join(["repos", repository, "docs", "#{package}-#{version}.tar.gz"]) end - defp list_upload_files(repository, package, version, files, upload_type) do + defp list_upload_files(repository, package, version, dir, files, upload_type) do Enum.flat_map(files, fn - {"docs_config.js", _data} -> + "docs_config.js" -> [] - {path, data} -> + path -> + source = Path.join(dir, path) + versioned_path = repository_path(repository, Path.join([package, to_string(version), path])) cdn_key = docspage_versioned_cdn_key(repository, package, version) - versioned = {versioned_path, cdn_key, data, public?(repository)} + versioned = {versioned_path, cdn_key, {:file, source}, public?(repository)} unversioned_path = repository_path(repository, Path.join([package, path])) cdn_key = docspage_unversioned_cdn_key(repository, package) - unversioned = {unversioned_path, cdn_key, data, public?(repository)} + unversioned = {unversioned_path, cdn_key, {:file, source}, public?(repository)} case upload_type do :both -> [versioned, unversioned] @@ -210,8 +227,12 @@ defmodule Hexdocs.Bucket do {bucket(public?), store_key, data, opts} end) |> Task.async_stream( - fn {bucket, key, data, opts} -> - put(bucket, key, data, opts) + fn + {bucket, key, {:file, source}, opts} -> + put_file(bucket, key, source, opts) + + {bucket, key, data, opts} -> + put(bucket, key, data, opts) end, max_concurrency: 10, timeout: 60_000 @@ -239,10 +260,6 @@ defmodule Hexdocs.Bucket do &delete_key?(&1, paths, repository, package, versions, upload_type) ) - Enum.each(keys_to_delete, fn key -> - Logger.info("Deleting #{bucket} #{key}") - end) - Hexdocs.Store.delete_many(bucket, keys_to_delete) end @@ -332,7 +349,10 @@ defmodule Hexdocs.Bucket do end defp put(bucket, key, data, opts) do - Logger.info("Uploading #{bucket} #{key}") Hexdocs.Store.put!(bucket, key, data, opts) end + + defp put_file(bucket, key, source, opts) do + Hexdocs.Store.put_file!(bucket, key, source, opts) + end end diff --git a/lib/hexdocs/http.ex b/lib/hexdocs/http.ex index fbe7df6..4bcc31f 100644 --- a/lib/hexdocs/http.ex +++ b/lib/hexdocs/http.ex @@ -70,6 +70,24 @@ defmodule Hexdocs.HTTP do end end + def put_file(url, headers, path) do + body = File.stream!(path, 65_536) + + case Req.put(url, + headers: headers, + body: body, + retry: false, + decode_body: false, + receive_timeout: @receive_timeout + ) do + {:ok, response} -> + {:ok, response.status, normalize_headers(response.headers), response.body} + + {:error, reason} -> + {:error, reason} + end + end + def post(url, headers, body, opts \\ []) do timeout = Keyword.get(opts, :receive_timeout, @receive_timeout) diff --git a/lib/hexdocs/queue.ex b/lib/hexdocs/queue.ex index 1f850c7..02622b9 100644 --- a/lib/hexdocs/queue.ex +++ b/lib/hexdocs/queue.ex @@ -58,16 +58,26 @@ defmodule Hexdocs.Queue do case key_components(key) do {:ok, repository, package, version} -> - body = Hexdocs.Store.get(:repo_bucket, key) - - case Hexdocs.Tar.unpack(body, repository: repository, package: package, version: version) do - {:ok, files} -> - update_index_sitemap(repository, key) - update_package_sitemap(repository, key, package, files) - Logger.info("#{key}: done") - - {:error, reason} -> - Logger.error("Failed unpack #{repository}/#{package} #{version}: #{reason}") + tarball_path = Hexdocs.TmpDir.tmp_file("docs-tarball") + + case Hexdocs.Store.get_to_file(:repo_bucket, key, tarball_path) do + :ok -> + case Hexdocs.Tar.unpack_to_dir({:file, tarball_path}, + repository: repository, + package: package, + version: version + ) do + {:ok, _dir, files} -> + update_index_sitemap(repository, key) + update_package_sitemap(repository, key, package, files) + Logger.info("#{key}: done") + + {:error, reason} -> + Logger.error("Failed unpack #{repository}/#{package} #{version}: #{reason}") + end + + nil -> + Logger.error("#{key}: package not found in store") end :error -> @@ -104,18 +114,20 @@ defmodule Hexdocs.Queue do version: version }) - body = Hexdocs.Store.get(:repo_bucket, key) + tarball_path = Hexdocs.TmpDir.tmp_file("docs-tarball") - if body do - case type do - :upload -> - process_upload(key, repository, package, version, body, start) + case Hexdocs.Store.get_to_file(:repo_bucket, key, tarball_path) do + :ok -> + case type do + :upload -> + process_upload(key, repository, package, version, {:file, tarball_path}, start) - :search -> - process_search(key, repository, package, version, body, start) - end - else - Logger.error("#{log_prefix} #{key}: package not found in store") + :search -> + process_search(key, repository, package, version, {:file, tarball_path}, start) + end + + nil -> + Logger.error("#{log_prefix} #{key}: package not found in store") end :error -> @@ -123,7 +135,7 @@ defmodule Hexdocs.Queue do end end - defp process_upload(key, repository, package, version, body, start) do + defp process_upload(key, repository, package, version, input, start) do {version, all_versions} = if package in @special_package_names do version = @@ -144,15 +156,20 @@ defmodule Hexdocs.Queue do {version, all_versions} end - case Hexdocs.Tar.unpack(body, repository: repository, package: package, version: version) do - {:ok, files} -> - files = rewrite_files(files) + case Hexdocs.Tar.unpack_to_dir(input, + repository: repository, + package: package, + version: version + ) do + {:ok, dir, files} -> + rewrite_files(dir, files) Hexdocs.Bucket.upload( repository, package, version, all_versions, + dir, files ) @@ -170,7 +187,7 @@ defmodule Hexdocs.Queue do end end - defp process_search(key, repository, package, version, body, start) do + defp process_search(key, repository, package, version, input, start) do if repository != "hexpm" do Logger.warning("SKIPPING SEARCH INDEX #{key} (repository is not hexpm)") else @@ -180,9 +197,9 @@ defmodule Hexdocs.Queue do :error when package in @special_package_names -> version end - case Hexdocs.Tar.unpack(body, package: package, version: version) do - {:ok, files} -> - update_search_index(key, package, version, files) + case Hexdocs.Tar.unpack_to_dir(input, package: package, version: version) do + {:ok, dir, files} -> + update_search_index(key, package, version, dir, files) elapsed = System.os_time(:millisecond) - start Logger.info("FINISHED INDEXING DOCS #{key} #{elapsed}ms") @@ -279,9 +296,12 @@ defmodule Hexdocs.Queue do {package, version} end - defp rewrite_files(files) do - Enum.map(files, fn {path, content} -> - {path, Hexdocs.FileRewriter.run(path, content)} + defp rewrite_files(dir, files) do + Enum.each(files, fn path -> + full_path = Path.join(dir, path) + content = File.read!(full_path) + rewritten = Hexdocs.FileRewriter.run(path, content) + File.write!(full_path, rewritten) end) end @@ -314,7 +334,7 @@ defmodule Hexdocs.Queue do defp update_package_sitemap("hexpm", key, package, files) do Logger.info("UPDATING PACKAGE SITEMAP #{key}") - pages = for {path, _content} <- files, Path.extname(path) == ".html", do: path + pages = for path <- files, Path.extname(path) == ".html", do: path body = Hexdocs.PackageSitemap.render(package, pages, DateTime.utc_now()) Hexdocs.Bucket.upload_package_sitemap(package, body) @@ -344,8 +364,13 @@ defmodule Hexdocs.Queue do :ok end - defp update_search_index(key, package, version, files) do - case Hexdocs.Search.find_search_items(package, version, files) do + defp update_search_index(key, package, version, dir, files) do + files_with_content = + Enum.map(files, fn path -> + {path, File.read!(Path.join(dir, path))} + end) + + case Hexdocs.Search.find_search_items(package, version, files_with_content) do {proglang, items} -> Logger.info("DELETING SEARCH INDEX #{key}") Hexdocs.Search.delete(package, version) diff --git a/lib/hexdocs/store/gs.ex b/lib/hexdocs/store/gs.ex index 04ea0cc..d041310 100644 --- a/lib/hexdocs/store/gs.ex +++ b/lib/hexdocs/store/gs.ex @@ -52,6 +52,18 @@ defmodule Hexdocs.Store.GS do end def put!(bucket, key, blob, opts) do + upload(bucket, key, opts, fn url, headers -> + Hexdocs.HTTP.put(url, headers, blob) + end) + end + + def put_file!(bucket, key, source, opts) do + upload(bucket, key, opts, fn url, headers -> + Hexdocs.HTTP.put_file(url, headers, source) + end) + end + + defp upload(bucket, key, opts, fun) do headers = headers() ++ meta_headers(Keyword.fetch!(opts, :meta)) ++ @@ -64,7 +76,7 @@ defmodule Hexdocs.Store.GS do headers = filter_nil_values(headers) {:ok, 200, _headers, _body} = - Hexdocs.HTTP.retry("gs", url, fn -> Hexdocs.HTTP.put(url, headers, blob) end) + Hexdocs.HTTP.retry("gs", url, fn -> fun.(url, headers) end) :ok end diff --git a/lib/hexdocs/store/impl.ex b/lib/hexdocs/store/impl.ex index 9ef1870..c90e67b 100644 --- a/lib/hexdocs/store/impl.ex +++ b/lib/hexdocs/store/impl.ex @@ -12,6 +12,11 @@ defmodule Hexdocs.Store.Impl do impl.get(name, key, opts) end + def get_to_file(bucket, key, dest, opts) do + {impl, name} = bucket(bucket) + impl.get_to_file(name, key, dest, opts) + end + def head_page(bucket, key, opts) do {impl, name} = bucket(bucket) impl.head_page(name, key, opts) @@ -37,6 +42,11 @@ defmodule Hexdocs.Store.Impl do impl.put!(name, key, body, opts) end + def put_file!(bucket, key, source, opts) do + {impl, name} = bucket(bucket) + impl.put_file!(name, key, source, opts) + end + def delete_many(bucket, keys) do {impl, name} = bucket(bucket) impl.delete_many(name, keys) diff --git a/lib/hexdocs/store/local.ex b/lib/hexdocs/store/local.ex index 8f8892a..b93b615 100644 --- a/lib/hexdocs/store/local.ex +++ b/lib/hexdocs/store/local.ex @@ -26,6 +26,17 @@ defmodule Hexdocs.Store.Local do end end + def get_to_file(bucket, key, dest, _opts) do + path = Path.join([dir(), bucket(bucket), key]) + + if File.regular?(path) do + File.cp!(path, dest) + :ok + else + nil + end + end + def head_page(bucket, key, _opts) do path = Path.join([dir(), bucket(bucket), key]) @@ -69,6 +80,12 @@ defmodule Hexdocs.Store.Local do File.write!(path, blob) end + def put_file!(bucket, key, source, _opts) do + path = Path.join([dir(), bucket(bucket), key]) + File.mkdir_p!(Path.dirname(path)) + File.cp!(source, path) + end + def delete(bucket, key) do [dir(), bucket(bucket), key] |> Path.join() diff --git a/lib/hexdocs/store/s3.ex b/lib/hexdocs/store/s3.ex index bb03463..5e43eb3 100644 --- a/lib/hexdocs/store/s3.ex +++ b/lib/hexdocs/store/s3.ex @@ -10,6 +10,13 @@ defmodule Hexdocs.Store.S3 do end end + def get_to_file(bucket, key, dest, _opts) do + case ExAws.S3.download_file(bucket, key, dest) |> ExAws.request() do + {:ok, _} -> :ok + {:error, {:http_error, 404, _}} -> nil + end + end + def list(bucket, prefix) do ExAws.S3.list_objects(bucket, prefix: prefix) |> ExAws.stream!() diff --git a/lib/hexdocs/store/store.ex b/lib/hexdocs/store/store.ex index b07a2d7..67e753b 100644 --- a/lib/hexdocs/store/store.ex +++ b/lib/hexdocs/store/store.ex @@ -15,6 +15,7 @@ defmodule Hexdocs.Store do @type opts :: Keyword.t() @callback get(bucket, key, opts) :: body | nil + @callback get_to_file(bucket, key, dest :: String.t(), opts) :: :ok | nil end defmodule Docs do @@ -33,6 +34,7 @@ defmodule Hexdocs.Store do @callback stream_page(bucket, key, opts) :: {status, headers, stream} @callback put(bucket, key, body, opts) :: term @callback put!(bucket, key, body, opts) :: term + @callback put_file!(bucket, key, source :: String.t(), opts) :: term @callback delete_many(bucket, [key]) :: [term] end @@ -40,10 +42,12 @@ defmodule Hexdocs.Store do def list(bucket, prefix), do: impl().list(bucket, prefix) def get(bucket, key, opts \\ []), do: impl().get(bucket, key, opts) + def get_to_file(bucket, key, dest, opts \\ []), do: impl().get_to_file(bucket, key, dest, opts) def head_page(bucket, key, opts \\ []), do: impl().head_page(bucket, key, opts) def get_page(bucket, key, opts \\ []), do: impl().get_page(bucket, key, opts) def stream_page(bucket, key, opts \\ []), do: impl().stream_page(bucket, key, opts) def put(bucket, key, body, opts \\ []), do: impl().put(bucket, key, body, opts) def put!(bucket, key, body, opts \\ []), do: impl().put!(bucket, key, body, opts) + def put_file!(bucket, key, source, opts \\ []), do: impl().put_file!(bucket, key, source, opts) def delete_many(bucket, keys), do: impl().delete_many(bucket, keys) end diff --git a/lib/hexdocs/tar.ex b/lib/hexdocs/tar.ex index 605622f..066cef8 100644 --- a/lib/hexdocs/tar.ex +++ b/lib/hexdocs/tar.ex @@ -1,68 +1,43 @@ defmodule Hexdocs.Tar do require Logger - @zlib_magic 16 + 15 - @compressed_max_size 16 * 1024 * 1024 - @uncompressed_max_size 128 * 1024 * 1024 - def create(files) do files = for {path, contents} <- files, do: {String.to_charlist(path), contents} {:ok, tarball} = :hex_tarball.create_docs(files) tarball end - def unpack(body, opts \\ []) do + def unpack_to_dir({:file, path}, opts \\ []) do repository = Keyword.get(opts, :repository, "UNKNOWN") package = Keyword.get(opts, :package, "UNKNOWN") version = Keyword.get(opts, :version, "UNKNOWN") - with {:ok, data} <- unzip(body), - {:ok, files} <- :erl_tar.extract({:binary, data}, [:memory]), - files = fix_paths(repository, package, version, files), - :ok <- check_version_dirs(files), - do: {:ok, files} - end - - defp unzip(data) when byte_size(data) > @compressed_max_size do - {:error, "too big"} - end - - defp unzip(data) do - stream = :zlib.open() + output_dir = Hexdocs.TmpDir.tmp_dir("docs") - try do - :zlib.inflateInit(stream, @zlib_magic) - uncompressed = unzip_inflate(stream, "", 0, :zlib.safeInflate(stream, data)) - :zlib.inflateEnd(stream) - uncompressed - catch - :error, :data_error -> - {:error, "invalid gzip"} - after - :zlib.close(stream) - end - end + case :hex_tarball.unpack_docs({:file, to_charlist(path)}, to_charlist(output_dir)) do + :ok -> + files = + output_dir + |> Path.join("**") + |> Path.wildcard(match_dot: true) + |> Enum.filter(&File.regular?(&1, raw: true)) + |> Enum.map(&Path.relative_to(&1, output_dir)) - defp unzip_inflate(_stream, _data, total, _) when total > @uncompressed_max_size do - {:error, "too big"} - end + files = fix_paths(repository, package, version, files) - defp unzip_inflate(stream, data, total, {:continue, uncompressed}) do - total = total + IO.iodata_length(uncompressed) - unzip_inflate(stream, [data | uncompressed], total, :zlib.safeInflate(stream, [])) - end + case check_version_dirs(files) do + :ok -> {:ok, output_dir, files} + {:error, _} = error -> error + end - defp unzip_inflate(_stream, data, total, {:finished, uncompressed}) do - if total + IO.iodata_length(uncompressed) > @uncompressed_max_size do - {:error, "too big"} - else - {:ok, IO.iodata_to_binary([data | uncompressed])} + {:error, reason} -> + {:error, inspect(reason)} end end defp check_version_dirs(files) do result = - Enum.all?(files, fn {path, _data} -> + Enum.all?(files, fn path -> first = Path.split(path) |> hd() Version.parse(first) == :error end) @@ -75,10 +50,10 @@ defmodule Hexdocs.Tar do end defp fix_paths(repository, package, version, files) do - Enum.flat_map(files, fn {path, data} -> + Enum.flat_map(files, fn path -> case safe_path(path) do {:ok, path} -> - [{path, data}] + [path] :error -> Logger.error("Unsafe path from #{repository}/#{package} #{version}: #{path}") diff --git a/lib/hexdocs/tmp_dir.ex b/lib/hexdocs/tmp_dir.ex new file mode 100644 index 0000000..e3d2969 --- /dev/null +++ b/lib/hexdocs/tmp_dir.ex @@ -0,0 +1,85 @@ +defmodule Hexdocs.TmpDir do + use GenServer + + @table __MODULE__ + + def start_link(opts \\ []) do + GenServer.start_link(__MODULE__, opts, name: __MODULE__) + end + + def tmp_file(prefix) do + path = path(prefix) + File.touch!(path) + track(path) + path + end + + def tmp_dir(prefix) do + path = path(prefix) + File.mkdir_p!(path) + track(path) + path + end + + defp path(prefix) do + random = Base.encode16(:crypto.strong_rand_bytes(4)) + Path.join(base_dir(), prefix <> "-" <> random) + end + + defp base_dir() do + dir = Application.get_env(:hexdocs, :tmp_dir) || System.tmp_dir!() + File.mkdir_p!(dir) + dir + end + + defp track(path) do + pid = self() + :ets.insert(@table, {pid, path}) + GenServer.cast(__MODULE__, {:monitor, pid}) + end + + @impl true + def init(_opts) do + Process.flag(:trap_exit, true) + :ets.new(@table, [:named_table, :duplicate_bag, :public]) + {:ok, %{monitors: MapSet.new()}} + end + + @impl true + def handle_cast({:monitor, pid}, state) do + if pid in state.monitors do + {:noreply, state} + else + Process.monitor(pid) + {:noreply, %{state | monitors: MapSet.put(state.monitors, pid)}} + end + end + + @impl true + def handle_info({:DOWN, _ref, :process, pid, _reason}, state) do + cleanup_pid(pid) + {:noreply, %{state | monitors: MapSet.delete(state.monitors, pid)}} + end + + @impl true + def terminate(_reason, _state) do + :ets.foldl( + fn {_pid, path}, :ok -> + File.rm_rf(path) + :ok + end, + :ok, + @table + ) + end + + defp cleanup_pid(pid) do + entries = :ets.lookup(@table, pid) + + Enum.each(entries, fn {_pid, path} -> + File.rm_rf(path) + end) + + :ets.delete(@table, pid) + end +end diff --git a/mix.exs b/mix.exs index c9145cf..8c68ccf 100644 --- a/mix.exs +++ b/mix.exs @@ -48,7 +48,7 @@ defmodule Hexdocs.MixProject do {:sentry, "~> 11.0"}, {:ssl_verify_fun, "~> 1.1", manager: :rebar3, override: true}, {:sweet_xml, "~> 0.7.0"}, - {:hex_core, "~> 0.11.0"}, + {:hex_core, "~> 0.15.0"}, {:mox, "~> 1.0", only: :test} ] end diff --git a/mix.lock b/mix.lock index 5437ed8..e900c2f 100644 --- a/mix.lock +++ b/mix.lock @@ -10,7 +10,7 @@ "finch": {:hex, :finch, "0.20.0", "5330aefb6b010f424dcbbc4615d914e9e3deae40095e73ab0c1bb0968933cadf", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2658131a74d051aabfcba936093c903b8e89da9a1b63e430bee62045fa9b2ee2"}, "gen_stage": {:hex, :gen_stage, "1.3.2", "7c77e5d1e97de2c6c2f78f306f463bca64bf2f4c3cdd606affc0100b89743b7b", [:mix], [], "hexpm", "0ffae547fa777b3ed889a6b9e1e64566217413d018cabd825f786e843ffe63e7"}, "goth": {:hex, :goth, "1.4.5", "ee37f96e3519bdecd603f20e7f10c758287088b6d77c0147cd5ee68cf224aade", [:mix], [{:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:jose, "~> 1.11", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "0fc2dce5bd710651ed179053d0300ce3a5d36afbdde11e500d57f05f398d5ed5"}, - "hex_core": {:hex, :hex_core, "0.11.0", "d1c6bbf2a4ee6b5f002bec6fa52b5080c53c8b63b7caf6eb88b943687547bff4", [:rebar3], [], "hexpm", "707893677a425491962a2db522f1d2b1f85f97ea27418b06f7929f1d30cde0b0"}, + "hex_core": {:hex, :hex_core, "0.15.0", "8eadc0ccb08e3742f2313073d04f39eaa7904617329039e9d3c402f5dd227673", [:rebar3], [], "hexpm", "c2093764c7af8ef0818c104fa141eba431e7be93f8374638c45c7037b26a52f8"}, "hpax": {:hex, :hpax, "1.0.3", "ed67ef51ad4df91e75cc6a1494f851850c0bd98ebc0be6e81b026e765ee535aa", [:mix], [], "hexpm", "8eab6e1cfa8d5918c2ce4ba43588e894af35dbd8e91e6e55c817bca5847df34a"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, "jose": {:hex, :jose, "1.11.10", "a903f5227417bd2a08c8a00a0cbcc458118be84480955e8d251297a425723f83", [:mix, :rebar3], [], "hexpm", "0d6cd36ff8ba174db29148fc112b5842186b68a90ce9fc2b3ec3afe76593e614"}, diff --git a/test/hexdocs/bucket_test.exs b/test/hexdocs/bucket_test.exs index b02c034..6f371f3 100644 --- a/test/hexdocs/bucket_test.exs +++ b/test/hexdocs/bucket_test.exs @@ -6,7 +6,8 @@ defmodule Hexdocs.BucketTest do test "upload", %{test: test} do version = Version.parse!("0.0.1") - Bucket.upload("buckettest", "#{test}", version, [], [{"index.html", "0.0.1"}]) + {dir, files} = create_files([{"index.html", "0.0.1"}]) + Bucket.upload("buckettest", "#{test}", version, [], dir, files) assert Store.get(@bucket, "buckettest/#{test}/0.0.1/index.html") == "0.0.1" assert Store.get(@bucket, "buckettest/#{test}/index.html") == "0.0.1" @@ -16,8 +17,10 @@ defmodule Hexdocs.BucketTest do first = Version.parse!("0.0.1") second = Version.parse!("0.0.2") - Bucket.upload("buckettest", "#{test}", first, [], [{"index.html", "0.0.1"}]) - Bucket.upload("buckettest", "#{test}", second, [first], [{"index.html", "0.0.2"}]) + {dir1, files1} = create_files([{"index.html", "0.0.1"}]) + {dir2, files2} = create_files([{"index.html", "0.0.2"}]) + Bucket.upload("buckettest", "#{test}", first, [], dir1, files1) + Bucket.upload("buckettest", "#{test}", second, [first], dir2, files2) assert Store.get(@bucket, "buckettest/#{test}/0.0.1/index.html") == "0.0.1" assert Store.get(@bucket, "buckettest/#{test}/0.0.2/index.html") == "0.0.2" @@ -28,8 +31,10 @@ defmodule Hexdocs.BucketTest do first = Version.parse!("0.0.1") second = Version.parse!("0.0.2") - Bucket.upload("buckettest", "#{test}", second, [], [{"index.html", "0.0.2"}]) - Bucket.upload("buckettest", "#{test}", first, [second], [{"index.html", "0.0.1"}]) + {dir1, files1} = create_files([{"index.html", "0.0.2"}]) + {dir2, files2} = create_files([{"index.html", "0.0.1"}]) + Bucket.upload("buckettest", "#{test}", second, [], dir1, files1) + Bucket.upload("buckettest", "#{test}", first, [second], dir2, files2) assert Store.get(@bucket, "buckettest/#{test}/0.0.1/index.html") == "0.0.1" assert Store.get(@bucket, "buckettest/#{test}/0.0.2/index.html") == "0.0.2" @@ -39,12 +44,11 @@ defmodule Hexdocs.BucketTest do test "overwrite docs", %{test: test} do version = Version.parse!("0.0.1") - Bucket.upload("buckettest", "#{test}", version, [], [ - {"index.html", "0.0.1"}, - {"remove.html", "remove"} - ]) + {dir1, files1} = create_files([{"index.html", "0.0.1"}, {"remove.html", "remove"}]) + Bucket.upload("buckettest", "#{test}", version, [], dir1, files1) - Bucket.upload("buckettest", "#{test}", version, [], [{"index.html", "updated"}]) + {dir2, files2} = create_files([{"index.html", "updated"}]) + Bucket.upload("buckettest", "#{test}", version, [], dir2, files2) assert Store.get(@bucket, "buckettest/#{test}/0.0.1/index.html") == "updated" assert Store.get(@bucket, "buckettest/#{test}/index.html") == "updated" @@ -57,13 +61,11 @@ defmodule Hexdocs.BucketTest do test = Atom.to_string(test) prefix_name = String.slice(test, -1000, String.length(test) - 1) - Bucket.upload("buckettest", "#{test}", version, [], [ - {"file2", ""} - ]) + {dir1, files1} = create_files([{"file2", ""}]) + Bucket.upload("buckettest", "#{test}", version, [], dir1, files1) - Bucket.upload("buckettest", "#{prefix_name}", version, [], [ - {"file1", ""} - ]) + {dir2, files2} = create_files([{"file1", ""}]) + Bucket.upload("buckettest", "#{prefix_name}", version, [], dir2, files2) assert Store.get(@bucket, "buckettest/#{prefix_name}/file1") assert Store.get(@bucket, "buckettest/#{prefix_name}/#{version}/file1") @@ -75,12 +77,11 @@ defmodule Hexdocs.BucketTest do first = Version.parse!("0.5.0") second = Version.parse!("1.0.0-beta") - Bucket.upload("buckettest", "#{test}", first, [], [ - {"index.html", "0.5.0"}, - {"dont_remove.html", "dont remove"} - ]) + {dir1, files1} = create_files([{"index.html", "0.5.0"}, {"dont_remove.html", "dont remove"}]) + Bucket.upload("buckettest", "#{test}", first, [], dir1, files1) - Bucket.upload("buckettest", "#{test}", second, [first], [{"index.html", "1.0.0-beta"}]) + {dir2, files2} = create_files([{"index.html", "1.0.0-beta"}]) + Bucket.upload("buckettest", "#{test}", second, [first], dir2, files2) assert Store.get(@bucket, "buckettest/#{test}/0.5.0/index.html") == "0.5.0" assert Store.get(@bucket, "buckettest/#{test}/0.5.0/dont_remove.html") == "dont remove" @@ -94,9 +95,12 @@ defmodule Hexdocs.BucketTest do second = Version.parse!("1.0.0-beta") third = Version.parse!("0.2.0") - Bucket.upload("buckettest", "#{test}", first, [], [{"index.html", "0.1.0"}]) - Bucket.upload("buckettest", "#{test}", second, [first], [{"index.html", "1.0.0-beta"}]) - Bucket.upload("buckettest", "#{test}", third, [first, second], [{"index.html", "0.2.0"}]) + {dir1, files1} = create_files([{"index.html", "0.1.0"}]) + {dir2, files2} = create_files([{"index.html", "1.0.0-beta"}]) + {dir3, files3} = create_files([{"index.html", "0.2.0"}]) + Bucket.upload("buckettest", "#{test}", first, [], dir1, files1) + Bucket.upload("buckettest", "#{test}", second, [first], dir2, files2) + Bucket.upload("buckettest", "#{test}", third, [first, second], dir3, files3) assert Store.get(@bucket, "buckettest/#{test}/0.1.0/index.html") == "0.1.0" assert Store.get(@bucket, "buckettest/#{test}/1.0.0-beta/index.html") == "1.0.0-beta" @@ -108,8 +112,10 @@ defmodule Hexdocs.BucketTest do first = Version.parse!("1.0.0-beta") second = Version.parse!("2.0.0-beta") - Bucket.upload("buckettest", "#{test}", first, [], [{"index.html", "1.0.0-beta"}]) - Bucket.upload("buckettest", "#{test}", second, [first], [{"index.html", "2.0.0-beta"}]) + {dir1, files1} = create_files([{"index.html", "1.0.0-beta"}]) + {dir2, files2} = create_files([{"index.html", "2.0.0-beta"}]) + Bucket.upload("buckettest", "#{test}", first, [], dir1, files1) + Bucket.upload("buckettest", "#{test}", second, [first], dir2, files2) assert Store.get(@bucket, "buckettest/#{test}/1.0.0-beta/index.html") == "1.0.0-beta" assert Store.get(@bucket, "buckettest/#{test}/2.0.0-beta/index.html") == "2.0.0-beta" @@ -120,18 +126,16 @@ defmodule Hexdocs.BucketTest do version = "1.0.0" all_versions = [] - Bucket.upload("buckettest", "#{test}", Version.parse!(version), all_versions, [ - {"index.html", version} - ]) + {dir, files} = create_files([{"index.html", version}]) + Bucket.upload("buckettest", "#{test}", Version.parse!(version), all_versions, dir, files) assert Store.get(@bucket, "buckettest/#{test}/docs_config.js") =~ "1.0.0" version = "2.0.0" all_versions = [Version.parse!("1.0.0")] - Bucket.upload("buckettest", "#{test}", Version.parse!(version), all_versions, [ - {"index.html", version} - ]) + {dir, files} = create_files([{"index.html", version}]) + Bucket.upload("buckettest", "#{test}", Version.parse!(version), all_versions, dir, files) assert Store.get(@bucket, "buckettest/#{test}/docs_config.js") =~ "1.0.0" assert Store.get(@bucket, "buckettest/#{test}/docs_config.js") =~ "2.0.0" @@ -139,12 +143,25 @@ defmodule Hexdocs.BucketTest do version = "1.1.0" all_versions = [Version.parse!("1.0.0"), Version.parse!("2.0.0")] - Bucket.upload("buckettest", "#{test}", Version.parse!(version), all_versions, [ - {"index.html", version} - ]) + {dir, files} = create_files([{"index.html", version}]) + Bucket.upload("buckettest", "#{test}", Version.parse!(version), all_versions, dir, files) assert Store.get(@bucket, "buckettest/#{test}/docs_config.js") =~ "1.0.0" assert Store.get(@bucket, "buckettest/#{test}/docs_config.js") =~ "1.1.0" assert Store.get(@bucket, "buckettest/#{test}/docs_config.js") =~ "2.0.0" end + + defp create_files(file_list) do + dir = Hexdocs.TmpDir.tmp_dir("test") + + files = + Enum.map(file_list, fn {path, content} -> + full_path = Path.join(dir, path) + File.mkdir_p!(Path.dirname(full_path)) + File.write!(full_path, content) + path + end) + + {dir, files} + end end diff --git a/test/hexdocs/debouncer_test.exs b/test/hexdocs/debouncer_test.exs index 4852816..3138f5b 100644 --- a/test/hexdocs/debouncer_test.exs +++ b/test/hexdocs/debouncer_test.exs @@ -3,7 +3,7 @@ defmodule Hexdocs.DebouncerTest do alias Hexdocs.Debouncer @short_grace_time 10 - @grace_time 100 + @grace_time 200 @long_grace_time 10000 setup do diff --git a/test/hexdocs/queue_test.exs b/test/hexdocs/queue_test.exs index 056e4e3..ac6903c 100644 --- a/test/hexdocs/queue_test.exs +++ b/test/hexdocs/queue_test.exs @@ -80,8 +80,7 @@ defmodule Hexdocs.QueueTest do assert Store.get(@public_bucket, "package_names.csv") == "package1\npackage2\n" end - @tag :capture_log - test "safe paths", %{test: test} do + test "unsafe paths", %{test: test} do Mox.expect(HexpmMock, :get_package, fn repo, package -> assert repo == "hexpm" assert package == "#{test}" @@ -99,13 +98,15 @@ defmodule Hexdocs.QueueTest do key = "docs/#{test}-1.0.0.tar.gz" Store.put!(:repo_bucket, key, tar) - ref = Broadway.test_message(Hexdocs.Queue, put_message(key)) - assert_receive {:ack, ^ref, [_], []} + log = + ExUnit.CaptureLog.capture_log(fn -> + ref = Broadway.test_message(Hexdocs.Queue, put_message(key)) + assert_receive {:ack, ^ref, [_], []} + end) - assert ls(@public_bucket, "#{test}/1.0.0/") == [ - "bar.html", - "dir/foo.html" - ] + assert log =~ "Failed unpack" + assert log =~ "unsafe_path" + assert ls(@public_bucket, "#{test}/1.0.0/") == [] end test "overwrite main docs with newer versions", %{test: test} do diff --git a/test/hexdocs/tar_test.exs b/test/hexdocs/tar_test.exs index 408d525..299e1ff 100644 --- a/test/hexdocs/tar_test.exs +++ b/test/hexdocs/tar_test.exs @@ -2,26 +2,36 @@ defmodule Hexdocs.TarTest do use ExUnit.Case, async: true alias Hexdocs.Tar - test "unzip tar" do + test "unpack_to_dir" do blob = Tar.create([{"index.html", "contents"}, {"foo.bar", "contents"}]) + path = Hexdocs.TmpDir.tmp_file("test-tarball") + File.write!(path, blob) - assert {:ok, files} = Tar.unpack(blob) + assert {:ok, dir, files} = Tar.unpack_to_dir({:file, path}) + assert File.dir?(dir) assert length(files) == 2 - assert {"index.html", "contents"} in files - assert {"foo.bar", "contents"} in files + assert "index.html" in files + assert "foo.bar" in files + assert File.read!(Path.join(dir, "index.html")) == "contents" + assert File.read!(Path.join(dir, "foo.bar")) == "contents" end test "invalid gzip" do - assert Tar.unpack("") == {:error, "invalid gzip"} + path = Hexdocs.TmpDir.tmp_file("test-tarball") + File.write!(path, "") + assert {:error, _} = Tar.unpack_to_dir({:file, path}) end test "do not allow root files/directories with version names" do reason = "root file or directory name not allowed to match a semver version" blob = Tar.create([{"1.0.0", "contents"}]) - assert Tar.unpack(blob) == {:error, reason} + path = Hexdocs.TmpDir.tmp_file("test-tarball") + File.write!(path, blob) + assert Tar.unpack_to_dir({:file, path}) == {:error, reason} blob = Tar.create([{"1.0.0/index.html", "contents"}]) - assert Tar.unpack(blob) == {:error, reason} + File.write!(path, blob) + assert Tar.unpack_to_dir({:file, path}) == {:error, reason} end end diff --git a/test/hexdocs/tmp_dir_test.exs b/test/hexdocs/tmp_dir_test.exs new file mode 100644 index 0000000..81034d3 --- /dev/null +++ b/test/hexdocs/tmp_dir_test.exs @@ -0,0 +1,79 @@ +defmodule Hexdocs.TmpDirTest do + use ExUnit.Case, async: true + + test "tmp_file/1 creates a file" do + path = Hexdocs.TmpDir.tmp_file("test") + assert File.exists?(path) + assert File.regular?(path) + end + + test "tmp_dir/1 creates a directory" do + path = Hexdocs.TmpDir.tmp_dir("test") + assert File.dir?(path) + end + + test "cleanup on normal process exit" do + test_pid = self() + + Task.start(fn -> + file = Hexdocs.TmpDir.tmp_file("test") + dir = Hexdocs.TmpDir.tmp_dir("test") + send(test_pid, {:paths, file, dir}) + end) + + assert_receive {:paths, file, dir} + Process.sleep(100) + + refute File.exists?(file) + refute File.exists?(dir) + end + + @tag :capture_log + test "cleanup on process crash" do + test_pid = self() + + Task.start(fn -> + file = Hexdocs.TmpDir.tmp_file("test") + dir = Hexdocs.TmpDir.tmp_dir("test") + send(test_pid, {:paths, file, dir}) + raise "crash" + end) + + assert_receive {:paths, file, dir} + Process.sleep(100) + + refute File.exists?(file) + refute File.exists?(dir) + end + + test "multiple paths for one process" do + test_pid = self() + + Task.start(fn -> + paths = + for i <- 1..5 do + file = Hexdocs.TmpDir.tmp_file("test-#{i}") + dir = Hexdocs.TmpDir.tmp_dir("test-#{i}") + {file, dir} + end + + send(test_pid, {:paths, paths}) + end) + + assert_receive {:paths, paths} + Process.sleep(100) + + for {file, dir} <- paths do + refute File.exists?(file) + refute File.exists?(dir) + end + end + + test "paths persist while process is alive" do + file = Hexdocs.TmpDir.tmp_file("test") + dir = Hexdocs.TmpDir.tmp_dir("test") + + assert File.exists?(file) + assert File.dir?(dir) + end +end From 828f148e42878ab749c6d65929b2bfd80b3c4506 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Tue, 10 Mar 2026 00:18:58 +0100 Subject: [PATCH 2/2] Fix flaky TmpDir cleanup tests Use GenServer.call instead of cast in track/1 so the monitor is set up synchronously before the caller can exit. Replace Process.sleep with deterministic waiting: monitor the task, wait for :DOWN, then sync with the GenServer via :sys.get_state to ensure cleanup has been processed. --- lib/hexdocs/tmp_dir.ex | 8 ++--- test/hexdocs/tmp_dir_test.exs | 56 +++++++++++++++++++++-------------- 2 files changed, 37 insertions(+), 27 deletions(-) diff --git a/lib/hexdocs/tmp_dir.ex b/lib/hexdocs/tmp_dir.ex index e3d2969..0a861a1 100644 --- a/lib/hexdocs/tmp_dir.ex +++ b/lib/hexdocs/tmp_dir.ex @@ -35,7 +35,7 @@ defmodule Hexdocs.TmpDir do defp track(path) do pid = self() :ets.insert(@table, {pid, path}) - GenServer.cast(__MODULE__, {:monitor, pid}) + GenServer.call(__MODULE__, {:monitor, pid}) end @impl true @@ -46,12 +46,12 @@ defmodule Hexdocs.TmpDir do end @impl true - def handle_cast({:monitor, pid}, state) do + def handle_call({:monitor, pid}, _from, state) do if pid in state.monitors do - {:noreply, state} + {:reply, :ok, state} else Process.monitor(pid) - {:noreply, %{state | monitors: MapSet.put(state.monitors, pid)}} + {:reply, :ok, %{state | monitors: MapSet.put(state.monitors, pid)}} end end diff --git a/test/hexdocs/tmp_dir_test.exs b/test/hexdocs/tmp_dir_test.exs index 81034d3..5e22f8a 100644 --- a/test/hexdocs/tmp_dir_test.exs +++ b/test/hexdocs/tmp_dir_test.exs @@ -15,14 +15,15 @@ defmodule Hexdocs.TmpDirTest do test "cleanup on normal process exit" do test_pid = self() - Task.start(fn -> - file = Hexdocs.TmpDir.tmp_file("test") - dir = Hexdocs.TmpDir.tmp_dir("test") - send(test_pid, {:paths, file, dir}) - end) + {:ok, task_pid} = + Task.start(fn -> + file = Hexdocs.TmpDir.tmp_file("test") + dir = Hexdocs.TmpDir.tmp_dir("test") + send(test_pid, {:paths, file, dir}) + end) assert_receive {:paths, file, dir} - Process.sleep(100) + wait_for_cleanup(task_pid) refute File.exists?(file) refute File.exists?(dir) @@ -32,15 +33,16 @@ defmodule Hexdocs.TmpDirTest do test "cleanup on process crash" do test_pid = self() - Task.start(fn -> - file = Hexdocs.TmpDir.tmp_file("test") - dir = Hexdocs.TmpDir.tmp_dir("test") - send(test_pid, {:paths, file, dir}) - raise "crash" - end) + {:ok, task_pid} = + Task.start(fn -> + file = Hexdocs.TmpDir.tmp_file("test") + dir = Hexdocs.TmpDir.tmp_dir("test") + send(test_pid, {:paths, file, dir}) + raise "crash" + end) assert_receive {:paths, file, dir} - Process.sleep(100) + wait_for_cleanup(task_pid) refute File.exists?(file) refute File.exists?(dir) @@ -49,19 +51,20 @@ defmodule Hexdocs.TmpDirTest do test "multiple paths for one process" do test_pid = self() - Task.start(fn -> - paths = - for i <- 1..5 do - file = Hexdocs.TmpDir.tmp_file("test-#{i}") - dir = Hexdocs.TmpDir.tmp_dir("test-#{i}") - {file, dir} - end + {:ok, task_pid} = + Task.start(fn -> + paths = + for i <- 1..5 do + file = Hexdocs.TmpDir.tmp_file("test-#{i}") + dir = Hexdocs.TmpDir.tmp_dir("test-#{i}") + {file, dir} + end - send(test_pid, {:paths, paths}) - end) + send(test_pid, {:paths, paths}) + end) assert_receive {:paths, paths} - Process.sleep(100) + wait_for_cleanup(task_pid) for {file, dir} <- paths do refute File.exists?(file) @@ -76,4 +79,11 @@ defmodule Hexdocs.TmpDirTest do assert File.exists?(file) assert File.dir?(dir) end + + defp wait_for_cleanup(task_pid) do + ref = Process.monitor(task_pid) + assert_receive {:DOWN, ^ref, :process, ^task_pid, _} + # Sync with the GenServer to ensure the :DOWN cleanup has been processed + :sys.get_state(Hexdocs.TmpDir) + end end