From 1fa7fb0ea9bbd3d4a6d56980a844b9ac33e91faf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Mon, 14 Sep 2026 00:39:48 +0200 Subject: [PATCH 01/13] Trim only SP and HTAB from Content-Length values String.trim_trailing/1 also strips Unicode whitespace, so values such as "5" followed by VT, FF, NBSP, NEL or U+3000 were accepted. RFC 9110 section 8.6 allows only digits, with optional whitespace around the field value. --- lib/mint/http1/parse.ex | 16 +++++++++++++++- test/mint/http1/parse_test.exs | 17 +++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/lib/mint/http1/parse.ex b/lib/mint/http1/parse.ex index f4833bb6..6a61cc53 100644 --- a/lib/mint/http1/parse.ex +++ b/lib/mint/http1/parse.ex @@ -102,7 +102,7 @@ defmodule Mint.HTTP1.Parse do defp chunk_extensions(_data, _state), do: :error def content_length_header(string) do - trimmed = String.trim_trailing(string) + trimmed = trim_trailing_whitespace(string) if ParsingTools.only_digits?(trimmed) do {:ok, String.to_integer(trimmed)} @@ -111,6 +111,20 @@ defmodule Mint.HTTP1.Parse do end end + defp trim_trailing_whitespace(<<>>), do: <<>> + + defp trim_trailing_whitespace(string) do + prefix_size = byte_size(string) - 1 + + case string do + <> when is_whitespace(char) -> + trim_trailing_whitespace(prefix) + + _other -> + string + end + end + def connection_header(string) do split_into_downcase_tokens(string) end diff --git a/test/mint/http1/parse_test.exs b/test/mint/http1/parse_test.exs index 2a680b46..d24ebcad 100644 --- a/test/mint/http1/parse_test.exs +++ b/test/mint/http1/parse_test.exs @@ -97,6 +97,23 @@ defmodule Mint.HTTP1.ParseTest do assert content_length_header("0") == {:ok, 0} assert content_length_header("100") == {:ok, 100} assert content_length_header("200 ") == {:ok, 200} + assert content_length_header("200\t") == {:ok, 200} + assert content_length_header("200 \t ") == {:ok, 200} + + assert content_length_header("200\v") == + {:error, {:invalid_content_length_header, "200\v"}} + + assert content_length_header("200\f") == + {:error, {:invalid_content_length_header, "200\f"}} + + assert content_length_header("200\u00A0") == + {:error, {:invalid_content_length_header, "200\u00A0"}} + + assert content_length_header("200\u0085") == + {:error, {:invalid_content_length_header, "200\u0085"}} + + assert content_length_header("200\u3000") == + {:error, {:invalid_content_length_header, "200\u3000"}} assert content_length_header("foo") == {:error, {:invalid_content_length_header, "foo"}} From 4538c6814f68983fd411fe2d6b4e17ccd8353ebd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Mon, 14 Sep 2026 00:41:57 +0200 Subject: [PATCH 02/13] Unfold obsolete line folding in HTTP/1 header values Folded header and trailer values were delivered with the CRLF and leading whitespace intact. RFC 9112 section 5.2 requires user agents to replace each obs-fold with a space before interpreting the value. With :stream_headers a header is emitted before its continuation line can be seen, so folds are rejected with :invalid_header in that mode. --- .dialyzer_ignore | 2 +- lib/mint/http1.ex | 42 ++++++++++++++++++++++++----------- lib/mint/http1/response.ex | 22 ++++++++++++++++++ test/mint/http1/conn_test.exs | 40 +++++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+), 14 deletions(-) diff --git a/.dialyzer_ignore b/.dialyzer_ignore index 00387569..26a72ed1 100644 --- a/.dialyzer_ignore +++ b/.dialyzer_ignore @@ -1,5 +1,5 @@ lib/mint/tunnel_proxy.ex:50 -lib/mint/http1.ex:1069 +lib/mint/http1.ex:1085 lib/mint/unsafe_proxy.ex:173 lib/mint/unsafe_proxy.ex:198 test/support diff --git a/lib/mint/http1.ex b/lib/mint/http1.ex index 96ca7c32..e1046411 100644 --- a/lib/mint/http1.ex +++ b/lib/mint/http1.ex @@ -159,7 +159,9 @@ defmodule Mint.HTTP1 do * `:stream_headers` - (`t:boolean/0`) if set to `true`, response headers and trailer headers will be emitted as they are parsed, rather than buffered until the complete header section is received. When enabled, you may receive multiple `{:headers, ref, headers}` responses - for a single request. Defaults to `false`. *Available since v1.10.0*. + for a single request, and header values that use obsolete line folding are rejected + with an `:invalid_header` error instead of being unfolded. Defaults to `false`. + *Available since v1.10.0*. """ @spec connect(Types.scheme(), Types.address(), :inet.port_number(), keyword()) :: @@ -1011,25 +1013,39 @@ defmodule Mint.HTTP1 do end end - defp decode_header(data, false = _stream_headers), do: Response.decode_header(data) + defp decode_header(data, false = _stream_headers) do + case Response.decode_header(data) do + {:ok, {name, value}, rest} -> {:ok, {name, Response.replace_obs_fold(value)}, rest} + other -> other + end + end defp decode_header(data, true = _stream_headers) do # By default, :erlang.decode_packet/3 asks for more data when a packet # containing a full header ends with a line feed (likely to handle line # folding). If we get a :more response on a packet that ends with a line # feed, we append a sentinel byte and attempt to decode again. - with :more <- Response.decode_header(data) do - data_size = byte_size(data) - - case data do - <<_::binary-size(^data_size - 1), ?\n>> -> - with {:ok, {name, value}, <<0>>} <- Response.decode_header(<>) do - {:ok, {name, value}, ""} - end - - _ -> - :more + # + # Headers are emitted before the next line is seen, so a folded + # continuation line can't be joined to its header. Folds are rejected. + result = + with :more <- Response.decode_header(data) do + data_size = byte_size(data) + + case data do + <<_::binary-size(^data_size - 1), ?\n>> -> + with {:ok, {name, value}, <<0>>} <- Response.decode_header(<>) do + {:ok, {name, value}, ""} + end + + _ -> + :more + end end + + case result do + {:ok, {_name, value}, _rest} = ok -> if Response.obs_fold?(value), do: :error, else: ok + other -> other end end diff --git a/lib/mint/http1/response.ex b/lib/mint/http1/response.ex index a2018c40..02ac0a42 100644 --- a/lib/mint/http1/response.ex +++ b/lib/mint/http1/response.ex @@ -38,6 +38,28 @@ defmodule Mint.HTTP1.Response do end end + def obs_fold?(value), do: :binary.match(value, "\n") != :nomatch + + # RFC 9112 5.2: a user agent must replace each received obs-fold with one or + # more SP octets before interpreting the field value. + def replace_obs_fold(value) do + if obs_fold?(value), do: replace_obs_fold(value, <<>>), else: value + end + + defp replace_obs_fold(<<"\r\n", rest::binary>>, acc), + do: replace_obs_fold(skip_whitespace(rest), <>) + + defp replace_obs_fold(<<"\n", rest::binary>>, acc), + do: replace_obs_fold(skip_whitespace(rest), <>) + + defp replace_obs_fold(<>, acc), + do: replace_obs_fold(rest, <>) + + defp replace_obs_fold(<<>>, acc), do: acc + + defp skip_whitespace(<>) when char in ~c"\s\t", do: skip_whitespace(rest) + defp skip_whitespace(rest), do: rest + defp header_name(atom) when is_atom(atom), do: atom |> Atom.to_string() |> header_name() defp header_name(binary) when is_binary(binary), do: Headers.lower_raw(binary) end diff --git a/test/mint/http1/conn_test.exs b/test/mint/http1/conn_test.exs index 975c1a14..e6e3e42f 100644 --- a/test/mint/http1/conn_test.exs +++ b/test/mint/http1/conn_test.exs @@ -105,6 +105,26 @@ defmodule Mint.HTTP1Test do HTTP1.stream(conn, {:tcp, conn.socket, "foo: bar\r\n"}) end + test "obsolete line folding in header values is replaced with a space", %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + + response = + "HTTP/1.1 200 OK\r\nFoo: bar\r\n baz\r\nBar: one\r\n\t two\n three\r\n" <> + "transfer-encoding: chunked\r\n\r\n0\r\nMy-Trailer: a\r\n b\r\n\r\n" + + assert {:ok, _conn, [status, headers, trailers, done]} = + HTTP1.stream(conn, {:tcp, conn.socket, response}) + + assert status == {:status, ref, 200} + + assert headers == + {:headers, ref, + [{"foo", "bar baz"}, {"bar", "one two three"}, {"transfer-encoding", "chunked"}]} + + assert trailers == {:headers, ref, [{"my-trailer", "a b"}]} + assert done == {:done, ref} + end + test "status and headers", %{conn: conn} do {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) @@ -1515,6 +1535,26 @@ defmodule Mint.HTTP1Test do assert {:headers, ^ref, [{"qux", "Quux"}]} = headers2 end + test "rejects obsolete line folding in header values", %{conn: conn} do + {:ok, conn, _ref} = HTTP1.request(conn, "GET", "/", [], nil) + + assert {:error, _conn, %HTTPError{reason: :invalid_header}, [_status]} = + HTTP1.stream( + conn, + {:tcp, conn.socket, "HTTP/1.1 200 OK\r\nFoo: bar\r\n baz\r\n\r\n"} + ) + end + + test "rejects a folded continuation line arriving after its header", %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + + assert {:ok, conn, [_status, {:headers, ^ref, [{"foo", "bar"}]}]} = + HTTP1.stream(conn, {:tcp, conn.socket, "HTTP/1.1 200 OK\r\nFoo: bar\r\n"}) + + assert {:error, _conn, %HTTPError{reason: :invalid_header}, []} = + HTTP1.stream(conn, {:tcp, conn.socket, " baz\r\n\r\n"}) + end + test "emits multiple headers from one packet together", %{conn: conn} do {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) From 1490be52b8392b8d541a97650839f156ac8e4c07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Mon, 14 Sep 2026 00:42:12 +0200 Subject: [PATCH 03/13] Validate HTTP/1 status line version, status code and reason phrase :erlang.decode_packet/3 accepts any major version, status codes with two or four digits, and reason phrases containing control bytes. Only HTTP/1.x with a single-digit minor version, three-digit status codes, and reason phrases made of HTAB, SP, VCHAR and obs-text are accepted. --- lib/mint/http1/response.ex | 17 +++++++++++++++-- test/mint/http1/conn_test.exs | 29 +++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/lib/mint/http1/response.ex b/lib/mint/http1/response.ex index 02ac0a42..7c1b311d 100644 --- a/lib/mint/http1/response.ex +++ b/lib/mint/http1/response.ex @@ -5,8 +5,13 @@ defmodule Mint.HTTP1.Response do def decode_status_line(binary) do case :erlang.decode_packet(:http_bin, binary, []) do - {:ok, {:http_response, version, status, reason}, rest} -> - {:ok, {version, status, reason}, rest} + {:ok, {:http_response, {1, minor} = version, status, reason}, rest} + when minor in 0..9 and status in 100..999 -> + if valid_reason_phrase?(reason) do + {:ok, {version, status, reason}, rest} + else + :error + end {:ok, _other, _rest} -> :error @@ -38,6 +43,14 @@ defmodule Mint.HTTP1.Response do end end + # RFC 9112 4: reason-phrase = 1*( HTAB / SP / VCHAR / obs-text ) + defp valid_reason_phrase?(<>) + when char == ?\t or char in 32..126 or char in 128..255, + do: valid_reason_phrase?(rest) + + defp valid_reason_phrase?(<<>>), do: true + defp valid_reason_phrase?(_other), do: false + def obs_fold?(value), do: :binary.match(value, "\n") != :nomatch # RFC 9112 5.2: a user agent must replace each received obs-fold with one or diff --git a/test/mint/http1/conn_test.exs b/test/mint/http1/conn_test.exs index e6e3e42f..75a68e29 100644 --- a/test/mint/http1/conn_test.exs +++ b/test/mint/http1/conn_test.exs @@ -50,6 +50,35 @@ defmodule Mint.HTTP1Test do HTTP1.stream(conn, {:tcp, conn.socket, " 200 OK\r\n"}) end + test "status line with an empty reason phrase", %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + + assert {:ok, _conn, [{:status, ^ref, 204}]} = + HTTP1.stream(conn, {:tcp, conn.socket, "HTTP/1.1 204\r\n"}) + end + + test "invalid status lines", %{port: port} do + lines = [ + "HTTP/1.1 2000 OK\r\n", + "HTTP/1.1 99 OK\r\n", + "HTTP/2.0 200 OK\r\n", + "HTTP/0.9 200 OK\r\n", + "HTTP/1.10 200 OK\r\n", + "HTTP/1.1 200 O\0K\r\n", + "HTTP/1.1 200 OK\r\r\n", + "HTTP/1.1 200OK\r\n" + ] + + for line <- lines do + assert {:ok, conn} = HTTP1.connect(:http, "localhost", port) + {:ok, conn, _ref} = HTTP1.request(conn, "GET", "/", [], nil) + + assert {:error, _conn, %HTTPError{reason: :invalid_status_line}, []} = + HTTP1.stream(conn, {:tcp, conn.socket, line}), + "expected #{inspect(line)} to be rejected" + end + end + test "limits an incomplete response status line", %{port: port} do assert {:ok, conn} = HTTP1.connect(:http, "localhost", port, max_header_list_size: 64) {:ok, conn, _ref} = HTTP1.request(conn, "GET", "/", [], nil) From 9c0d178a3e6705d810444f80333afc89d14938d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Mon, 14 Sep 2026 01:19:06 +0200 Subject: [PATCH 04/13] Reject HTTP/1 header values containing control characters Values with CR, NUL, DEL or other control bytes were delivered verbatim in headers and trailers. RFC 9110 5.5 allows only HTAB, SP, VCHAR and obs-text in a field value and requires recipients to reject or replace CR, LF and NUL. The response now fails with :invalid_header or :invalid_trailer_header, the same check the request encoder applies to outgoing values. --- .dialyzer_ignore | 2 +- lib/mint/http1.ex | 15 +++++++++++--- lib/mint/http1/response.ex | 11 +++++++++++ test/mint/http1/conn_test.exs | 37 +++++++++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 4 deletions(-) diff --git a/.dialyzer_ignore b/.dialyzer_ignore index 26a72ed1..316442b1 100644 --- a/.dialyzer_ignore +++ b/.dialyzer_ignore @@ -1,5 +1,5 @@ lib/mint/tunnel_proxy.ex:50 -lib/mint/http1.ex:1085 +lib/mint/http1.ex:1094 lib/mint/unsafe_proxy.ex:173 lib/mint/unsafe_proxy.ex:198 test/support diff --git a/lib/mint/http1.ex b/lib/mint/http1.ex index e1046411..de39dce6 100644 --- a/lib/mint/http1.ex +++ b/lib/mint/http1.ex @@ -1015,7 +1015,7 @@ defmodule Mint.HTTP1 do defp decode_header(data, false = _stream_headers) do case Response.decode_header(data) do - {:ok, {name, value}, rest} -> {:ok, {name, Response.replace_obs_fold(value)}, rest} + {:ok, {name, value}, rest} -> validate_header(name, Response.replace_obs_fold(value), rest) other -> other end end @@ -1027,7 +1027,8 @@ defmodule Mint.HTTP1 do # feed, we append a sentinel byte and attempt to decode again. # # Headers are emitted before the next line is seen, so a folded - # continuation line can't be joined to its header. Folds are rejected. + # continuation line can't be joined to its header. Folds are rejected by + # the value validation since they contain a line feed. result = with :more <- Response.decode_header(data) do data_size = byte_size(data) @@ -1044,11 +1045,19 @@ defmodule Mint.HTTP1 do end case result do - {:ok, {_name, value}, _rest} = ok -> if Response.obs_fold?(value), do: :error, else: ok + {:ok, {name, value}, rest} -> validate_header(name, value, rest) other -> other end end + defp validate_header(name, value, rest) do + if Response.valid_header_value?(value) do + {:ok, {name, value}, rest} + else + :error + end + end + defp next_request(%{request: nil} = conn, data, responses) do # TODO: Figure out if we should keep buffering even though there are no # requests in flight diff --git a/lib/mint/http1/response.ex b/lib/mint/http1/response.ex index 7c1b311d..7c02e9f9 100644 --- a/lib/mint/http1/response.ex +++ b/lib/mint/http1/response.ex @@ -53,6 +53,17 @@ defmodule Mint.HTTP1.Response do def obs_fold?(value), do: :binary.match(value, "\n") != :nomatch + # RFC 9110 5.5: field-value = *field-content, field-vchar = VCHAR / obs-text, + # with HTAB and SP allowed between field-vchars. A recipient of CR, LF or NUL + # must reject the message or replace them, and other control characters are + # not allowed at all. + def valid_header_value?(<>) + when char == ?\t or char in 32..126 or char in 128..255, + do: valid_header_value?(rest) + + def valid_header_value?(<<>>), do: true + def valid_header_value?(_other), do: false + # RFC 9112 5.2: a user agent must replace each received obs-fold with one or # more SP octets before interpreting the field value. def replace_obs_fold(value) do diff --git a/test/mint/http1/conn_test.exs b/test/mint/http1/conn_test.exs index 75a68e29..2e7d655f 100644 --- a/test/mint/http1/conn_test.exs +++ b/test/mint/http1/conn_test.exs @@ -134,6 +134,43 @@ defmodule Mint.HTTP1Test do HTTP1.stream(conn, {:tcp, conn.socket, "foo: bar\r\n"}) end + test "header values with control characters are rejected", %{port: port} do + for value <- ["b\rar", "b\0ar", "b\x7Far", "b\x01ar"], stream_headers <- [false, true] do + assert {:ok, conn} = HTTP1.connect(:http, "localhost", port, stream_headers: stream_headers) + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + response = "HTTP/1.1 200 OK\r\nfoo: " <> value <> "\r\ncontent-length: 0\r\n\r\n" + + assert {:error, conn, %HTTPError{reason: :invalid_header}, [{:status, ^ref, 200}]} = + HTTP1.stream(conn, {:tcp, conn.socket, response}) + + assert_closed_and_released(conn) + end + end + + test "trailer values with control characters are rejected", %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + + response = + "HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n" <> + "1\r\nX\r\n0\r\nfoo: b\0ar\r\n\r\n" + + assert {:error, conn, %HTTPError{reason: :invalid_trailer_header}, responses} = + HTTP1.stream(conn, {:tcp, conn.socket, response}) + + assert [{:status, ^ref, 200}, {:headers, ^ref, _}, {:data, ^ref, "X"}] = responses + assert_closed_and_released(conn) + end + + test "header values with obs-text are accepted", %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + response = "HTTP/1.1 200 OK\r\nfoo: b\xC3\xA4r\r\ncontent-length: 0\r\n\r\n" + + assert {:ok, _conn, [{:status, ^ref, 200}, {:headers, ^ref, headers}, {:done, ^ref}]} = + HTTP1.stream(conn, {:tcp, conn.socket, response}) + + assert headers == [{"foo", "bär"}, {"content-length", "0"}] + end + test "obsolete line folding in header values is replaced with a space", %{conn: conn} do {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) From 6c6d7570fe177c3d134a948de771bf82136b312b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Mon, 14 Sep 2026 01:20:29 +0200 Subject: [PATCH 05/13] Reject HTTP/1 header and trailer lines with an empty field name :erlang.decode_packet/3 accepts a line such as ": bar" and returns an empty name, which was delivered to callers as {"", "bar"}. RFC 9110 5.1 defines field-name as a token of at least one character. --- lib/mint/http1.ex | 2 +- lib/mint/http1/response.ex | 10 ++++++++++ test/mint/http1/conn_test.exs | 27 +++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/lib/mint/http1.ex b/lib/mint/http1.ex index de39dce6..0f65a7fd 100644 --- a/lib/mint/http1.ex +++ b/lib/mint/http1.ex @@ -1051,7 +1051,7 @@ defmodule Mint.HTTP1 do end defp validate_header(name, value, rest) do - if Response.valid_header_value?(value) do + if Response.valid_header_name?(name) and Response.valid_header_value?(value) do {:ok, {name, value}, rest} else :error diff --git a/lib/mint/http1/response.ex b/lib/mint/http1/response.ex index 7c02e9f9..d13dd2b2 100644 --- a/lib/mint/http1/response.ex +++ b/lib/mint/http1/response.ex @@ -1,6 +1,8 @@ defmodule Mint.HTTP1.Response do @moduledoc false + import Mint.HTTP1.Parse + alias Mint.Core.Headers def decode_status_line(binary) do @@ -53,6 +55,14 @@ defmodule Mint.HTTP1.Response do def obs_fold?(value), do: :binary.match(value, "\n") != :nomatch + # RFC 9110 5.1: field-name = token, token = 1*tchar + def valid_header_name?(<<>>), do: false + def valid_header_name?(name), do: tchars?(name) + + defp tchars?(<>) when is_tchar(char), do: tchars?(rest) + defp tchars?(<<>>), do: true + defp tchars?(_other), do: false + # RFC 9110 5.5: field-value = *field-content, field-vchar = VCHAR / obs-text, # with HTAB and SP allowed between field-vchars. A recipient of CR, LF or NUL # must reject the message or replace them, and other control characters are diff --git a/test/mint/http1/conn_test.exs b/test/mint/http1/conn_test.exs index 2e7d655f..62fa1eec 100644 --- a/test/mint/http1/conn_test.exs +++ b/test/mint/http1/conn_test.exs @@ -147,6 +147,33 @@ defmodule Mint.HTTP1Test do end end + test "empty header names are rejected", %{port: port} do + for stream_headers <- [false, true] do + assert {:ok, conn} = HTTP1.connect(:http, "localhost", port, stream_headers: stream_headers) + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + response = "HTTP/1.1 200 OK\r\n: bar\r\ncontent-length: 0\r\n\r\n" + + assert {:error, conn, %HTTPError{reason: :invalid_header}, [{:status, ^ref, 200}]} = + HTTP1.stream(conn, {:tcp, conn.socket, response}) + + assert_closed_and_released(conn) + end + end + + test "empty trailer names are rejected", %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + + response = + "HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n" <> + "1\r\nX\r\n0\r\n: bar\r\n\r\n" + + assert {:error, conn, %HTTPError{reason: :invalid_trailer_header}, responses} = + HTTP1.stream(conn, {:tcp, conn.socket, response}) + + assert [{:status, ^ref, 200}, {:headers, ^ref, _}, {:data, ^ref, "X"}] = responses + assert_closed_and_released(conn) + end + test "trailer values with control characters are rejected", %{conn: conn} do {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) From 986376342ba75e66c9b44a421a52ef560da813de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Mon, 14 Sep 2026 01:21:40 +0200 Subject: [PATCH 06/13] Trim trailing whitespace from HTTP/1 header values RFC 9112 5.1 excludes optional whitespace after a field value from the value itself. :erlang.decode_packet/3 strips the leading whitespace but keeps the trailing one, so values such as "bar " reached callers with the whitespace and only Content-Length trimmed it. Trim every header and trailer value in one place and make the Content-Length parser accept digits only. --- .dialyzer_ignore | 2 +- lib/mint/http1.ex | 2 ++ lib/mint/http1/parse.ex | 12 ++++++------ test/mint/http1/conn_test.exs | 34 ++++++++++++++++++++++++++++++++++ test/mint/http1/parse_test.exs | 9 ++++++--- 5 files changed, 49 insertions(+), 10 deletions(-) diff --git a/.dialyzer_ignore b/.dialyzer_ignore index 316442b1..418203ae 100644 --- a/.dialyzer_ignore +++ b/.dialyzer_ignore @@ -1,5 +1,5 @@ lib/mint/tunnel_proxy.ex:50 -lib/mint/http1.ex:1094 +lib/mint/http1.ex:1096 lib/mint/unsafe_proxy.ex:173 lib/mint/unsafe_proxy.ex:198 test/support diff --git a/lib/mint/http1.ex b/lib/mint/http1.ex index 0f65a7fd..c5e4f62e 100644 --- a/lib/mint/http1.ex +++ b/lib/mint/http1.ex @@ -1051,6 +1051,8 @@ defmodule Mint.HTTP1 do end defp validate_header(name, value, rest) do + value = Parse.trim_trailing_whitespace(value) + if Response.valid_header_name?(name) and Response.valid_header_value?(value) do {:ok, {name, value}, rest} else diff --git a/lib/mint/http1/parse.ex b/lib/mint/http1/parse.ex index 6a61cc53..167045ba 100644 --- a/lib/mint/http1/parse.ex +++ b/lib/mint/http1/parse.ex @@ -102,18 +102,18 @@ defmodule Mint.HTTP1.Parse do defp chunk_extensions(_data, _state), do: :error def content_length_header(string) do - trimmed = trim_trailing_whitespace(string) - - if ParsingTools.only_digits?(trimmed) do - {:ok, String.to_integer(trimmed)} + if ParsingTools.only_digits?(string) do + {:ok, String.to_integer(string)} else {:error, {:invalid_content_length_header, string}} end end - defp trim_trailing_whitespace(<<>>), do: <<>> + # RFC 9112 5.1: optional whitespace after the field value is not part of it. + # :erlang.decode_packet/3 strips the leading whitespace but keeps the trailing one. + def trim_trailing_whitespace(<<>>), do: <<>> - defp trim_trailing_whitespace(string) do + def trim_trailing_whitespace(string) do prefix_size = byte_size(string) - 1 case string do diff --git a/test/mint/http1/conn_test.exs b/test/mint/http1/conn_test.exs index 62fa1eec..9223daec 100644 --- a/test/mint/http1/conn_test.exs +++ b/test/mint/http1/conn_test.exs @@ -188,6 +188,40 @@ defmodule Mint.HTTP1Test do assert_closed_and_released(conn) end + test "trailing whitespace in header values is trimmed", %{port: port} do + for stream_headers <- [false, true] do + assert {:ok, conn} = HTTP1.connect(:http, "localhost", port, stream_headers: stream_headers) + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + + response = + "HTTP/1.1 200 OK\r\nfoo: bar \t \r\nbaz: \t\r\ntransfer-encoding: chunked \r\n\r\n" <> + "1\r\nX\r\n0\r\nmy-trailer: value\t\r\n\r\n" + + assert {:ok, _conn, responses} = HTTP1.stream(conn, {:tcp, conn.socket, response}) + + assert [ + {:status, ^ref, 200}, + {:headers, ^ref, headers}, + {:data, ^ref, "X"}, + {:headers, ^ref, trailers}, + {:done, ^ref} + ] = responses + + assert headers == [{"foo", "bar"}, {"baz", ""}, {"transfer-encoding", "chunked"}] + assert trailers == [{"my-trailer", "value"}] + end + end + + test "trailing whitespace in content-length is trimmed", %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + response = "HTTP/1.1 200 OK\r\ncontent-length: 1 \t\r\n\r\nX" + + assert {:ok, _conn, [_status, {:headers, ^ref, headers}, {:data, ^ref, "X"}, {:done, ^ref}]} = + HTTP1.stream(conn, {:tcp, conn.socket, response}) + + assert headers == [{"content-length", "1"}] + end + test "header values with obs-text are accepted", %{conn: conn} do {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) response = "HTTP/1.1 200 OK\r\nfoo: b\xC3\xA4r\r\ncontent-length: 0\r\n\r\n" diff --git a/test/mint/http1/parse_test.exs b/test/mint/http1/parse_test.exs index d24ebcad..8cbe0dbd 100644 --- a/test/mint/http1/parse_test.exs +++ b/test/mint/http1/parse_test.exs @@ -96,9 +96,12 @@ defmodule Mint.HTTP1.ParseTest do test "content_length_header/1" do assert content_length_header("0") == {:ok, 0} assert content_length_header("100") == {:ok, 100} - assert content_length_header("200 ") == {:ok, 200} - assert content_length_header("200\t") == {:ok, 200} - assert content_length_header("200 \t ") == {:ok, 200} + + assert content_length_header("200 ") == + {:error, {:invalid_content_length_header, "200 "}} + + assert content_length_header("200\t") == + {:error, {:invalid_content_length_header, "200\t"}} assert content_length_header("200\v") == {:error, {:invalid_content_length_header, "200\v"}} From 44dc6369874f82dcc8270aa7c64bd4ea1331aad8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sat, 19 Sep 2026 19:31:21 +0200 Subject: [PATCH 07/13] Replace HTTP/1 obs-folds consistently and drop the whitespace they leave An obs-fold at the start of a header value ("Foo:" followed by a continuation line) was replaced by a space that stayed in the delivered value. RFC 9112 5.1 excludes the optional whitespace around a value, so leading whitespace is now trimmed like trailing whitespace already was. With the :stream_headers option a folded value was rejected even when the whole header section arrived in one message, while the default mode replaced the fold. The replacement now applies in both modes; a continuation line that arrives after its header was emitted is still rejected. --- lib/mint/http1.ex | 8 ++++---- lib/mint/http1/parse.ex | 10 ++++++++-- test/mint/http1/conn_test.exs | 16 +++++++++++++--- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/lib/mint/http1.ex b/lib/mint/http1.ex index c5e4f62e..2aef8f3e 100644 --- a/lib/mint/http1.ex +++ b/lib/mint/http1.ex @@ -1027,8 +1027,8 @@ defmodule Mint.HTTP1 do # feed, we append a sentinel byte and attempt to decode again. # # Headers are emitted before the next line is seen, so a folded - # continuation line can't be joined to its header. Folds are rejected by - # the value validation since they contain a line feed. + # continuation line that arrives later can't be joined to its header and + # is rejected as an invalid header line. result = with :more <- Response.decode_header(data) do data_size = byte_size(data) @@ -1045,13 +1045,13 @@ defmodule Mint.HTTP1 do end case result do - {:ok, {name, value}, rest} -> validate_header(name, value, rest) + {:ok, {name, value}, rest} -> validate_header(name, Response.replace_obs_fold(value), rest) other -> other end end defp validate_header(name, value, rest) do - value = Parse.trim_trailing_whitespace(value) + value = value |> Parse.trim_leading_whitespace() |> Parse.trim_trailing_whitespace() if Response.valid_header_name?(name) and Response.valid_header_value?(value) do {:ok, {name, value}, rest} diff --git a/lib/mint/http1/parse.ex b/lib/mint/http1/parse.ex index 167045ba..09ec7b18 100644 --- a/lib/mint/http1/parse.ex +++ b/lib/mint/http1/parse.ex @@ -109,8 +109,14 @@ defmodule Mint.HTTP1.Parse do end end - # RFC 9112 5.1: optional whitespace after the field value is not part of it. - # :erlang.decode_packet/3 strips the leading whitespace but keeps the trailing one. + # RFC 9112 5.1: optional whitespace around the field value is not part of it. + # :erlang.decode_packet/3 strips the leading whitespace but keeps the trailing one, + # and replacing an obs-fold at the start of a value leaves a leading space. + def trim_leading_whitespace(<>) when is_whitespace(char), + do: trim_leading_whitespace(rest) + + def trim_leading_whitespace(string), do: string + def trim_trailing_whitespace(<<>>), do: <<>> def trim_trailing_whitespace(string) do diff --git a/test/mint/http1/conn_test.exs b/test/mint/http1/conn_test.exs index 9223daec..f8d200c7 100644 --- a/test/mint/http1/conn_test.exs +++ b/test/mint/http1/conn_test.exs @@ -1597,6 +1597,16 @@ defmodule Mint.HTTP1Test do end end + test "an obs-fold at the start of a header value leaves no leading whitespace", + %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + response = "HTTP/1.1 200 OK\r\nFoo:\r\n bar\r\nContent-Length: 0\r\n\r\n" + assert {:ok, _conn, responses} = HTTP1.stream(conn, {:tcp, conn.socket, response}) + + assert [{:status, ^ref, 200}, {:headers, ^ref, headers}, {:done, ^ref}] = responses + assert headers == [{"foo", "bar"}, {"content-length", "0"}] + end + defp request_string(string) do String.replace(string, "\n", "\r\n") end @@ -1662,10 +1672,10 @@ defmodule Mint.HTTP1Test do assert {:headers, ^ref, [{"qux", "Quux"}]} = headers2 end - test "rejects obsolete line folding in header values", %{conn: conn} do - {:ok, conn, _ref} = HTTP1.request(conn, "GET", "/", [], nil) + test "replaces obsolete line folding received together with its header", %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) - assert {:error, _conn, %HTTPError{reason: :invalid_header}, [_status]} = + assert {:ok, _conn, [_status, {:headers, ^ref, [{"foo", "bar baz"}]}]} = HTTP1.stream( conn, {:tcp, conn.socket, "HTTP/1.1 200 OK\r\nFoo: bar\r\n baz\r\n\r\n"} From 49c16c56ce8678c13170437fe66b2555419d7ab6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Thu, 24 Sep 2026 13:35:39 +0200 Subject: [PATCH 08/13] Reject leading zeros in the HTTP/1 status line version and status code :erlang.decode_packet/3 returns the version and status code as integers, so "HTTP/01.1", "HTTP/1.01" and a status code of "0200" were accepted as HTTP/1.1 and 200. RFC 9112 defines the version as "HTTP/" DIGIT "." DIGIT and the status code as three digits, so the raw status line is checked as well. --- lib/mint/http1/response.ex | 15 ++++++++++++++- test/mint/http1/conn_test.exs | 5 ++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/lib/mint/http1/response.ex b/lib/mint/http1/response.ex index d13dd2b2..0b931198 100644 --- a/lib/mint/http1/response.ex +++ b/lib/mint/http1/response.ex @@ -9,7 +9,10 @@ defmodule Mint.HTTP1.Response do case :erlang.decode_packet(:http_bin, binary, []) do {:ok, {:http_response, {1, minor} = version, status, reason}, rest} when minor in 0..9 and status in 100..999 -> - if valid_reason_phrase?(reason) do + line_size = byte_size(binary) - byte_size(rest) + + if valid_version_and_status?(binary_part(binary, 0, line_size)) and + valid_reason_phrase?(reason) do {:ok, {version, status, reason}, rest} else :error @@ -45,6 +48,16 @@ defmodule Mint.HTTP1.Response do end end + # :erlang.decode_packet/3 returns the version and status code as integers, so + # leading zeros such as "HTTP/01.1" or "0200" have to be checked on the raw line. + # RFC 9112 2.3 and 4: HTTP-version = "HTTP/" DIGIT "." DIGIT, status-code = 3DIGIT. + defp valid_version_and_status?(<<"HTTP/1.", minor, ?\s, a, b, c, next, _rest::binary>>) + when minor in ?0..?9 and a in ?0..?9 and b in ?0..?9 and c in ?0..?9 and + next in [?\s, ?\r, ?\n], + do: true + + defp valid_version_and_status?(_line), do: false + # RFC 9112 4: reason-phrase = 1*( HTAB / SP / VCHAR / obs-text ) defp valid_reason_phrase?(<>) when char == ?\t or char in 32..126 or char in 128..255, diff --git a/test/mint/http1/conn_test.exs b/test/mint/http1/conn_test.exs index f8d200c7..d49ba795 100644 --- a/test/mint/http1/conn_test.exs +++ b/test/mint/http1/conn_test.exs @@ -66,7 +66,10 @@ defmodule Mint.HTTP1Test do "HTTP/1.10 200 OK\r\n", "HTTP/1.1 200 O\0K\r\n", "HTTP/1.1 200 OK\r\r\n", - "HTTP/1.1 200OK\r\n" + "HTTP/1.1 200OK\r\n", + "HTTP/1.1 0200 OK\r\n", + "HTTP/01.1 200 OK\r\n", + "HTTP/1.01 200 OK\r\n" ] for line <- lines do From dfde784feb97d8e121fc7056e7477fa7b92db185 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Thu, 24 Sep 2026 13:35:40 +0200 Subject: [PATCH 09/13] Describe how :stream_headers handles obsolete line folding --- .dialyzer_ignore | 2 +- lib/mint/http1.ex | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.dialyzer_ignore b/.dialyzer_ignore index 418203ae..480c2a34 100644 --- a/.dialyzer_ignore +++ b/.dialyzer_ignore @@ -1,5 +1,5 @@ lib/mint/tunnel_proxy.ex:50 -lib/mint/http1.ex:1096 +lib/mint/http1.ex:1097 lib/mint/unsafe_proxy.ex:173 lib/mint/unsafe_proxy.ex:198 test/support diff --git a/lib/mint/http1.ex b/lib/mint/http1.ex index 2aef8f3e..2ffa45e2 100644 --- a/lib/mint/http1.ex +++ b/lib/mint/http1.ex @@ -159,9 +159,10 @@ defmodule Mint.HTTP1 do * `:stream_headers` - (`t:boolean/0`) if set to `true`, response headers and trailer headers will be emitted as they are parsed, rather than buffered until the complete header section is received. When enabled, you may receive multiple `{:headers, ref, headers}` responses - for a single request, and header values that use obsolete line folding are rejected - with an `:invalid_header` error instead of being unfolded. Defaults to `false`. - *Available since v1.10.0*. + for a single request. A folded continuation line (obsolete line folding) is unfolded + when it arrives in the same message as its header, and rejected with an + `:invalid_header` error when it arrives after its header was emitted. Defaults to + `false`. *Available since v1.10.0*. """ @spec connect(Types.scheme(), Types.address(), :inet.port_number(), keyword()) :: From 5abde10714bf12d84a37bbeec62d025b268b9e38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Thu, 24 Sep 2026 13:40:11 +0200 Subject: [PATCH 10/13] Replace the whitespace before an HTTP/1 obs-fold too RFC 9112 5.2 defines obs-fold as OWS CRLF RWS, but only the line break and the whitespace after it were replaced, so "one\t\r\n two" was delivered as "one\t two" instead of "one two". --- lib/mint/http1/response.ex | 7 ++++--- test/mint/http1/conn_test.exs | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/lib/mint/http1/response.ex b/lib/mint/http1/response.ex index 0b931198..85aef6ad 100644 --- a/lib/mint/http1/response.ex +++ b/lib/mint/http1/response.ex @@ -88,16 +88,17 @@ defmodule Mint.HTTP1.Response do def valid_header_value?(_other), do: false # RFC 9112 5.2: a user agent must replace each received obs-fold with one or - # more SP octets before interpreting the field value. + # more SP octets before interpreting the field value. obs-fold = OWS CRLF RWS, + # so the whitespace on both sides of the line break is replaced too. def replace_obs_fold(value) do if obs_fold?(value), do: replace_obs_fold(value, <<>>), else: value end defp replace_obs_fold(<<"\r\n", rest::binary>>, acc), - do: replace_obs_fold(skip_whitespace(rest), <>) + do: replace_obs_fold(skip_whitespace(rest), <>) defp replace_obs_fold(<<"\n", rest::binary>>, acc), - do: replace_obs_fold(skip_whitespace(rest), <>) + do: replace_obs_fold(skip_whitespace(rest), <>) defp replace_obs_fold(<>, acc), do: replace_obs_fold(rest, <>) diff --git a/test/mint/http1/conn_test.exs b/test/mint/http1/conn_test.exs index d49ba795..f1c18133 100644 --- a/test/mint/http1/conn_test.exs +++ b/test/mint/http1/conn_test.exs @@ -1600,6 +1600,38 @@ defmodule Mint.HTTP1Test do end end + test "whitespace before an obs-fold is replaced with the fold", %{port: port} do + for stream_headers <- [false, true] do + assert {:ok, conn} = + HTTP1.connect(:http, "localhost", port, stream_headers: stream_headers) + + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + + assert {:ok, conn, _responses} = + HTTP1.stream(conn, {:tcp, conn.socket, "HTTP/1.1 200 OK\r\nFoo: one\t \r"}) + + assert {:ok, _conn, responses} = + HTTP1.stream(conn, {:tcp, conn.socket, "\n two\r\nContent-Length: 0\r\n\r\n"}) + + assert {"foo", "one two"} in for( + {:headers, ^ref, headers} <- responses, + h <- headers, + do: h + ) + end + end + + test "whitespace before an obs-fold in a trailer is replaced with the fold", %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + + response = + "HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n" <> + "0\r\nfoo: one\t\r\n two\r\n\r\n" + + assert {:ok, _conn, responses} = HTTP1.stream(conn, {:tcp, conn.socket, response}) + assert {:headers, ref, [{"foo", "one two"}]} in responses + end + test "an obs-fold at the start of a header value leaves no leading whitespace", %{conn: conn} do {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) From d47dc4c84cb7cbe9cf75474e7d2877901f20a89c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Thu, 24 Sep 2026 13:46:49 +0200 Subject: [PATCH 11/13] Parse the HTTP/1 status line without :erlang.decode_packet/3 decode_packet/3 accepts status lines that RFC 9112 doesn't, and returns the version and status code as integers, so the raw line had to be checked again after it. The status line is now matched directly against the RFC 9112 grammar in one pass. The only difference in accepted lines is that the reason phrase keeps whitespace after the separating space, as the grammar requires, instead of having it stripped. --- lib/mint/http1/response.ex | 58 +++++++++++++++++++------------ test/mint/http1/response_test.exs | 55 +++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 23 deletions(-) create mode 100644 test/mint/http1/response_test.exs diff --git a/lib/mint/http1/response.ex b/lib/mint/http1/response.ex index 85aef6ad..576883b6 100644 --- a/lib/mint/http1/response.ex +++ b/lib/mint/http1/response.ex @@ -5,30 +5,52 @@ defmodule Mint.HTTP1.Response do alias Mint.Core.Headers + # RFC 9112 4: status-line = HTTP-version SP status-code SP [ reason-phrase ] + # RFC 9112 2.2 allows a bare LF as the line terminator. def decode_status_line(binary) do - case :erlang.decode_packet(:http_bin, binary, []) do - {:ok, {:http_response, {1, minor} = version, status, reason}, rest} - when minor in 0..9 and status in 100..999 -> - line_size = byte_size(binary) - byte_size(rest) + case :binary.split(binary, "\n") do + [line, rest] -> + line = strip_trailing_cr(line) - if valid_version_and_status?(binary_part(binary, 0, line_size)) and - valid_reason_phrase?(reason) do + with {:ok, version, status, reason} <- parse_status_line(line), + true <- valid_reason_phrase?(reason) do {:ok, {version, status, reason}, rest} else - :error + _other -> :error end - {:ok, _other, _rest} -> - :error - - {:more, _length} -> + [_incomplete] -> :more + end + end - {:error, _reason} -> - :error + defp strip_trailing_cr(line) do + size = byte_size(line) - 1 + + case line do + <> -> line + line -> line + end + end + + # RFC 9112 2.3: HTTP-version = "HTTP/" DIGIT "." DIGIT, and only HTTP/1.x + # responses are accepted. RFC 9112 4: status-code = 3DIGIT. + defp parse_status_line(<<"HTTP/1.", minor, ?\s, a, b, c, rest::binary>>) + when minor in ?0..?9 and a in ?1..?9 and b in ?0..?9 and c in ?0..?9 do + reason = + case rest do + <<>> -> {:ok, ""} + <> -> {:ok, reason} + _other -> :error + end + + with {:ok, reason} <- reason do + {:ok, {1, minor - ?0}, (a - ?0) * 100 + (b - ?0) * 10 + (c - ?0), reason} end end + defp parse_status_line(_line), do: :error + def decode_header(binary) do case :erlang.decode_packet(:httph_bin, binary, []) do {:ok, {:http_header, _unused, name, _reserved, value}, rest} -> @@ -48,16 +70,6 @@ defmodule Mint.HTTP1.Response do end end - # :erlang.decode_packet/3 returns the version and status code as integers, so - # leading zeros such as "HTTP/01.1" or "0200" have to be checked on the raw line. - # RFC 9112 2.3 and 4: HTTP-version = "HTTP/" DIGIT "." DIGIT, status-code = 3DIGIT. - defp valid_version_and_status?(<<"HTTP/1.", minor, ?\s, a, b, c, next, _rest::binary>>) - when minor in ?0..?9 and a in ?0..?9 and b in ?0..?9 and c in ?0..?9 and - next in [?\s, ?\r, ?\n], - do: true - - defp valid_version_and_status?(_line), do: false - # RFC 9112 4: reason-phrase = 1*( HTAB / SP / VCHAR / obs-text ) defp valid_reason_phrase?(<>) when char == ?\t or char in 32..126 or char in 128..255, diff --git a/test/mint/http1/response_test.exs b/test/mint/http1/response_test.exs new file mode 100644 index 00000000..d4a3fe4b --- /dev/null +++ b/test/mint/http1/response_test.exs @@ -0,0 +1,55 @@ +defmodule Mint.HTTP1.ResponseTest do + use ExUnit.Case, async: true + + import Mint.HTTP1.Response, only: [decode_status_line: 1] + + describe "decode_status_line/1" do + test "valid status lines" do + assert decode_status_line("HTTP/1.1 200 OK\r\nrest") == {:ok, {{1, 1}, 200, "OK"}, "rest"} + + assert decode_status_line("HTTP/1.0 404 Not Found\r\n") == + {:ok, {{1, 0}, 404, "Not Found"}, ""} + + assert decode_status_line("HTTP/1.1 200 OK\nrest") == {:ok, {{1, 1}, 200, "OK"}, "rest"} + assert decode_status_line("HTTP/1.1 204\r\n") == {:ok, {{1, 1}, 204, ""}, ""} + assert decode_status_line("HTTP/1.1 204 \r\n") == {:ok, {{1, 1}, 204, ""}, ""} + assert decode_status_line("HTTP/1.1 200 OK \r\n") == {:ok, {{1, 1}, 200, "OK "}, ""} + assert decode_status_line("HTTP/1.1 200 OK\r\n") == {:ok, {{1, 1}, 200, " OK"}, ""} + assert decode_status_line("HTTP/1.1 200 A\tB\r\n") == {:ok, {{1, 1}, 200, "A\tB"}, ""} + assert decode_status_line("HTTP/1.1 200 caf\xE9\r\n") == {:ok, {{1, 1}, 200, "caf\xE9"}, ""} + assert decode_status_line("HTTP/1.9 999 X\r\n") == {:ok, {{1, 9}, 999, "X"}, ""} + end + + test "incomplete status lines" do + for line <- ["", "H", "HTTP/1.", "HTTP/1.1 20", "HTTP/1.1 200", "HTTP/1.1 200 OK\r"] do + assert decode_status_line(line) == :more, "expected #{inspect(line)} to need more data" + end + end + + test "invalid status lines" do + lines = [ + "\r\nHTTP/1.1 200 OK\r\n", + "HTTP/1.1 2000 OK\r\n", + "HTTP/1.1 99 OK\r\n", + "HTTP/1.1 099 OK\r\n", + "HTTP/1.1 0200 OK\r\n", + "HTTP/2.0 200 OK\r\n", + "HTTP/0.9 200 OK\r\n", + "HTTP/01.1 200 OK\r\n", + "HTTP/1.01 200 OK\r\n", + "HTTP/1.10 200 OK\r\n", + "http/1.1 200 OK\r\n", + "HTTP/1.1 200 OK\r\n", + "HTTP/1.1 200OK\r\n", + "HTTP/1.1 2x0 OK\r\n", + "HTTP/1.1 200 O\0K\r\n", + "HTTP/1.1 200 OK\r\r\n", + "HTTP/1.1 200 OK\rX\r\n" + ] + + for line <- lines do + assert decode_status_line(line) == :error, "expected #{inspect(line)} to be rejected" + end + end + end +end From d7aef748ef161fde8713699c86af0fd080958ed6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Thu, 24 Sep 2026 14:02:18 +0200 Subject: [PATCH 12/13] Parse the HTTP/1 status line in a single pass Splitting the line on LF with :binary.split/2 and validating the reason phrase afterwards was slower than :erlang.decode_packet/3 with the same validation. The reason phrase is now validated while looking for the end of the line. --- lib/mint/http1/response.ex | 83 ++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 44 deletions(-) diff --git a/lib/mint/http1/response.ex b/lib/mint/http1/response.ex index 576883b6..70932d8d 100644 --- a/lib/mint/http1/response.ex +++ b/lib/mint/http1/response.ex @@ -6,50 +6,53 @@ defmodule Mint.HTTP1.Response do alias Mint.Core.Headers # RFC 9112 4: status-line = HTTP-version SP status-code SP [ reason-phrase ] + # RFC 9112 2.3: HTTP-version = "HTTP/" DIGIT "." DIGIT, and only HTTP/1.x + # responses are accepted. RFC 9112 4: status-code = 3DIGIT. + # The reason phrase is validated while looking for the end of the line, and # RFC 9112 2.2 allows a bare LF as the line terminator. - def decode_status_line(binary) do - case :binary.split(binary, "\n") do - [line, rest] -> - line = strip_trailing_cr(line) - - with {:ok, version, status, reason} <- parse_status_line(line), - true <- valid_reason_phrase?(reason) do - {:ok, {version, status, reason}, rest} - else - _other -> :error - end - - [_incomplete] -> - :more + def decode_status_line(<<"HTTP/1.", minor, ?\s, a, b, c, rest::binary>>) + when minor in ?0..?9 and a in ?1..?9 and b in ?0..?9 and c in ?0..?9 do + version = {1, minor - ?0} + status = (a - ?0) * 100 + (b - ?0) * 10 + (c - ?0) + + case rest do + <> -> decode_reason_phrase(reason, reason, 0, version, status) + _other -> decode_empty_reason_phrase(rest, version, status) end end - defp strip_trailing_cr(line) do - size = byte_size(line) - 1 - - case line do - <> -> line - line -> line + def decode_status_line(binary) do + if byte_size(binary) < byte_size("HTTP/1.1 200") and not String.contains?(binary, "\n") do + :more + else + :error end end - # RFC 9112 2.3: HTTP-version = "HTTP/" DIGIT "." DIGIT, and only HTTP/1.x - # responses are accepted. RFC 9112 4: status-code = 3DIGIT. - defp parse_status_line(<<"HTTP/1.", minor, ?\s, a, b, c, rest::binary>>) - when minor in ?0..?9 and a in ?1..?9 and b in ?0..?9 and c in ?0..?9 do - reason = - case rest do - <<>> -> {:ok, ""} - <> -> {:ok, reason} - _other -> :error - end - - with {:ok, reason} <- reason do - {:ok, {1, minor - ?0}, (a - ?0) * 100 + (b - ?0) * 10 + (c - ?0), reason} - end - end + # RFC 9112 4: reason-phrase = 1*( HTAB / SP / VCHAR / obs-text ) + defp decode_reason_phrase(<<"\r\n", rest::binary>>, reason, size, version, status), + do: {:ok, {version, status, binary_part(reason, 0, size)}, rest} - defp parse_status_line(_line), do: :error + defp decode_reason_phrase(<<"\n", rest::binary>>, reason, size, version, status), + do: {:ok, {version, status, binary_part(reason, 0, size)}, rest} + + defp decode_reason_phrase(<>, reason, size, version, status) + when char == ?\t or char in 32..126 or char in 128..255, + do: decode_reason_phrase(rest, reason, size + 1, version, status) + + defp decode_reason_phrase(data, _reason, _size, _version, _status) when data in ["", "\r"], + do: :more + + defp decode_reason_phrase(_data, _reason, _size, _version, _status), do: :error + + defp decode_empty_reason_phrase(<<"\r\n", rest::binary>>, version, status), + do: {:ok, {version, status, ""}, rest} + + defp decode_empty_reason_phrase(<<"\n", rest::binary>>, version, status), + do: {:ok, {version, status, ""}, rest} + + defp decode_empty_reason_phrase(data, _version, _status) when data in ["", "\r"], do: :more + defp decode_empty_reason_phrase(_data, _version, _status), do: :error def decode_header(binary) do case :erlang.decode_packet(:httph_bin, binary, []) do @@ -70,14 +73,6 @@ defmodule Mint.HTTP1.Response do end end - # RFC 9112 4: reason-phrase = 1*( HTAB / SP / VCHAR / obs-text ) - defp valid_reason_phrase?(<>) - when char == ?\t or char in 32..126 or char in 128..255, - do: valid_reason_phrase?(rest) - - defp valid_reason_phrase?(<<>>), do: true - defp valid_reason_phrase?(_other), do: false - def obs_fold?(value), do: :binary.match(value, "\n") != :nomatch # RFC 9110 5.1: field-name = token, token = 1*tchar From 3edf068dd9cb574f059b93e485ad0524a51e791d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Thu, 24 Sep 2026 14:08:27 +0200 Subject: [PATCH 13/13] Parse HTTP/1 header lines with SWAR instead of :erlang.decode_packet/3 Header values were decoded with decode_packet/3 and then scanned again to replace obs-folds, trim whitespace and validate the name and value bytes. Header lines are now parsed in one pass that reads 7 bytes at a time as an integer and uses bit tricks to find the colon and the line end while validating the name and value. Bare LF line endings and obs-folds are handled by the parser, so the sentinel byte :stream_headers appended to make decode_packet/3 emit a header at the end of the data isn't needed. Co-authored-by: Wojtek Mach --- .dialyzer_ignore | 2 +- lib/mint/http1.ex | 50 +----- lib/mint/http1/response.ex | 242 ++++++++++++++++++++++++------ test/mint/http1/response_test.exs | 145 +++++++++++++++++- 4 files changed, 347 insertions(+), 92 deletions(-) diff --git a/.dialyzer_ignore b/.dialyzer_ignore index 480c2a34..ebae937c 100644 --- a/.dialyzer_ignore +++ b/.dialyzer_ignore @@ -1,5 +1,5 @@ lib/mint/tunnel_proxy.ex:50 -lib/mint/http1.ex:1097 +lib/mint/http1.ex:1055 lib/mint/unsafe_proxy.ex:173 lib/mint/unsafe_proxy.ex:198 test/support diff --git a/lib/mint/http1.ex b/lib/mint/http1.ex index 2ffa45e2..744f8d1a 100644 --- a/lib/mint/http1.ex +++ b/lib/mint/http1.ex @@ -1014,52 +1014,10 @@ defmodule Mint.HTTP1 do end end - defp decode_header(data, false = _stream_headers) do - case Response.decode_header(data) do - {:ok, {name, value}, rest} -> validate_header(name, Response.replace_obs_fold(value), rest) - other -> other - end - end - - defp decode_header(data, true = _stream_headers) do - # By default, :erlang.decode_packet/3 asks for more data when a packet - # containing a full header ends with a line feed (likely to handle line - # folding). If we get a :more response on a packet that ends with a line - # feed, we append a sentinel byte and attempt to decode again. - # - # Headers are emitted before the next line is seen, so a folded - # continuation line that arrives later can't be joined to its header and - # is rejected as an invalid header line. - result = - with :more <- Response.decode_header(data) do - data_size = byte_size(data) - - case data do - <<_::binary-size(^data_size - 1), ?\n>> -> - with {:ok, {name, value}, <<0>>} <- Response.decode_header(<>) do - {:ok, {name, value}, ""} - end - - _ -> - :more - end - end - - case result do - {:ok, {name, value}, rest} -> validate_header(name, Response.replace_obs_fold(value), rest) - other -> other - end - end - - defp validate_header(name, value, rest) do - value = value |> Parse.trim_leading_whitespace() |> Parse.trim_trailing_whitespace() - - if Response.valid_header_name?(name) and Response.valid_header_value?(value) do - {:ok, {name, value}, rest} - else - :error - end - end + # With :stream_headers a header is emitted as soon as its line ends, before the + # next line is seen, so a folded continuation line that arrives later can't be + # joined to it and is rejected as an invalid header line. + defp decode_header(data, stream_headers?), do: Response.decode_header(data, stream_headers?) defp next_request(%{request: nil} = conn, data, responses) do # TODO: Figure out if we should keep buffering even though there are no diff --git a/lib/mint/http1/response.ex b/lib/mint/http1/response.ex index 70932d8d..835e0c44 100644 --- a/lib/mint/http1/response.ex +++ b/lib/mint/http1/response.ex @@ -1,6 +1,7 @@ defmodule Mint.HTTP1.Response do @moduledoc false + import Bitwise, only: [|||: 2] import Mint.HTTP1.Parse alias Mint.Core.Headers @@ -54,67 +55,220 @@ defmodule Mint.HTTP1.Response do defp decode_empty_reason_phrase(data, _version, _status) when data in ["", "\r"], do: :more defp decode_empty_reason_phrase(_data, _version, _status), do: :error - def decode_header(binary) do - case :erlang.decode_packet(:httph_bin, binary, []) do - {:ok, {:http_header, _unused, name, _reserved, value}, rest} -> - {:ok, {header_name(name), value}, rest} + # Header lines are scanned a word of bytes at a time (SWAR, SIMD within a + # register): 7 bytes are read as one integer, which stays a small integer on + # 64-bit systems, and bit tricks tell whether any byte in it is one the scan + # has to stop at. Only words that contain such a byte are looked at byte by byte. + @word 7 + @bits @word * 8 + @ones Enum.reduce(1..@word, 0, fn _, acc -> Bitwise.bor(Bitwise.bsl(acc, 8), 0x01) end) + @highs @ones * 0x80 + @lows @ones * 0x7F + @colons @ones * ?: + @dels @ones * 0x7F + @spaces @ones * 0x20 - {:ok, :http_eoh, rest} -> - {:ok, :eof, rest} + # Sets the high bit of every byte of `word` that equals the byte in `pattern` + # (low bytes can also get false positives above a real match, so matches are + # confirmed byte by byte). + defmacrop has_byte(word, pattern) do + quote do + x = Bitwise.bxor(unquote(word), unquote(pattern)) + Bitwise.band(Bitwise.band(x - @ones, Bitwise.bnot(x)), @highs) + end + end - {:ok, _other, _rest} -> - :error + # Sets the high bit of bytes of `word` that are below the byte in `pattern`. + defmacrop has_less(word, pattern) do + quote do + x = unquote(word) + Bitwise.band(Bitwise.band(x - unquote(pattern), Bitwise.bnot(x)), @highs) + end + end + + # Sets the high bit of ASCII bytes of the word between lo and hi. `x7` is the + # word with the high bit of every byte cleared and `nx` has the high bit set + # for every byte below 0x80. + defmacrop has_between(x7, nx, lo, hi) do + quote do + Bitwise.band( + Bitwise.band( + unquote(@ones * (127 + hi + 1)) - unquote(x7), + unquote(x7) + unquote(@ones * (127 - (lo - 1))) + ), + unquote(nx) + ) + end + end + + # Decodes one header or trailer line. The name is lowercased and the value has + # its surrounding whitespace removed and its obsolete line folds replaced with + # a space. A line that ends exactly at the end of the data returns :more unless + # emit_at_end? is true, since a folded continuation line could follow it. + def decode_header(binary, emit_at_end? \\ false) - {:more, _length} -> - :more + def decode_header(<<"\r\n", rest::binary>>, _emit_at_end?), do: {:ok, :eof, rest} + def decode_header(<<"\n", rest::binary>>, _emit_at_end?), do: {:ok, :eof, rest} + def decode_header(data, _emit_at_end?) when data in ["", "\r"], do: :more - {:error, _reason} -> + def decode_header(binary, emit_at_end?) do + case find_colon(binary, 0) do + {:ok, 0} -> :error + + {:ok, name_size} -> + <> = binary + decode_header_value(rest, [], emit_at_end?, Headers.lower_raw(name)) + + other -> + other + end + end + + defp decode_header_value(data, segments, emit_at_end?, name) do + with {:ok, size} <- find_line_end(data, 0) do + <> = data + + with {:ok, rest} <- skip_line_end(rest) do + segments = [segment | segments] + + case rest do + <> when char in ~c"\s\t" -> + decode_header_value(rest, segments, emit_at_end?, name) + + <<>> when not emit_at_end? -> + :more + + _other -> + {:ok, {name, join_segments(segments)}, rest} + end + end + end + end + + defp skip_line_end(<<"\r\n", rest::binary>>), do: {:ok, rest} + defp skip_line_end(<<"\n", rest::binary>>), do: {:ok, rest} + defp skip_line_end("\r"), do: :more + defp skip_line_end(_other), do: :error + + # RFC 9112 5.1 excludes the whitespace around the value from it, and RFC 9112 + # 5.2 has a user agent replace each obs-fold (OWS CRLF RWS) with a space. + defp join_segments([segment]), do: trim_whitespace(segment) + + defp join_segments(segments) do + segments + |> Enum.reverse() + |> Enum.map(&trim_whitespace/1) + |> Enum.reject(&(&1 == "")) + |> Enum.join(" ") + end + + defp trim_whitespace(value), + do: value |> trim_leading_whitespace() |> trim_trailing_whitespace() + + # RFC 9110 5.1: field-name = token, token = 1*tchar. Returns the position of + # the colon, or :error on the first byte that isn't a tchar. + defp find_colon(<>, index) do + x7 = Bitwise.band(word, @lows) + nx = Bitwise.band(Bitwise.bnot(word), @highs) + + mask = + Bitwise.band(word, @highs) ||| + has_byte(word, @colons) ||| + has_less(word, @spaces + @ones) ||| + has_byte(word, @dels) ||| + has_between(x7, nx, 0x22, 0x22) ||| + has_between(x7, nx, 0x28, 0x29) ||| + has_between(x7, nx, 0x2C, 0x2C) ||| + has_between(x7, nx, 0x2F, 0x2F) ||| + has_between(x7, nx, 0x3B, 0x40) ||| + has_between(x7, nx, 0x5B, 0x5D) ||| + has_between(x7, nx, 0x7B, 0x7B) ||| + has_between(x7, nx, 0x7D, 0x7D) + + case check_colon(mask, word) do + :continue -> find_colon(rest, index + @word) + {:ok, position} -> {:ok, index + position} + :error -> :error end end - def obs_fold?(value), do: :binary.match(value, "\n") != :nomatch + defp find_colon(<>, index), do: {:ok, index} + + defp find_colon(<>, index) when is_tchar(char), + do: find_colon(rest, index + 1) - # RFC 9110 5.1: field-name = token, token = 1*tchar - def valid_header_name?(<<>>), do: false - def valid_header_name?(name), do: tchars?(name) + defp find_colon(<<>>, _index), do: :more + defp find_colon(_other, _index), do: :error - defp tchars?(<>) when is_tchar(char), do: tchars?(rest) - defp tchars?(<<>>), do: true - defp tchars?(_other), do: false + defp check_colon(0, _word), do: :continue + + defp check_colon(mask, word) do + position = lowest_position(mask) + + case Bitwise.band(Bitwise.bsr(word, position * 8), 0xFF) do + ?: -> {:ok, position} + char when is_tchar(char) -> check_colon(Bitwise.band(mask, mask - 1), word) + _other -> :error + end + end # RFC 9110 5.5: field-value = *field-content, field-vchar = VCHAR / obs-text, - # with HTAB and SP allowed between field-vchars. A recipient of CR, LF or NUL - # must reject the message or replace them, and other control characters are - # not allowed at all. - def valid_header_value?(<>) - when char == ?\t or char in 32..126 or char in 128..255, - do: valid_header_value?(rest) - - def valid_header_value?(<<>>), do: true - def valid_header_value?(_other), do: false - - # RFC 9112 5.2: a user agent must replace each received obs-fold with one or - # more SP octets before interpreting the field value. obs-fold = OWS CRLF RWS, - # so the whitespace on both sides of the line break is replaced too. - def replace_obs_fold(value) do - if obs_fold?(value), do: replace_obs_fold(value, <<>>), else: value + # with HTAB and SP allowed between field-vchars. Returns the position of the + # CR or LF ending the line, or :error on any other control character. + defp find_line_end( + <> = data, + index + ) do + mask = + has_less(w1, @spaces) ||| has_byte(w1, @dels) ||| has_less(w2, @spaces) ||| + has_byte(w2, @dels) + + if mask == 0 do + find_line_end(rest, index + 2 * @word) + else + find_line_end_in_word(data, index) + end + end + + defp find_line_end(data, index), do: find_line_end_in_word(data, index) + + defp find_line_end_in_word(<>, index) do + mask = has_less(word, @spaces) ||| has_byte(word, @dels) + + case check_line_end(mask, word) do + :continue -> find_line_end(rest, index + @word) + {:ok, position} -> {:ok, index + position} + :error -> :error + end end - defp replace_obs_fold(<<"\r\n", rest::binary>>, acc), - do: replace_obs_fold(skip_whitespace(rest), <>) + defp find_line_end_in_word(<>, index) when char in ~c"\r\n", + do: {:ok, index} - defp replace_obs_fold(<<"\n", rest::binary>>, acc), - do: replace_obs_fold(skip_whitespace(rest), <>) + defp find_line_end_in_word(<>, index), do: find_line_end(rest, index + 1) - defp replace_obs_fold(<>, acc), - do: replace_obs_fold(rest, <>) + defp find_line_end_in_word(<>, index) when char >= 0x20 and char != 0x7F, + do: find_line_end(rest, index + 1) - defp replace_obs_fold(<<>>, acc), do: acc + defp find_line_end_in_word(<<>>, _index), do: :more + defp find_line_end_in_word(_other, _index), do: :error - defp skip_whitespace(<>) when char in ~c"\s\t", do: skip_whitespace(rest) - defp skip_whitespace(rest), do: rest + defp check_line_end(0, _word), do: :continue - defp header_name(atom) when is_atom(atom), do: atom |> Atom.to_string() |> header_name() - defp header_name(binary) when is_binary(binary), do: Headers.lower_raw(binary) + defp check_line_end(mask, word) do + position = lowest_position(mask) + + case Bitwise.band(Bitwise.bsr(word, position * 8), 0xFF) do + char when char in ~c"\r\n" -> {:ok, position} + ?\t -> check_line_end(Bitwise.band(mask, mask - 1), word) + char when char < 0x20 or char == 0x7F -> :error + _other -> check_line_end(Bitwise.band(mask, mask - 1), word) + end + end + + for k <- 0..(@word - 1) do + defp lowest_position(mask) when Bitwise.band(mask, unquote(Bitwise.bsl(0x80, 8 * k))) != 0, + do: unquote(k) + end end diff --git a/test/mint/http1/response_test.exs b/test/mint/http1/response_test.exs index d4a3fe4b..7fef6be9 100644 --- a/test/mint/http1/response_test.exs +++ b/test/mint/http1/response_test.exs @@ -1,7 +1,8 @@ defmodule Mint.HTTP1.ResponseTest do use ExUnit.Case, async: true + use ExUnitProperties - import Mint.HTTP1.Response, only: [decode_status_line: 1] + import Mint.HTTP1.Response, only: [decode_status_line: 1, decode_header: 1, decode_header: 2] describe "decode_status_line/1" do test "valid status lines" do @@ -52,4 +53,146 @@ defmodule Mint.HTTP1.ResponseTest do end end end + + describe "decode_header/2" do + test "end of the header section" do + assert decode_header("\r\nbody") == {:ok, :eof, "body"} + assert decode_header("\nbody") == {:ok, :eof, "body"} + end + + test "lowercases names and trims values" do + assert decode_header("Content-Type: text/plain\r\nX") == + {:ok, {"content-type", "text/plain"}, "X"} + + assert decode_header("A: x y \t\r\nX") == {:ok, {"a", "x y"}, "X"} + assert decode_header("A:\r\nX") == {:ok, {"a", ""}, "X"} + assert decode_header("A: \t \r\nX") == {:ok, {"a", ""}, "X"} + + assert decode_header("Location: http://x.y:8080/p\r\nX") == + {:ok, {"location", "http://x.y:8080/p"}, "X"} + end + + test "accepts LF-only line endings" do + assert decode_header("A: b\nX") == {:ok, {"a", "b"}, "X"} + end + + test "replaces obsolete line folds with a space" do + assert decode_header("A: b\r\n c\r\nX") == {:ok, {"a", "b c"}, "X"} + assert decode_header("A: b \t\r\n\t c\nX") == {:ok, {"a", "b c"}, "X"} + assert decode_header("A:\r\n b\r\nX") == {:ok, {"a", "b"}, "X"} + assert decode_header("A: b\r\n \r\n c\r\nX") == {:ok, {"a", "b c"}, "X"} + end + + test "needs the next byte to know whether a line is folded" do + assert decode_header("A: b\r\n") == :more + assert decode_header("A: b\r\n c\r\n") == :more + assert decode_header("A: b\r\n", true) == {:ok, {"a", "b"}, ""} + assert decode_header("A: b\n", true) == {:ok, {"a", "b"}, ""} + end + + test "incomplete lines" do + line = "Content-Type: text/plain\r\nContent-Length: 12345\r\n\r\n" + + for size <- 0..(byte_size("Content-Type: text/plain\r\n") - 1) do + <> = line + assert decode_header(prefix) == :more, "prefix #{inspect(prefix)}" + end + + assert decode_header(line) == + {:ok, {"content-type", "text/plain"}, "Content-Length: 12345\r\n\r\n"} + end + + test "names and values across word boundaries" do + for name_size <- 1..20, value_size <- 0..20 do + name = String.duplicate("a", name_size) + value = String.duplicate("b", value_size) + + assert decode_header("#{name}: #{value}\r\nX") == {:ok, {name, value}, "X"} + + assert decode_header("#{name}:\t#{value}\tc\r\nX") == + {:ok, {name, String.trim_leading("#{value}\tc")}, "X"} + end + end + + test "all tchars are allowed in names" do + tchars = "!#$%&'*+-.^_`|~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + assert decode_header("#{tchars}: v\r\nX") == {:ok, {String.downcase(tchars), "v"}, "X"} + end + + test "invalid names" do + assert decode_header(": v\r\nX") == :error + assert decode_header(" A: v\r\nX") == :error + assert decode_header("no colon here\r\nX") == :error + + for separator <- ~c"\"(),/;<=>?@[\\]{} \t\x01\x7F" ++ [0x80, 0xFF], position <- 0..16 do + name = String.duplicate("a", position) <> <> <> "b" + assert decode_header("#{name}: v\r\nX") == :error, inspect(name) + end + end + + test "invalid values" do + assert decode_header("A: b\rc\r\nX") == :error + + for byte <- [0x00, 0x01, 0x0B, 0x1F, 0x7F], position <- 0..16 do + value = String.duplicate("v", position) <> <> <> "w" + assert decode_header("A: #{value}\r\nX") == :error, inspect(value) + end + end + + property "decodes generated header lines" do + tchars = ~c"!#$%&'*+-.^_`|~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + + vchar = one_of([integer(0x21..0x7E), integer(0x80..0xFF)]) + whitespace = string(~c" \t", max_length: 3) + word = map(list_of(vchar, min_length: 1, max_length: 12), &:erlang.list_to_binary/1) + line_end = member_of(["\r\n", "\n"]) + + check all name <- string(tchars, min_length: 1, max_length: 24), + lines <- + list_of(list_of({word, whitespace}, max_length: 4), + min_length: 1, + max_length: 3 + ), + leading <- whitespace, + trailing <- whitespace, + line_ends <- list_of(line_end, length: length(lines)), + fold_whitespace <- + list_of(string(~c" \t", min_length: 1, max_length: 3), length: length(lines)) do + segments = + Enum.map(lines, fn words -> Enum.map_join(words, fn {w, ws} -> w <> ws end) end) + + raw = + [segments, line_ends, fold_whitespace] + |> Enum.zip() + |> Enum.with_index() + |> Enum.map_join(fn {{segment, line_end, fold}, index} -> + prefix = if index == 0, do: "", else: fold + + prefix <> + segment <> if(index == length(segments) - 1, do: trailing, else: "") <> line_end + end) + + expected = + segments + |> Enum.map(&String.trim(&1, " ")) + |> Enum.map(&trim_ows/1) + |> Enum.reject(&(&1 == "")) + |> Enum.join(" ") + + assert decode_header(name <> ":" <> leading <> raw <> "X") == + {:ok, {String.downcase(name), expected}, "X"} + end + end + + test "obs-text and HTAB are allowed in values" do + for position <- 0..16 do + value = "x" <> String.duplicate("v", position) <> "\t\x80\xC3\xA9\xFFw" + assert decode_header("A: #{value}\r\nX") == {:ok, {"a", value}, "X"} + end + end + end + + defp trim_ows(value) do + value |> String.replace(~r/^[ \t]+/, "") |> String.replace(~r/[ \t]+$/, "") + end end