diff --git a/lib/mint/http1.ex b/lib/mint/http1.ex index a3980255..8af319fc 100644 --- a/lib/mint/http1.ex +++ b/lib/mint/http1.ex @@ -159,7 +159,10 @@ defmodule Mint.HTTP1 do * `:stream_headers` - (`t:boolean/0`) if set to `true`, response headers and trailer headers will be emitted as they are parsed, rather than buffered until the complete header section is received. When enabled, you may receive multiple `{:headers, ref, headers}` responses - for a single request. Defaults to `false`. *Available since v1.10.0*. + for a single request. A folded continuation line (obsolete line folding) is unfolded + when it arrives in the same message as its header, and rejected with an + `:invalid_header` error when it arrives after its header was emitted. Defaults to + `false`. *Available since v1.10.0*. """ @spec connect(Types.scheme(), Types.address(), :inet.port_number(), keyword()) :: @@ -1011,27 +1014,10 @@ defmodule Mint.HTTP1 do end end - defp decode_header(data, false = _stream_headers), do: Response.decode_header(data) - - defp decode_header(data, true = _stream_headers) do - # By default, :erlang.decode_packet/3 asks for more data when a packet - # containing a full header ends with a line feed (likely to handle line - # folding). If we get a :more response on a packet that ends with a line - # feed, we append a sentinel byte and attempt to decode again. - with :more <- Response.decode_header(data) do - data_size = byte_size(data) - - case data do - <<_::binary-size(^data_size - 1), ?\n>> -> - with {:ok, {name, value}, <<0>>} <- Response.decode_header(<>) do - {:ok, {name, value}, ""} - end - - _ -> - :more - end - end - end + # With :stream_headers a header is emitted as soon as its line ends, before the + # next line is seen, so a folded continuation line that arrives later can't be + # joined to it and is rejected as an invalid header line. + defp decode_header(data, stream_headers?), do: Response.decode_header(data, stream_headers?) defp next_request(%{request: nil} = conn, data, responses) do # TODO: Figure out if we should keep buffering even though there are no diff --git a/lib/mint/http1/parse.ex b/lib/mint/http1/parse.ex index f4833bb6..09ec7b18 100644 --- a/lib/mint/http1/parse.ex +++ b/lib/mint/http1/parse.ex @@ -102,15 +102,35 @@ defmodule Mint.HTTP1.Parse do defp chunk_extensions(_data, _state), do: :error def content_length_header(string) do - trimmed = String.trim_trailing(string) - - if ParsingTools.only_digits?(trimmed) do - {:ok, String.to_integer(trimmed)} + if ParsingTools.only_digits?(string) do + {:ok, String.to_integer(string)} else {:error, {:invalid_content_length_header, string}} end end + # RFC 9112 5.1: optional whitespace around the field value is not part of it. + # :erlang.decode_packet/3 strips the leading whitespace but keeps the trailing one, + # and replacing an obs-fold at the start of a value leaves a leading space. + def trim_leading_whitespace(<>) when is_whitespace(char), + do: trim_leading_whitespace(rest) + + def trim_leading_whitespace(string), do: string + + def trim_trailing_whitespace(<<>>), do: <<>> + + def trim_trailing_whitespace(string) do + prefix_size = byte_size(string) - 1 + + case string do + <> when is_whitespace(char) -> + trim_trailing_whitespace(prefix) + + _other -> + string + end + end + def connection_header(string) do split_into_downcase_tokens(string) end diff --git a/lib/mint/http1/response.ex b/lib/mint/http1/response.ex index a2018c40..835e0c44 100644 --- a/lib/mint/http1/response.ex +++ b/lib/mint/http1/response.ex @@ -1,43 +1,274 @@ defmodule Mint.HTTP1.Response do @moduledoc false + import Bitwise, only: [|||: 2] + import Mint.HTTP1.Parse + alias Mint.Core.Headers + # RFC 9112 4: status-line = HTTP-version SP status-code SP [ reason-phrase ] + # RFC 9112 2.3: HTTP-version = "HTTP/" DIGIT "." DIGIT, and only HTTP/1.x + # responses are accepted. RFC 9112 4: status-code = 3DIGIT. + # The reason phrase is validated while looking for the end of the line, and + # RFC 9112 2.2 allows a bare LF as the line terminator. + def decode_status_line(<<"HTTP/1.", minor, ?\s, a, b, c, rest::binary>>) + when minor in ?0..?9 and a in ?1..?9 and b in ?0..?9 and c in ?0..?9 do + version = {1, minor - ?0} + status = (a - ?0) * 100 + (b - ?0) * 10 + (c - ?0) + + case rest do + <> -> decode_reason_phrase(reason, reason, 0, version, status) + _other -> decode_empty_reason_phrase(rest, version, status) + end + end + def decode_status_line(binary) do - case :erlang.decode_packet(:http_bin, binary, []) do - {:ok, {:http_response, version, status, reason}, rest} -> - {:ok, {version, status, reason}, rest} + if byte_size(binary) < byte_size("HTTP/1.1 200") and not String.contains?(binary, "\n") do + :more + else + :error + end + end - {:ok, _other, _rest} -> - :error + # RFC 9112 4: reason-phrase = 1*( HTAB / SP / VCHAR / obs-text ) + defp decode_reason_phrase(<<"\r\n", rest::binary>>, reason, size, version, status), + do: {:ok, {version, status, binary_part(reason, 0, size)}, rest} - {:more, _length} -> - :more + defp decode_reason_phrase(<<"\n", rest::binary>>, reason, size, version, status), + do: {:ok, {version, status, binary_part(reason, 0, size)}, rest} - {:error, _reason} -> - :error + defp decode_reason_phrase(<>, reason, size, version, status) + when char == ?\t or char in 32..126 or char in 128..255, + do: decode_reason_phrase(rest, reason, size + 1, version, status) + + defp decode_reason_phrase(data, _reason, _size, _version, _status) when data in ["", "\r"], + do: :more + + defp decode_reason_phrase(_data, _reason, _size, _version, _status), do: :error + + defp decode_empty_reason_phrase(<<"\r\n", rest::binary>>, version, status), + do: {:ok, {version, status, ""}, rest} + + defp decode_empty_reason_phrase(<<"\n", rest::binary>>, version, status), + do: {:ok, {version, status, ""}, rest} + + defp decode_empty_reason_phrase(data, _version, _status) when data in ["", "\r"], do: :more + defp decode_empty_reason_phrase(_data, _version, _status), do: :error + + # Header lines are scanned a word of bytes at a time (SWAR, SIMD within a + # register): 7 bytes are read as one integer, which stays a small integer on + # 64-bit systems, and bit tricks tell whether any byte in it is one the scan + # has to stop at. Only words that contain such a byte are looked at byte by byte. + @word 7 + @bits @word * 8 + @ones Enum.reduce(1..@word, 0, fn _, acc -> Bitwise.bor(Bitwise.bsl(acc, 8), 0x01) end) + @highs @ones * 0x80 + @lows @ones * 0x7F + @colons @ones * ?: + @dels @ones * 0x7F + @spaces @ones * 0x20 + + # Sets the high bit of every byte of `word` that equals the byte in `pattern` + # (low bytes can also get false positives above a real match, so matches are + # confirmed byte by byte). + defmacrop has_byte(word, pattern) do + quote do + x = Bitwise.bxor(unquote(word), unquote(pattern)) + Bitwise.band(Bitwise.band(x - @ones, Bitwise.bnot(x)), @highs) + end + end + + # Sets the high bit of bytes of `word` that are below the byte in `pattern`. + defmacrop has_less(word, pattern) do + quote do + x = unquote(word) + Bitwise.band(Bitwise.band(x - unquote(pattern), Bitwise.bnot(x)), @highs) + end + end + + # Sets the high bit of ASCII bytes of the word between lo and hi. `x7` is the + # word with the high bit of every byte cleared and `nx` has the high bit set + # for every byte below 0x80. + defmacrop has_between(x7, nx, lo, hi) do + quote do + Bitwise.band( + Bitwise.band( + unquote(@ones * (127 + hi + 1)) - unquote(x7), + unquote(x7) + unquote(@ones * (127 - (lo - 1))) + ), + unquote(nx) + ) end end - def decode_header(binary) do - case :erlang.decode_packet(:httph_bin, binary, []) do - {:ok, {:http_header, _unused, name, _reserved, value}, rest} -> - {:ok, {header_name(name), value}, rest} + # Decodes one header or trailer line. The name is lowercased and the value has + # its surrounding whitespace removed and its obsolete line folds replaced with + # a space. A line that ends exactly at the end of the data returns :more unless + # emit_at_end? is true, since a folded continuation line could follow it. + def decode_header(binary, emit_at_end? \\ false) - {:ok, :http_eoh, rest} -> - {:ok, :eof, rest} + def decode_header(<<"\r\n", rest::binary>>, _emit_at_end?), do: {:ok, :eof, rest} + def decode_header(<<"\n", rest::binary>>, _emit_at_end?), do: {:ok, :eof, rest} + def decode_header(data, _emit_at_end?) when data in ["", "\r"], do: :more - {:ok, _other, _rest} -> + def decode_header(binary, emit_at_end?) do + case find_colon(binary, 0) do + {:ok, 0} -> :error - {:more, _length} -> - :more + {:ok, name_size} -> + <> = binary + decode_header_value(rest, [], emit_at_end?, Headers.lower_raw(name)) - {:error, _reason} -> - :error + other -> + other end end - defp header_name(atom) when is_atom(atom), do: atom |> Atom.to_string() |> header_name() - defp header_name(binary) when is_binary(binary), do: Headers.lower_raw(binary) + defp decode_header_value(data, segments, emit_at_end?, name) do + with {:ok, size} <- find_line_end(data, 0) do + <> = data + + with {:ok, rest} <- skip_line_end(rest) do + segments = [segment | segments] + + case rest do + <> when char in ~c"\s\t" -> + decode_header_value(rest, segments, emit_at_end?, name) + + <<>> when not emit_at_end? -> + :more + + _other -> + {:ok, {name, join_segments(segments)}, rest} + end + end + end + end + + defp skip_line_end(<<"\r\n", rest::binary>>), do: {:ok, rest} + defp skip_line_end(<<"\n", rest::binary>>), do: {:ok, rest} + defp skip_line_end("\r"), do: :more + defp skip_line_end(_other), do: :error + + # RFC 9112 5.1 excludes the whitespace around the value from it, and RFC 9112 + # 5.2 has a user agent replace each obs-fold (OWS CRLF RWS) with a space. + defp join_segments([segment]), do: trim_whitespace(segment) + + defp join_segments(segments) do + segments + |> Enum.reverse() + |> Enum.map(&trim_whitespace/1) + |> Enum.reject(&(&1 == "")) + |> Enum.join(" ") + end + + defp trim_whitespace(value), + do: value |> trim_leading_whitespace() |> trim_trailing_whitespace() + + # RFC 9110 5.1: field-name = token, token = 1*tchar. Returns the position of + # the colon, or :error on the first byte that isn't a tchar. + defp find_colon(<>, index) do + x7 = Bitwise.band(word, @lows) + nx = Bitwise.band(Bitwise.bnot(word), @highs) + + mask = + Bitwise.band(word, @highs) ||| + has_byte(word, @colons) ||| + has_less(word, @spaces + @ones) ||| + has_byte(word, @dels) ||| + has_between(x7, nx, 0x22, 0x22) ||| + has_between(x7, nx, 0x28, 0x29) ||| + has_between(x7, nx, 0x2C, 0x2C) ||| + has_between(x7, nx, 0x2F, 0x2F) ||| + has_between(x7, nx, 0x3B, 0x40) ||| + has_between(x7, nx, 0x5B, 0x5D) ||| + has_between(x7, nx, 0x7B, 0x7B) ||| + has_between(x7, nx, 0x7D, 0x7D) + + case check_colon(mask, word) do + :continue -> find_colon(rest, index + @word) + {:ok, position} -> {:ok, index + position} + :error -> :error + end + end + + defp find_colon(<>, index), do: {:ok, index} + + defp find_colon(<>, index) when is_tchar(char), + do: find_colon(rest, index + 1) + + defp find_colon(<<>>, _index), do: :more + defp find_colon(_other, _index), do: :error + + defp check_colon(0, _word), do: :continue + + defp check_colon(mask, word) do + position = lowest_position(mask) + + case Bitwise.band(Bitwise.bsr(word, position * 8), 0xFF) do + ?: -> {:ok, position} + char when is_tchar(char) -> check_colon(Bitwise.band(mask, mask - 1), word) + _other -> :error + end + end + + # RFC 9110 5.5: field-value = *field-content, field-vchar = VCHAR / obs-text, + # with HTAB and SP allowed between field-vchars. Returns the position of the + # CR or LF ending the line, or :error on any other control character. + defp find_line_end( + <> = data, + index + ) do + mask = + has_less(w1, @spaces) ||| has_byte(w1, @dels) ||| has_less(w2, @spaces) ||| + has_byte(w2, @dels) + + if mask == 0 do + find_line_end(rest, index + 2 * @word) + else + find_line_end_in_word(data, index) + end + end + + defp find_line_end(data, index), do: find_line_end_in_word(data, index) + + defp find_line_end_in_word(<>, index) do + mask = has_less(word, @spaces) ||| has_byte(word, @dels) + + case check_line_end(mask, word) do + :continue -> find_line_end(rest, index + @word) + {:ok, position} -> {:ok, index + position} + :error -> :error + end + end + + defp find_line_end_in_word(<>, index) when char in ~c"\r\n", + do: {:ok, index} + + defp find_line_end_in_word(<>, index), do: find_line_end(rest, index + 1) + + defp find_line_end_in_word(<>, index) when char >= 0x20 and char != 0x7F, + do: find_line_end(rest, index + 1) + + defp find_line_end_in_word(<<>>, _index), do: :more + defp find_line_end_in_word(_other, _index), do: :error + + defp check_line_end(0, _word), do: :continue + + defp check_line_end(mask, word) do + position = lowest_position(mask) + + case Bitwise.band(Bitwise.bsr(word, position * 8), 0xFF) do + char when char in ~c"\r\n" -> {:ok, position} + ?\t -> check_line_end(Bitwise.band(mask, mask - 1), word) + char when char < 0x20 or char == 0x7F -> :error + _other -> check_line_end(Bitwise.band(mask, mask - 1), word) + end + end + + for k <- 0..(@word - 1) do + defp lowest_position(mask) when Bitwise.band(mask, unquote(Bitwise.bsl(0x80, 8 * k))) != 0, + do: unquote(k) + end end diff --git a/test/mint/http1/conn_test.exs b/test/mint/http1/conn_test.exs index 975c1a14..f1c18133 100644 --- a/test/mint/http1/conn_test.exs +++ b/test/mint/http1/conn_test.exs @@ -50,6 +50,38 @@ defmodule Mint.HTTP1Test do HTTP1.stream(conn, {:tcp, conn.socket, " 200 OK\r\n"}) end + test "status line with an empty reason phrase", %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + + assert {:ok, _conn, [{:status, ^ref, 204}]} = + HTTP1.stream(conn, {:tcp, conn.socket, "HTTP/1.1 204\r\n"}) + end + + test "invalid status lines", %{port: port} do + lines = [ + "HTTP/1.1 2000 OK\r\n", + "HTTP/1.1 99 OK\r\n", + "HTTP/2.0 200 OK\r\n", + "HTTP/0.9 200 OK\r\n", + "HTTP/1.10 200 OK\r\n", + "HTTP/1.1 200 O\0K\r\n", + "HTTP/1.1 200 OK\r\r\n", + "HTTP/1.1 200OK\r\n", + "HTTP/1.1 0200 OK\r\n", + "HTTP/01.1 200 OK\r\n", + "HTTP/1.01 200 OK\r\n" + ] + + for line <- lines do + assert {:ok, conn} = HTTP1.connect(:http, "localhost", port) + {:ok, conn, _ref} = HTTP1.request(conn, "GET", "/", [], nil) + + assert {:error, _conn, %HTTPError{reason: :invalid_status_line}, []} = + HTTP1.stream(conn, {:tcp, conn.socket, line}), + "expected #{inspect(line)} to be rejected" + end + end + test "limits an incomplete response status line", %{port: port} do assert {:ok, conn} = HTTP1.connect(:http, "localhost", port, max_header_list_size: 64) {:ok, conn, _ref} = HTTP1.request(conn, "GET", "/", [], nil) @@ -105,6 +137,124 @@ defmodule Mint.HTTP1Test do HTTP1.stream(conn, {:tcp, conn.socket, "foo: bar\r\n"}) end + test "header values with control characters are rejected", %{port: port} do + for value <- ["b\rar", "b\0ar", "b\x7Far", "b\x01ar"], stream_headers <- [false, true] do + assert {:ok, conn} = HTTP1.connect(:http, "localhost", port, stream_headers: stream_headers) + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + response = "HTTP/1.1 200 OK\r\nfoo: " <> value <> "\r\ncontent-length: 0\r\n\r\n" + + assert {:error, conn, %HTTPError{reason: :invalid_header}, [{:status, ^ref, 200}]} = + HTTP1.stream(conn, {:tcp, conn.socket, response}) + + assert_closed_and_released(conn) + end + end + + test "empty header names are rejected", %{port: port} do + for stream_headers <- [false, true] do + assert {:ok, conn} = HTTP1.connect(:http, "localhost", port, stream_headers: stream_headers) + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + response = "HTTP/1.1 200 OK\r\n: bar\r\ncontent-length: 0\r\n\r\n" + + assert {:error, conn, %HTTPError{reason: :invalid_header}, [{:status, ^ref, 200}]} = + HTTP1.stream(conn, {:tcp, conn.socket, response}) + + assert_closed_and_released(conn) + end + end + + test "empty trailer names are rejected", %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + + response = + "HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n" <> + "1\r\nX\r\n0\r\n: bar\r\n\r\n" + + assert {:error, conn, %HTTPError{reason: :invalid_trailer_header}, responses} = + HTTP1.stream(conn, {:tcp, conn.socket, response}) + + assert [{:status, ^ref, 200}, {:headers, ^ref, _}, {:data, ^ref, "X"}] = responses + assert_closed_and_released(conn) + end + + test "trailer values with control characters are rejected", %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + + response = + "HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n" <> + "1\r\nX\r\n0\r\nfoo: b\0ar\r\n\r\n" + + assert {:error, conn, %HTTPError{reason: :invalid_trailer_header}, responses} = + HTTP1.stream(conn, {:tcp, conn.socket, response}) + + assert [{:status, ^ref, 200}, {:headers, ^ref, _}, {:data, ^ref, "X"}] = responses + assert_closed_and_released(conn) + end + + test "trailing whitespace in header values is trimmed", %{port: port} do + for stream_headers <- [false, true] do + assert {:ok, conn} = HTTP1.connect(:http, "localhost", port, stream_headers: stream_headers) + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + + response = + "HTTP/1.1 200 OK\r\nfoo: bar \t \r\nbaz: \t\r\ntransfer-encoding: chunked \r\n\r\n" <> + "1\r\nX\r\n0\r\nmy-trailer: value\t\r\n\r\n" + + assert {:ok, _conn, responses} = HTTP1.stream(conn, {:tcp, conn.socket, response}) + + assert [ + {:status, ^ref, 200}, + {:headers, ^ref, headers}, + {:data, ^ref, "X"}, + {:headers, ^ref, trailers}, + {:done, ^ref} + ] = responses + + assert headers == [{"foo", "bar"}, {"baz", ""}, {"transfer-encoding", "chunked"}] + assert trailers == [{"my-trailer", "value"}] + end + end + + test "trailing whitespace in content-length is trimmed", %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + response = "HTTP/1.1 200 OK\r\ncontent-length: 1 \t\r\n\r\nX" + + assert {:ok, _conn, [_status, {:headers, ^ref, headers}, {:data, ^ref, "X"}, {:done, ^ref}]} = + HTTP1.stream(conn, {:tcp, conn.socket, response}) + + assert headers == [{"content-length", "1"}] + end + + test "header values with obs-text are accepted", %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + response = "HTTP/1.1 200 OK\r\nfoo: b\xC3\xA4r\r\ncontent-length: 0\r\n\r\n" + + assert {:ok, _conn, [{:status, ^ref, 200}, {:headers, ^ref, headers}, {:done, ^ref}]} = + HTTP1.stream(conn, {:tcp, conn.socket, response}) + + assert headers == [{"foo", "bär"}, {"content-length", "0"}] + end + + test "obsolete line folding in header values is replaced with a space", %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + + response = + "HTTP/1.1 200 OK\r\nFoo: bar\r\n baz\r\nBar: one\r\n\t two\n three\r\n" <> + "transfer-encoding: chunked\r\n\r\n0\r\nMy-Trailer: a\r\n b\r\n\r\n" + + assert {:ok, _conn, [status, headers, trailers, done]} = + HTTP1.stream(conn, {:tcp, conn.socket, response}) + + assert status == {:status, ref, 200} + + assert headers == + {:headers, ref, + [{"foo", "bar baz"}, {"bar", "one two three"}, {"transfer-encoding", "chunked"}]} + + assert trailers == {:headers, ref, [{"my-trailer", "a b"}]} + assert done == {:done, ref} + end + test "status and headers", %{conn: conn} do {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) @@ -1450,6 +1600,48 @@ defmodule Mint.HTTP1Test do end end + test "whitespace before an obs-fold is replaced with the fold", %{port: port} do + for stream_headers <- [false, true] do + assert {:ok, conn} = + HTTP1.connect(:http, "localhost", port, stream_headers: stream_headers) + + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + + assert {:ok, conn, _responses} = + HTTP1.stream(conn, {:tcp, conn.socket, "HTTP/1.1 200 OK\r\nFoo: one\t \r"}) + + assert {:ok, _conn, responses} = + HTTP1.stream(conn, {:tcp, conn.socket, "\n two\r\nContent-Length: 0\r\n\r\n"}) + + assert {"foo", "one two"} in for( + {:headers, ^ref, headers} <- responses, + h <- headers, + do: h + ) + end + end + + test "whitespace before an obs-fold in a trailer is replaced with the fold", %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + + response = + "HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n" <> + "0\r\nfoo: one\t\r\n two\r\n\r\n" + + assert {:ok, _conn, responses} = HTTP1.stream(conn, {:tcp, conn.socket, response}) + assert {:headers, ref, [{"foo", "one two"}]} in responses + end + + test "an obs-fold at the start of a header value leaves no leading whitespace", + %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + response = "HTTP/1.1 200 OK\r\nFoo:\r\n bar\r\nContent-Length: 0\r\n\r\n" + assert {:ok, _conn, responses} = HTTP1.stream(conn, {:tcp, conn.socket, response}) + + assert [{:status, ^ref, 200}, {:headers, ^ref, headers}, {:done, ^ref}] = responses + assert headers == [{"foo", "bar"}, {"content-length", "0"}] + end + defp request_string(string) do String.replace(string, "\n", "\r\n") end @@ -1515,6 +1707,26 @@ defmodule Mint.HTTP1Test do assert {:headers, ^ref, [{"qux", "Quux"}]} = headers2 end + test "replaces obsolete line folding received together with its header", %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + + assert {:ok, _conn, [_status, {:headers, ^ref, [{"foo", "bar baz"}]}]} = + HTTP1.stream( + conn, + {:tcp, conn.socket, "HTTP/1.1 200 OK\r\nFoo: bar\r\n baz\r\n\r\n"} + ) + end + + test "rejects a folded continuation line arriving after its header", %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + + assert {:ok, conn, [_status, {:headers, ^ref, [{"foo", "bar"}]}]} = + HTTP1.stream(conn, {:tcp, conn.socket, "HTTP/1.1 200 OK\r\nFoo: bar\r\n"}) + + assert {:error, _conn, %HTTPError{reason: :invalid_header}, []} = + HTTP1.stream(conn, {:tcp, conn.socket, " baz\r\n\r\n"}) + end + test "emits multiple headers from one packet together", %{conn: conn} do {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) diff --git a/test/mint/http1/parse_test.exs b/test/mint/http1/parse_test.exs index 2a680b46..8cbe0dbd 100644 --- a/test/mint/http1/parse_test.exs +++ b/test/mint/http1/parse_test.exs @@ -96,7 +96,27 @@ defmodule Mint.HTTP1.ParseTest do test "content_length_header/1" do assert content_length_header("0") == {:ok, 0} assert content_length_header("100") == {:ok, 100} - assert content_length_header("200 ") == {:ok, 200} + + assert content_length_header("200 ") == + {:error, {:invalid_content_length_header, "200 "}} + + assert content_length_header("200\t") == + {:error, {:invalid_content_length_header, "200\t"}} + + assert content_length_header("200\v") == + {:error, {:invalid_content_length_header, "200\v"}} + + assert content_length_header("200\f") == + {:error, {:invalid_content_length_header, "200\f"}} + + assert content_length_header("200\u00A0") == + {:error, {:invalid_content_length_header, "200\u00A0"}} + + assert content_length_header("200\u0085") == + {:error, {:invalid_content_length_header, "200\u0085"}} + + assert content_length_header("200\u3000") == + {:error, {:invalid_content_length_header, "200\u3000"}} assert content_length_header("foo") == {:error, {:invalid_content_length_header, "foo"}} diff --git a/test/mint/http1/response_test.exs b/test/mint/http1/response_test.exs new file mode 100644 index 00000000..7fef6be9 --- /dev/null +++ b/test/mint/http1/response_test.exs @@ -0,0 +1,198 @@ +defmodule Mint.HTTP1.ResponseTest do + use ExUnit.Case, async: true + use ExUnitProperties + + import Mint.HTTP1.Response, only: [decode_status_line: 1, decode_header: 1, decode_header: 2] + + describe "decode_status_line/1" do + test "valid status lines" do + assert decode_status_line("HTTP/1.1 200 OK\r\nrest") == {:ok, {{1, 1}, 200, "OK"}, "rest"} + + assert decode_status_line("HTTP/1.0 404 Not Found\r\n") == + {:ok, {{1, 0}, 404, "Not Found"}, ""} + + assert decode_status_line("HTTP/1.1 200 OK\nrest") == {:ok, {{1, 1}, 200, "OK"}, "rest"} + assert decode_status_line("HTTP/1.1 204\r\n") == {:ok, {{1, 1}, 204, ""}, ""} + assert decode_status_line("HTTP/1.1 204 \r\n") == {:ok, {{1, 1}, 204, ""}, ""} + assert decode_status_line("HTTP/1.1 200 OK \r\n") == {:ok, {{1, 1}, 200, "OK "}, ""} + assert decode_status_line("HTTP/1.1 200 OK\r\n") == {:ok, {{1, 1}, 200, " OK"}, ""} + assert decode_status_line("HTTP/1.1 200 A\tB\r\n") == {:ok, {{1, 1}, 200, "A\tB"}, ""} + assert decode_status_line("HTTP/1.1 200 caf\xE9\r\n") == {:ok, {{1, 1}, 200, "caf\xE9"}, ""} + assert decode_status_line("HTTP/1.9 999 X\r\n") == {:ok, {{1, 9}, 999, "X"}, ""} + end + + test "incomplete status lines" do + for line <- ["", "H", "HTTP/1.", "HTTP/1.1 20", "HTTP/1.1 200", "HTTP/1.1 200 OK\r"] do + assert decode_status_line(line) == :more, "expected #{inspect(line)} to need more data" + end + end + + test "invalid status lines" do + lines = [ + "\r\nHTTP/1.1 200 OK\r\n", + "HTTP/1.1 2000 OK\r\n", + "HTTP/1.1 99 OK\r\n", + "HTTP/1.1 099 OK\r\n", + "HTTP/1.1 0200 OK\r\n", + "HTTP/2.0 200 OK\r\n", + "HTTP/0.9 200 OK\r\n", + "HTTP/01.1 200 OK\r\n", + "HTTP/1.01 200 OK\r\n", + "HTTP/1.10 200 OK\r\n", + "http/1.1 200 OK\r\n", + "HTTP/1.1 200 OK\r\n", + "HTTP/1.1 200OK\r\n", + "HTTP/1.1 2x0 OK\r\n", + "HTTP/1.1 200 O\0K\r\n", + "HTTP/1.1 200 OK\r\r\n", + "HTTP/1.1 200 OK\rX\r\n" + ] + + for line <- lines do + assert decode_status_line(line) == :error, "expected #{inspect(line)} to be rejected" + end + end + end + + describe "decode_header/2" do + test "end of the header section" do + assert decode_header("\r\nbody") == {:ok, :eof, "body"} + assert decode_header("\nbody") == {:ok, :eof, "body"} + end + + test "lowercases names and trims values" do + assert decode_header("Content-Type: text/plain\r\nX") == + {:ok, {"content-type", "text/plain"}, "X"} + + assert decode_header("A: x y \t\r\nX") == {:ok, {"a", "x y"}, "X"} + assert decode_header("A:\r\nX") == {:ok, {"a", ""}, "X"} + assert decode_header("A: \t \r\nX") == {:ok, {"a", ""}, "X"} + + assert decode_header("Location: http://x.y:8080/p\r\nX") == + {:ok, {"location", "http://x.y:8080/p"}, "X"} + end + + test "accepts LF-only line endings" do + assert decode_header("A: b\nX") == {:ok, {"a", "b"}, "X"} + end + + test "replaces obsolete line folds with a space" do + assert decode_header("A: b\r\n c\r\nX") == {:ok, {"a", "b c"}, "X"} + assert decode_header("A: b \t\r\n\t c\nX") == {:ok, {"a", "b c"}, "X"} + assert decode_header("A:\r\n b\r\nX") == {:ok, {"a", "b"}, "X"} + assert decode_header("A: b\r\n \r\n c\r\nX") == {:ok, {"a", "b c"}, "X"} + end + + test "needs the next byte to know whether a line is folded" do + assert decode_header("A: b\r\n") == :more + assert decode_header("A: b\r\n c\r\n") == :more + assert decode_header("A: b\r\n", true) == {:ok, {"a", "b"}, ""} + assert decode_header("A: b\n", true) == {:ok, {"a", "b"}, ""} + end + + test "incomplete lines" do + line = "Content-Type: text/plain\r\nContent-Length: 12345\r\n\r\n" + + for size <- 0..(byte_size("Content-Type: text/plain\r\n") - 1) do + <> = line + assert decode_header(prefix) == :more, "prefix #{inspect(prefix)}" + end + + assert decode_header(line) == + {:ok, {"content-type", "text/plain"}, "Content-Length: 12345\r\n\r\n"} + end + + test "names and values across word boundaries" do + for name_size <- 1..20, value_size <- 0..20 do + name = String.duplicate("a", name_size) + value = String.duplicate("b", value_size) + + assert decode_header("#{name}: #{value}\r\nX") == {:ok, {name, value}, "X"} + + assert decode_header("#{name}:\t#{value}\tc\r\nX") == + {:ok, {name, String.trim_leading("#{value}\tc")}, "X"} + end + end + + test "all tchars are allowed in names" do + tchars = "!#$%&'*+-.^_`|~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + assert decode_header("#{tchars}: v\r\nX") == {:ok, {String.downcase(tchars), "v"}, "X"} + end + + test "invalid names" do + assert decode_header(": v\r\nX") == :error + assert decode_header(" A: v\r\nX") == :error + assert decode_header("no colon here\r\nX") == :error + + for separator <- ~c"\"(),/;<=>?@[\\]{} \t\x01\x7F" ++ [0x80, 0xFF], position <- 0..16 do + name = String.duplicate("a", position) <> <> <> "b" + assert decode_header("#{name}: v\r\nX") == :error, inspect(name) + end + end + + test "invalid values" do + assert decode_header("A: b\rc\r\nX") == :error + + for byte <- [0x00, 0x01, 0x0B, 0x1F, 0x7F], position <- 0..16 do + value = String.duplicate("v", position) <> <> <> "w" + assert decode_header("A: #{value}\r\nX") == :error, inspect(value) + end + end + + property "decodes generated header lines" do + tchars = ~c"!#$%&'*+-.^_`|~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + + vchar = one_of([integer(0x21..0x7E), integer(0x80..0xFF)]) + whitespace = string(~c" \t", max_length: 3) + word = map(list_of(vchar, min_length: 1, max_length: 12), &:erlang.list_to_binary/1) + line_end = member_of(["\r\n", "\n"]) + + check all name <- string(tchars, min_length: 1, max_length: 24), + lines <- + list_of(list_of({word, whitespace}, max_length: 4), + min_length: 1, + max_length: 3 + ), + leading <- whitespace, + trailing <- whitespace, + line_ends <- list_of(line_end, length: length(lines)), + fold_whitespace <- + list_of(string(~c" \t", min_length: 1, max_length: 3), length: length(lines)) do + segments = + Enum.map(lines, fn words -> Enum.map_join(words, fn {w, ws} -> w <> ws end) end) + + raw = + [segments, line_ends, fold_whitespace] + |> Enum.zip() + |> Enum.with_index() + |> Enum.map_join(fn {{segment, line_end, fold}, index} -> + prefix = if index == 0, do: "", else: fold + + prefix <> + segment <> if(index == length(segments) - 1, do: trailing, else: "") <> line_end + end) + + expected = + segments + |> Enum.map(&String.trim(&1, " ")) + |> Enum.map(&trim_ows/1) + |> Enum.reject(&(&1 == "")) + |> Enum.join(" ") + + assert decode_header(name <> ":" <> leading <> raw <> "X") == + {:ok, {String.downcase(name), expected}, "X"} + end + end + + test "obs-text and HTAB are allowed in values" do + for position <- 0..16 do + value = "x" <> String.duplicate("v", position) <> "\t\x80\xC3\xA9\xFFw" + assert decode_header("A: #{value}\r\nX") == {:ok, {"a", value}, "X"} + end + end + end + + defp trim_ows(value) do + value |> String.replace(~r/^[ \t]+/, "") |> String.replace(~r/[ \t]+$/, "") + end +end