Fix fuzz property gaps and a flaky HTTP/2 test

test/mint/http1/fuzz_test.exs

@@ -42,33 +42,45 @@ defmodule Mint.HTTP1.FuzzTest do
     ])
   end
 
+  # Returns the framing headers with the body length RFC 9112 6.3 gives them:
+  # an integer, :until_close, or nil when the headers don't describe the body.
+  # Chunked framing applies only when chunked is the final transfer coding.
   defp framing_headers_gen(body) do
     derived =
       case body do
         :none ->
-          [[], [{"content-length", "0"}]]
+          [{[], :until_close}, {[{"content-length", "0"}], 0}]
 
-        {:cl, _} ->
-          [[{"content-length", :match}]]
+        {:cl, bytes} ->
+          [{[{"content-length", :match}], byte_size(bytes)}]
+
+        {:chunked, chunks, _, _} ->
+          size = chunks |> Enum.map(&byte_size/1) |> Enum.sum()
 
-        {:chunked, _, _, _} ->
-          [[{"transfer-encoding", "chunked"}]]
+          [
+            {[{"transfer-encoding", "chunked"}], size},
+            {[{"transfer-encoding", "gzip, chunked"}], size},
+            {[{"transfer-encoding", "chunked, gzip"}], :until_close}
+          ]
 
         {:close, _} ->
-          [[], [{"connection", "close"}]]
+          [{[], :until_close}, {[{"connection", "close"}], :until_close}]
       end
 
     frequency([
       {9, member_of(derived)},
       {1,
-       member_of([
-         [{"content-length", "5"}],
-         [{"content-length", "-1"}],
-         [{"content-length", "2, 2"}],
-         [{"content-length", "3"}, {"content-length", "3"}],
-         [{"transfer-encoding", "chunked"}, {"content-length", "3"}],
-         [{"transfer-encoding", "chunked"}]
-       ])}
+       map(
+         member_of([
+           [{"content-length", "5"}],
+           [{"content-length", "-1"}],
+           [{"content-length", "2, 2"}],
+           [{"content-length", "3"}, {"content-length", "3"}],
+           [{"transfer-encoding", "chunked"}, {"content-length", "3"}],
+           [{"transfer-encoding", "chunked"}]
+         ]),
+         &{&1, nil}
+       )}
     ])
   end
 
@@ -88,14 +100,23 @@ defmodule Mint.HTTP1.FuzzTest do
                 {1, member_of(["600", "99", "101", "1000", "20"])}
               ]),
             body <- body_gen(),
-            framing <- framing_headers_gen(body),
+            {framing, body_length} <- framing_headers_gen(body),
             headers <- list_of(header_gen(), max_length: 3) do
+      # The framing headers give the body length only in 1.1 responses with a
+      # status that has a body, no 101 prelude and no extra framing headers.
+      known_length? =
+        version == "1.1" and status in ["200", "404", "500"] and
+          not List.keymember?(preludes, "101", 0) and
+          not List.keymember?(headers, "content-length", 0) and
+          not List.keymember?(headers, "transfer-encoding", 0)
+
       %{
         preludes: preludes,
         version: version,
         status: status,
         headers: framing ++ headers,
-        body: body
+        body: body,
+        body_length: if(known_length?, do: body_length)
       }
     end
   end
@@ -211,8 +232,10 @@ defmodule Mint.HTTP1.FuzzTest do
         end
       end)
 
-    bytes = responses |> Enum.map(&render_response/1) |> IO.iodata_to_binary()
+    rendered = Enum.map(responses, &render_response/1)
+    bytes = IO.iodata_to_binary(rendered)
     bytes = mutate(bytes <> scenario.extra, scenario.mutation)
+    meta = put_expected_body_sizes(meta, refs, scenario, rendered)
 
     state = %{
       scenario: scenario,
@@ -249,7 +272,38 @@ defmodule Mint.HTTP1.FuzzTest do
     drain_mailbox()
   end
 
-  defp new_meta(method), do: %{method: method, status: nil, content_length: nil, body_size: 0}
+  defp new_meta(method) do
+    %{method: method, status: nil, content_length: nil, body_size: 0, expected_body_size: nil}
+  end
+
+  # The body size of a response to an initial request is known when the bytes
+  # are sent unmutated, every initial request was issued, and the response and
+  # all the ones before it are to GET or POST and have a known body length. A
+  # body read until close takes every byte after the response's header
+  # section, so no later response has a known size.
+  defp put_expected_body_sizes(meta, refs, scenario, rendered) do
+    if scenario.mutation == nil and length(refs) == length(scenario.methods) do
+      responses = Enum.zip(scenario.responses, rendered)
+      sizes = expected_body_sizes(scenario.methods, responses, scenario.extra)
+
+      Enum.zip_reduce(refs, sizes, meta, fn ref, size, meta ->
+        put_in(meta[ref].expected_body_size, size)
+      end)
+    else
+      meta
+    end
+  end
+
+  defp expected_body_sizes([{method, _} | methods], [{response, [_head, body]} | rest], extra)
+       when method in ["GET", "POST"] do
+    case response.body_length do
+      nil -> []
+      :until_close -> [IO.iodata_length([body, Enum.map(rest, &elem(&1, 1)), extra])]
+      size -> [size | expected_body_sizes(methods, rest, extra)]
+    end
+  end
+
+  defp expected_body_sizes(_methods, _responses, _extra), do: []
 
   defp describe(state) do
     "scenario: #{inspect(state.scenario, limit: :infinity)}\nbytes: #{inspect(state.bytes, limit: :infinity)}\nlog: #{inspect(state.log, limit: :infinity)}"
@@ -308,9 +362,9 @@ defmodule Mint.HTTP1.FuzzTest do
 
   defp run_client_ops(conn, state) do
     ops =
-      for {index, op, arg1, arg2} <- state.scenario.ops,
-          index == state.index,
-          do: {op, arg1, arg2}
+      for op <- state.scenario.ops,
+          elem(op, 0) == state.index,
+          do: Tuple.delete_at(op, 0)
 
     Enum.reduce_while(ops, {:open, conn, state}, fn op, {:open, conn, state} ->
       state = %{state | log: state.log ++ [{:client, op}]}
@@ -497,8 +551,13 @@ defmodule Mint.HTTP1.FuzzTest do
   end
 
   defp update_meta(meta, {:done, ref}, _ref_state, state) when is_map_key(meta, ref) do
-    %{method: method, status: status, content_length: content_length, body_size: body_size} =
-      meta[ref]
+    %{
+      method: method,
+      status: status,
+      content_length: content_length,
+      body_size: body_size,
+      expected_body_size: expected_body_size
+    } = meta[ref]
 
     # A 101 response hands the connection to another protocol, whose bytes are
     # delivered as data whatever the request method was.
@@ -518,6 +577,11 @@ defmodule Mint.HTTP1.FuzzTest do
           "response completed with #{body_size} body bytes but content-length #{content_length}\n#{describe(state)}"
         )
 
+      expected_body_size != nil and body_size != expected_body_size ->
+        flunk(
+          "response completed with #{body_size} body bytes but its framing gives #{expected_body_size}\n#{describe(state)}"
+        )
+
       true ->
         meta
     end
@@ -567,14 +631,16 @@ defmodule Mint.HTTP1.FuzzTest do
       end
 
     [
-      prelude_text,
-      "HTTP/",
-      version,
-      " ",
-      status,
-      " Reason\r\n",
-      render_headers(headers, body_length),
-      "\r\n",
+      [
+        prelude_text,
+        "HTTP/",
+        version,
+        " ",
+        status,
+        " Reason\r\n",
+        render_headers(headers, body_length),
+        "\r\n"
+      ],
       body_bytes
     ]
   end

test/mint/http2/conn_test.exs

@@ -1957,8 +1957,10 @@ defmodule Mint.HTTP2Test do
 
   describe "interim responses (1xx)" do
     test "multiple before a single HEADERS", %{conn: conn} do
-      info_status1 = Enum.random(100..199)
-      info_status2 = Enum.random(100..199)
+      # 101 is rejected in HTTP/2 (RFC 9113 §8.6).
+      info_statuses = Enum.to_list(100..199) -- [101]
+      info_status1 = Enum.random(info_statuses)
+      info_status2 = Enum.random(info_statuses)
 
       {conn, ref} = open_request(conn)
 

test/mint/http2/fuzz_test.exs

@@ -11,6 +11,7 @@ defmodule Mint.HTTP2.FuzzTest do
 
   @recv_timeout 300
   @runs String.to_integer(System.get_env("FUZZ_RUNS", "10"))
+  @initial_window_size 65_535
 
   ## Generators
 
@@ -248,7 +249,7 @@ defmodule Mint.HTTP2.FuzzTest do
        ) do
     drain_mailbox()
     {:ok, port, task} = TestServer.listen_and_accept()
-    conn = start_connection(port, task, scenario)
+    {conn, preface_frames} = start_connection(port, task, scenario)
 
     {conn, refs, sids, meta} =
       Enum.reduce(requests, {conn, [], [], %{}}, fn {method, body}, {conn, refs, sids, meta} ->
@@ -289,10 +290,11 @@ defmodule Mint.HTTP2.FuzzTest do
       meta: meta,
       actions: actions,
       mode: mode,
-      window: %{server_view: conn.receive_window_remaining, buffer: "", exact?: true},
+      window: %{server_view: @initial_window_size, buffer: "", exact?: true},
       log: []
     }
 
+    state = Enum.reduce(preface_frames, state, &account_client_frame/2)
     run(conn, actions, state)
     _ = HTTP2.close(conn)
     _ = :ssl.close(state.server.socket)
@@ -342,8 +344,9 @@ defmodule Mint.HTTP2.FuzzTest do
     end
   end
 
-  # The server's view of the connection window is the initial window minus the
-  # DATA payloads it sent plus the WINDOW_UPDATE increments it received. Once
+  # The server's view of the connection window is the initial 65,535 bytes (RFC
+  # 9113 6.9.2) minus the DATA payloads it sent plus the WINDOW_UPDATE increments
+  # it received on stream 0, including the one after the client preface. Once
   # Mint has processed a segment it must agree with Mint's own view; raw bytes
   # can swallow later frames into a partial frame, so the check is skipped once
   # any were sent.
@@ -616,7 +619,8 @@ defmodule Mint.HTTP2.FuzzTest do
   end
 
   # RFC 9113 8.1.1: the body must match a valid content-length, and responses to
-  # HEAD, 204 and 304 responses have no content.
+  # HEAD, 204 and 304 responses have no content. A 2xx response to CONNECT opens a
+  # tunnel and the DATA after it is tunnel data (RFC 9110 9.3.6, RFC 9113 8.5).
   defp update_meta(meta, {:status, ref, status}, _ref_state, _state) when is_map_key(meta, ref) do
     put_in(meta[ref].status, status)
   end
@@ -642,16 +646,16 @@ defmodule Mint.HTTP2.FuzzTest do
     %{method: method, status: status, content_length: content_length, body_size: body_size} =
       meta[ref]
 
-    bodiless? = method == "HEAD" or status in [204, 304]
+    tunnel? = method == "CONNECT" and status in 200..299
+    bodiless? = method == "HEAD" or (status in [204, 304] and not tunnel?)
 
     cond do
       bodiless? and body_size > 0 ->
         flunk(
           "#{method} #{status} response completed with #{body_size} body bytes\n#{describe(state)}"
         )
 
-      not bodiless? and content_length != nil and body_size != content_length and
-          not (method == "CONNECT" and status in 200..299) ->
+      not bodiless? and not tunnel? and content_length != nil and body_size != content_length ->
         flunk(
           "response completed with #{body_size} body bytes but content-length #{content_length}\n#{describe(state)}"
         )
@@ -887,7 +891,23 @@ defmodule Mint.HTTP2.FuzzTest do
       )
 
     {:ok, server_socket} = Task.await(server_socket_task)
-    :ok = TestServer.perform_http2_handshake(server_socket)
+
+    {:ok, "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" <> rest} =
+      :ssl.recv(server_socket, 0, @recv_timeout)
+
+    # The client preface is a SETTINGS frame, followed by a WINDOW_UPDATE on
+    # stream 0 when the client raised its connection-level receive window.
+    {:ok, settings(flags: 0), rest} = Frame.decode_next(rest)
+
+    preface_frames =
+      case rest do
+        "" ->
+          []
+
+        _ ->
+          assert {:ok, window_update(stream_id: 0) = frame, ""} = Frame.decode_next(rest)
+          [frame]
+      end
 
     :ok =
       :ssl.send(server_socket, [
@@ -908,7 +928,7 @@ defmodule Mint.HTTP2.FuzzTest do
     {:ok, settings(flags: ^ack_flags, params: []), ""} = Frame.decode_next(data)
     :ok = :ssl.setopts(server_socket, active: true)
     Process.put(:fuzz_server, TestServer.new(server_socket))
-    conn
+    {conn, preface_frames}
   end
 
   defp recv_all_frames do

test/support/mint/http2/test_server.ex

@@ -121,6 +121,7 @@ defmodule Mint.HTTP2.TestServer do
       Task.async(fn ->
         # Let's accept a new connection.
         {:ok, socket} = :ssl.transport_accept(listen_socket)
+        :ok = :ssl.close(listen_socket)
 
         if function_exported?(:ssl, :handshake, 1) do
           {:ok, _} = apply(:ssl, :handshake, [socket])