Allow app to send arbitrary Content-Length/Transfer-Encoding in HEAD responses.

Previously, the app could control Content-Length by passing `expectedBodySize`. This is great for enabling code that "just works" by handling GET and HEAD requests identically.

However, in somewhat more-complicated situations -- especilaly in proxies -- you end up having to write special-case hacks for HEAD requests to deal with the fact that the body is actually empty, but has a non-zero "expected" size.

We can make life easier for proxies by allowing the application to directly set the Content-Length and Transfer-Encoding headers in the case of HEAD responses, much like we allow applications to set WebSocket-related headers on non-WebSocket requests/responses.

This change actually fixes a bug in Cloudflare Workers where Content-Length is not passed through correctly for HEAD responses. No changes are needed on the Workers side (except adding a test).

c++/src/kj/compat/http-test.c++

@@ -768,6 +768,20 @@ kj::ArrayPtr<const HttpResponseTestCase> responseTestCases() {
       HttpMethod::HEAD,
     },
 
+    {
+      "HTTP/1.1 200 OK\r\n"
+      "Content-Length: foobar\r\n"
+      "Content-Type: text/plain\r\n"
+      "\r\n",
+
+      200, "OK",
+      {{HttpHeaderId::CONTENT_TYPE, "text/plain"},
+       {HttpHeaderId::CONTENT_LENGTH, "foobar"}},
+      123, {},
+
+      HttpMethod::HEAD,
+    },
+
     {
       "HTTP/1.1 200 OK\r\n"
       "Content-Length: 8\r\n"

c++/src/kj/compat/http.c++

@@ -489,6 +489,7 @@ namespace BuiltinHeaderIndices {
 #undef HEADER_ID
 };
 
+constexpr uint HEAD_RESPONSE_CONNECTION_HEADERS_COUNT = BuiltinHeaderIndices::CONTENT_LENGTH;
 constexpr uint CONNECTION_HEADERS_COUNT = BuiltinHeaderIndices::SEC_WEBSOCKET_KEY;
 constexpr uint WEBSOCKET_CONNECTION_HEADERS_COUNT = BuiltinHeaderIndices::HOST;
 
@@ -3539,7 +3540,19 @@ private:
       connectionHeaders[BuiltinHeaderIndices::TRANSFER_ENCODING] = "chunked";
     }
 
-    httpOutput.writeHeaders(headers.serializeResponse(statusCode, statusText, connectionHeaders));
+    // For HEAD requests, if the application specified a Content-Length or Transfer-Encoding
+    // header, use that instead of whatever we decided above.
+    kj::ArrayPtr<kj::StringPtr> connectionHeadersArray = connectionHeaders;
+    if (method == HttpMethod::HEAD) {
+      if (headers.get(HttpHeaderId::CONTENT_LENGTH) != nullptr ||
+          headers.get(HttpHeaderId::TRANSFER_ENCODING) != nullptr) {
+        connectionHeadersArray = connectionHeadersArray
+            .slice(0, HEAD_RESPONSE_CONNECTION_HEADERS_COUNT);
+      }
+    }
+
+    httpOutput.writeHeaders(headers.serializeResponse(
+        statusCode, statusText, connectionHeadersArray));
 
     kj::Own<kj::AsyncOutputStream> bodyStream;
     if (method == HttpMethod::HEAD) {

c++/src/kj/compat/http.h

@@ -128,13 +128,15 @@ public:
 #define KJ_HTTP_FOR_EACH_BUILTIN_HEADER(MACRO) \
   /* Headers that are always read-only. */ \
   MACRO(CONNECTION, "Connection") \
-  MACRO(CONTENT_LENGTH, "Content-Length") \
   MACRO(KEEP_ALIVE, "Keep-Alive") \
   MACRO(TE, "TE") \
   MACRO(TRAILER, "Trailer") \
-  MACRO(TRANSFER_ENCODING, "Transfer-Encoding") \
   MACRO(UPGRADE, "Upgrade") \
   \
+  /* Headers that are read-only except in the case of a response to a HEAD request. */ \
+  MACRO(CONTENT_LENGTH, "Content-Length") \
+  MACRO(TRANSFER_ENCODING, "Transfer-Encoding") \
+  \
   /* Headers that are read-only for WebSocket handshakes. */ \
   MACRO(SEC_WEBSOCKET_KEY, "Sec-WebSocket-Key") \
   MACRO(SEC_WEBSOCKET_VERSION, "Sec-WebSocket-Version") \