Pass unrecognized Content-Encoding values through instead of failing (#2518)

Since 8bba34e, prepare_content_receiver() rejected every non-empty
Content-Encoding that create_decompressor() could not handle, conflating an
unrecognized content coding with a known one whose support was not compiled
in. A response labeled `Content-Encoding: UTF-8` -- a header some servers
misuse to advertise a character set -- therefore failed with 415, which the
client surfaced as the generic Error::Read. Before that commit the raw body
came back untouched.

Restore that behavior: reject only a coding cpp-httplib recognizes but was
not built with, and leave an unrecognized one (including "identity") alone.
Match codings case-insensitively while here, as RFC 9110 8.4.1 requires;
otherwise `Content-Encoding: GZIP` would look unrecognized and hand back a
still-compressed body.

open_stream() had the mirror-image problem. It silently passed the payload
through whenever create_decompressor() returned null, so a gzip response on
a build without zlib reached the caller still compressed, and it never
checked is_valid() -- gzip_decompressor::decompress() only asserts, so a
failed inflateInit2() was undefined behavior in release builds. It now
applies the same policy as the buffered path.

Add Error::UnsupportedContentEncoding so callers can tell this apart from a
read failure, and report an unusable decompressor as Error::Compression. The
status read_content() writes was previously discarded into an uninitialized
dummy_status.

README.md

@@ -936,6 +936,7 @@ enum class Error {
   UnsupportedAddressFamily,
   HTTPParsing,
   InvalidRangeHeader,
+  UnsupportedContentEncoding,
 };
 ```
 

httplib.h

@@ -1514,6 +1514,7 @@ enum class Error {
   UnsupportedAddressFamily,
   HTTPParsing,
   InvalidRangeHeader,
+  UnsupportedContentEncoding,
 
   // For internal use only
   SSLPeerCouldBeClosed_,
@@ -7121,19 +7122,49 @@ inline bool zstd_decompressor::decompress(const char *data, size_t data_length,
 }
 #endif
 
+inline bool contains_case_ignore(const std::string &s, const char *token) {
+  auto token_end = token + std::strlen(token);
+  return std::search(s.begin(), s.end(), token, token_end, [](char a, char b) {
+           return case_ignore::to_lower(a) == case_ignore::to_lower(b);
+         }) != s.end();
+}
+
+// Content codings are case-insensitive (RFC 9110 8.4.1). Matching them
+// case-sensitively would make a response labeled e.g. "GZIP" look like an
+// unknown coding, and its payload would be handed back still compressed.
+inline bool is_zlib_encoding(const std::string &encoding) {
+  return case_ignore::equal(encoding, "gzip") ||
+         case_ignore::equal(encoding, "deflate");
+}
+
+inline bool is_brotli_encoding(const std::string &encoding) {
+  return contains_case_ignore(encoding, "br");
+}
+
+inline bool is_zstd_encoding(const std::string &encoding) {
+  return contains_case_ignore(encoding, "zstd");
+}
+
+// Returns true if the content coding is one cpp-httplib is able to decompress
+// when the corresponding support is compiled in.
+inline bool is_known_content_encoding(const std::string &encoding) {
+  return is_zlib_encoding(encoding) || is_brotli_encoding(encoding) ||
+         is_zstd_encoding(encoding);
+}
+
 inline std::unique_ptr<decompressor>
 create_decompressor(const std::string &encoding) {
   std::unique_ptr<decompressor> decompressor;
 
-  if (encoding == "gzip" || encoding == "deflate") {
+  if (is_zlib_encoding(encoding)) {
 #ifdef CPPHTTPLIB_ZLIB_SUPPORT
     decompressor = detail::make_unique<gzip_decompressor>();
 #endif
-  } else if (encoding.find("br") != std::string::npos) {
+  } else if (is_brotli_encoding(encoding)) {
 #ifdef CPPHTTPLIB_BROTLI_SUPPORT
     decompressor = detail::make_unique<brotli_decompressor>();
 #endif
-  } else if (encoding == "zstd" || encoding.find("zstd") != std::string::npos) {
+  } else if (is_zstd_encoding(encoding)) {
 #ifdef CPPHTTPLIB_ZSTD_SUPPORT
     decompressor = detail::make_unique<zstd_decompressor>();
 #endif
@@ -7470,9 +7501,12 @@ bool prepare_content_receiver(T &x, int &status,
     std::unique_ptr<decompressor> decompressor;
 
     if (!encoding.empty()) {
+      // A coding we know about but were not built with is an error. An
+      // unrecognized coding (including "identity") is left alone and the
+      // payload is passed through as-is, since some servers misuse the header,
+      // e.g. by sending a character set such as "Content-Encoding: UTF-8".
       decompressor = detail::create_decompressor(encoding);
-      if (!decompressor) {
-        // Unsupported encoding or no support compiled in
+      if (!decompressor && detail::is_known_content_encoding(encoding)) {
         status = StatusCode::UnsupportedMediaType_415;
         return false;
       }
@@ -9776,6 +9810,7 @@ inline std::string to_string(const Error error) {
   case Error::UnsupportedAddressFamily: return "Unsupported address family";
   case Error::HTTPParsing: return "HTTP parsing failed";
   case Error::InvalidRangeHeader: return "Invalid Range header";
+  case Error::UnsupportedContentEncoding: return "Unsupported Content-Encoding";
   default: break;
   }
 
@@ -13436,7 +13471,20 @@ ClientImpl::open_stream(const std::string &method, const std::string &path,
 
   auto content_encoding = handle.response->get_header_value("Content-Encoding");
   if (!content_encoding.empty()) {
+    // Same policy as prepare_content_receiver(): reject a coding we know about
+    // but were not built with, pass an unrecognized one through as-is.
     handle.decompressor_ = detail::create_decompressor(content_encoding);
+    if (!handle.decompressor_) {
+      if (detail::is_known_content_encoding(content_encoding)) {
+        handle.error = Error::UnsupportedContentEncoding;
+        handle.response.reset();
+        return handle;
+      }
+    } else if (!handle.decompressor_->is_valid()) {
+      handle.error = Error::Compression;
+      handle.response.reset();
+      return handle;
+    }
   }
 
   return handle;
@@ -14397,14 +14445,26 @@ inline bool ClientImpl::process_request(Stream &strm, Request &req,
     }
 
     if (res.status != StatusCode::NotModified_304) {
-      int dummy_status;
+      auto content_status = 0;
       auto max_length = (!has_payload_max_length_ && req.content_receiver)
                             ? (std::numeric_limits<size_t>::max)()
                             : payload_max_length_;
-      if (!detail::read_content(strm, res, max_length, dummy_status,
+      if (!detail::read_content(strm, res, max_length, content_status,
                                 std::move(progress), std::move(out),
                                 decompress_)) {
-        if (error != Error::Canceled) { error = Error::Read; }
+        if (error != Error::Canceled) {
+          // Tell the caller apart from a plain read failure when the body could
+          // not be decoded because of its Content-Encoding.
+          switch (content_status) {
+          case StatusCode::UnsupportedMediaType_415:
+            error = Error::UnsupportedContentEncoding;
+            break;
+          case StatusCode::InternalServerError_500:
+            error = Error::Compression;
+            break;
+          default: error = Error::Read; break;
+          }
+        }
         output_error_log(error, &req);
         return false;
       }

test/test.cc

@@ -10433,6 +10433,146 @@ TEST(PayloadLimitBypassTest, StreamingGzipDecompression) {
 }
 #endif
 
+// Some servers misuse Content-Encoding to advertise a character set, e.g.
+// `Content-Encoding: UTF-8` on a JPEG. Such a value is not a content coding, so
+// the payload must be passed through untouched instead of being rejected.
+TEST(ContentEncodingTest, UnknownEncodingIsPassedThrough) {
+  const std::string body = "\xff\xd8\xff\xe0 not really a jpeg";
+
+  Server svr;
+
+  svr.Get("/image", [&](const Request & /*req*/, Response &res) {
+    res.set_content(body, "image/jpeg");
+    res.set_header("Content-Encoding", "UTF-8");
+  });
+
+  svr.Get("/identity", [&](const Request & /*req*/, Response &res) {
+    res.set_content(body, "image/jpeg");
+    res.set_header("Content-Encoding", "identity");
+  });
+
+  svr.Post("/echo", [](const Request &req, Response &res) {
+    res.set_content(req.body, "image/jpeg");
+  });
+
+  thread t = thread([&]() { svr.listen(HOST, PORT); });
+  auto se = detail::scope_exit([&] {
+    svr.stop();
+    t.join();
+    ASSERT_FALSE(svr.is_running());
+  });
+
+  svr.wait_until_ready();
+
+  Client cli(HOST, PORT);
+
+  {
+    auto res = cli.Get("/image");
+    ASSERT_TRUE(res) << "Error: " << to_string(res.error());
+    EXPECT_EQ(StatusCode::OK_200, res->status);
+    EXPECT_EQ(body, res->body);
+  }
+
+  {
+    auto res = cli.Get("/identity");
+    ASSERT_TRUE(res) << "Error: " << to_string(res.error());
+    EXPECT_EQ(StatusCode::OK_200, res->status);
+    EXPECT_EQ(body, res->body);
+  }
+
+  {
+    // The same applies to a request body reaching the server.
+    Headers headers = {{"Content-Encoding", "UTF-8"}};
+    auto res = cli.Post("/echo", headers, body, "image/jpeg");
+    ASSERT_TRUE(res) << "Error: " << to_string(res.error());
+    EXPECT_EQ(StatusCode::OK_200, res->status);
+    EXPECT_EQ(body, res->body);
+  }
+}
+
+// "Hello World!" as gzip. Hard-coded so that the test below can serve a
+// gzip-encoded response even when the build has no zlib support.
+static const char GZIPPED_HELLO_WORLD[] = {
+    '\x1f', '\x8b', '\x08', '\x00', '\x00', '\x00', '\x00', '\x00',
+    '\x02', '\x03', '\xf3', '\x48', '\xcd', '\xc9', '\xc9', '\x57',
+    '\x08', '\xcf', '\x2f', '\xca', '\x49', '\x51', '\x04', '\x00',
+    '\xa3', '\x1c', '\x29', '\x1c', '\x0c', '\x00', '\x00', '\x00'};
+
+// A content coding cpp-httplib recognizes but was not built with must be
+// reported as such. Handing the still-compressed payload back to the caller
+// would silently corrupt it.
+TEST(ContentEncodingTest, KnownEncodingWithoutSupportIsReported) {
+  const std::string gzipped(GZIPPED_HELLO_WORLD, sizeof(GZIPPED_HELLO_WORLD));
+
+  Server svr;
+
+  // "image/jpeg" keeps the server from applying a content coding of its own,
+  // so the hand-crafted Content-Encoding below survives.
+  svr.Get("/gzipped", [&](const Request & /*req*/, Response &res) {
+    res.set_content(gzipped, "image/jpeg");
+    res.set_header("Content-Encoding", "gzip");
+  });
+
+  // Content codings are case-insensitive (RFC 9110 8.4.1).
+  svr.Get("/gzipped-uppercase", [&](const Request & /*req*/, Response &res) {
+    res.set_content(gzipped, "image/jpeg");
+    res.set_header("Content-Encoding", "GZIP");
+  });
+
+  thread t = thread([&]() { svr.listen(HOST, PORT); });
+  auto se = detail::scope_exit([&] {
+    svr.stop();
+    t.join();
+    ASSERT_FALSE(svr.is_running());
+  });
+
+  svr.wait_until_ready();
+
+  Client cli(HOST, PORT);
+
+  {
+    auto res = cli.Get("/gzipped");
+#ifdef CPPHTTPLIB_ZLIB_SUPPORT
+    ASSERT_TRUE(res) << "Error: " << to_string(res.error());
+    EXPECT_EQ("Hello World!", res->body);
+#else
+    ASSERT_FALSE(res);
+    EXPECT_EQ(Error::UnsupportedContentEncoding, res.error());
+#endif
+  }
+
+  {
+    // open_stream() must behave the same way.
+    auto handle = cli.open_stream("GET", "/gzipped");
+#ifdef CPPHTTPLIB_ZLIB_SUPPORT
+    ASSERT_TRUE(handle.is_valid());
+    std::string received;
+    char buf[256];
+    ssize_t n;
+    while ((n = handle.read(buf, sizeof(buf))) > 0) {
+      received.append(buf, static_cast<size_t>(n));
+    }
+    EXPECT_EQ("Hello World!", received);
+#else
+    EXPECT_FALSE(handle.is_valid());
+    EXPECT_EQ(Error::UnsupportedContentEncoding, handle.error);
+#endif
+  }
+
+  {
+    // A differently-cased coding must not be mistaken for an unknown one, or
+    // the body would be handed back still compressed.
+    auto res = cli.Get("/gzipped-uppercase");
+#ifdef CPPHTTPLIB_ZLIB_SUPPORT
+    ASSERT_TRUE(res) << "Error: " << to_string(res.error());
+    EXPECT_EQ("Hello World!", res->body);
+#else
+    ASSERT_FALSE(res);
+    EXPECT_EQ(Error::UnsupportedContentEncoding, res.error());
+#endif
+  }
+}
+
 // Regression test for DoS vulnerability: a malicious server sending a response
 // without Content-Length header must not cause unbounded memory consumption on
 // the client side. The client should stop reading after a reasonable limit,
@@ -16292,6 +16432,10 @@ protected:
     svr_.Get("/large", [](const Request &, Response &res) {
       res.set_content(std::string(10000, 'X'), "text/plain");
     });
+    svr_.Get("/unknown-encoding", [](const Request &, Response &res) {
+      res.set_content("Hello World!", "image/jpeg");
+      res.set_header("Content-Encoding", "UTF-8");
+    });
     svr_.Get("/chunked", [](const Request &, Response &res) {
       res.set_chunked_content_provider("text/plain",
                                        [](size_t offset, DataSink &sink) {
@@ -16380,6 +16524,13 @@ TEST_F(OpenStreamTest, Basic) {
   EXPECT_EQ("Hello World!", read_all(handle));
 }
 
+TEST_F(OpenStreamTest, UnknownContentEncodingIsPassedThrough) {
+  Client cli("127.0.0.1", port_);
+  auto handle = cli.open_stream("GET", "/unknown-encoding");
+  ASSERT_TRUE(handle.is_valid());
+  EXPECT_EQ("Hello World!", read_all(handle));
+}
+
 TEST_F(OpenStreamTest, SmallBuffer) {
   Client cli("127.0.0.1", port_);
   auto handle = cli.open_stream("GET", "/hello");