Add benchmark tests and related configurations for performance evaluation

.github/workflows/test.yaml

@@ -228,7 +228,7 @@ jobs:
         for ($i = 0; $i -lt $shards; $i++) {
           $log = "shard_${i}.log"
           $procs += Start-Process -FilePath ./Release/httplib-test.exe `
-            -ArgumentList "--gtest_color=yes","--gtest_filter=${{ github.event.inputs.gtest_filter || '*' }}-*BenchmarkTest*" `
+            -ArgumentList "--gtest_color=yes","--gtest_filter=${{ github.event.inputs.gtest_filter || '*' }}" `
             -NoNewWindow -PassThru -RedirectStandardOutput $log -RedirectStandardError "${log}.err" `
             -Environment @{ GTEST_TOTAL_SHARDS="$shards"; GTEST_SHARD_INDEX="$i" }
         }
@@ -248,9 +248,6 @@ jobs:
         }
         if ($failed) { exit 1 }
         Write-Host "All shards passed."
-    - name: Run benchmark tests with retry ${{ matrix.config.name }}
-      if: ${{ matrix.config.run_tests }}
-      run: ctest --output-on-failure --test-dir build -C Release -R "BenchmarkTest" --repeat until-pass:5
 
     env:
       VCPKG_ROOT: "C:/vcpkg"

.github/workflows/test_benchmark.yaml

@@ -0,0 +1,79 @@
+name: benchmark
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  ubuntu:
+    runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+      - name: build and run
+        run: cd test && make test_benchmark && ./test_benchmark
+
+  macos:
+    runs-on: macos-latest
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+      - name: build and run
+        run: cd test && make test_benchmark && ./test_benchmark
+
+  windows:
+    runs-on: windows-latest
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
+    steps:
+      - name: Prepare Git for Checkout on Windows
+        run: |
+          git config --global core.autocrlf false
+          git config --global core.eol lf
+      - name: checkout
+        uses: actions/checkout@v4
+      - name: Export GitHub Actions cache environment variables
+        uses: actions/github-script@v7
+        with:
+          script: |
+            core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
+            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
+      - name: Cache vcpkg packages
+        id: vcpkg-cache
+        uses: actions/cache@v4
+        with:
+          path: C:/vcpkg/installed
+          key: vcpkg-installed-windows-gtest
+      - name: Install vcpkg dependencies
+        if: steps.vcpkg-cache.outputs.cache-hit != 'true'
+        run: vcpkg install gtest
+      - name: Configure and build
+        shell: pwsh
+        run: |
+          $cmake_content = @"
+          cmake_minimum_required(VERSION 3.14)
+          project(httplib-benchmark CXX)
+          find_package(GTest REQUIRED)
+          add_executable(httplib-benchmark test/test_benchmark.cc)
+          target_include_directories(httplib-benchmark PRIVATE .)
+          target_link_libraries(httplib-benchmark PRIVATE GTest::gtest_main)
+          target_compile_options(httplib-benchmark PRIVATE "$<$<CXX_COMPILER_ID:MSVC>:/utf-8>")
+          "@
+          New-Item -ItemType Directory -Force -Path build_bench/test | Out-Null
+          Set-Content -Path build_bench/CMakeLists.txt -Value $cmake_content
+          Copy-Item -Path httplib.h -Destination build_bench/
+          Copy-Item -Path test/test_benchmark.cc -Destination build_bench/test/
+          cmake -B build_bench/build -S build_bench `
+            -DCMAKE_TOOLCHAIN_FILE="$env:VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake"
+          cmake --build build_bench/build --config Release
+      - name: Run with retry
+        run: ctest --output-on-failure --test-dir build_bench/build -C Release --repeat until-pass:5
+    env:
+      VCPKG_ROOT: "C:/vcpkg"
+      VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite"

.gitignore

@@ -51,6 +51,7 @@ test/test_split_wolfssl
 test/test_split_no_tls
 test/test_websocket_heartbeat
 test/test_thread_pool
+test/test_benchmark
 test/test.xcodeproj/xcuser*
 test/test.xcodeproj/*/xcuser*
 test/*.o

justfile

@@ -36,6 +36,7 @@ others:
     @(cd test && make fuzz_test)
     @(cd test && make test_websocket_heartbeat && ./test_websocket_heartbeat)
     @(cd test && make test_thread_pool && ./test_thread_pool)
+    @(cd test && make test_benchmark && ./test_benchmark)
 
 build:
     @(cd test && make test_split)

test/Makefile

@@ -219,6 +219,16 @@ style_check: $(STYLE_CHECK_FILES)
 		echo "All files are properly formatted."; \
 	fi
 
+BENCHMARK_LIBS = -lpthread
+ifneq ($(OS), Windows_NT)
+	ifeq ($(shell uname -s), Darwin)
+		BENCHMARK_LIBS += -framework CoreFoundation -framework CFNetwork
+	endif
+endif
+
+test_benchmark : test_benchmark.cc ../httplib.h Makefile
+	$(CXX) -o $@ -I.. $(CXXFLAGS) test_benchmark.cc gtest/src/gtest-all.cc gtest/src/gtest_main.cc -Igtest -Igtest/include $(BENCHMARK_LIBS)
+
 test_websocket_heartbeat : test_websocket_heartbeat.cc ../httplib.h Makefile
 	$(CXX) -o $@ -I.. $(CXXFLAGS) test_websocket_heartbeat.cc $(TEST_ARGS)
 	@file $@
@@ -254,5 +264,5 @@ cert.pem:
 	./gen-certs.sh
 
 clean:
-	rm -rf test test_split test_mbedtls test_split_mbedtls test_wolfssl test_split_wolfssl test_no_tls, test_split_no_tls test_proxy test_proxy_mbedtls test_proxy_wolfssl server_fuzzer *.pem *.0 *.o *.1 *.srl httplib.h httplib.cc _build* *.dSYM *_shard_*.log cpp-httplib
+	rm -rf test test_split test_mbedtls test_split_mbedtls test_wolfssl test_split_wolfssl test_no_tls, test_split_no_tls test_proxy test_proxy_mbedtls test_proxy_wolfssl test_benchmark server_fuzzer *.pem *.0 *.o *.1 *.srl httplib.h httplib.cc _build* *.dSYM *_shard_*.log cpp-httplib
 

test/test.cc

@@ -102,71 +102,6 @@ static void read_file(const std::string &path, std::string &out) {
   fs.read(&out[0], static_cast<std::streamsize>(size));
 }
 
-void performance_test(const char *host) {
-  Server svr;
-
-  svr.Get("/benchmark", [&](const Request & /*req*/, Response &res) {
-    res.set_content("Benchmark Response", "text/plain");
-  });
-
-  auto listen_thread = std::thread([&]() { svr.listen(host, PORT); });
-  auto se = detail::scope_exit([&] {
-    svr.stop();
-    listen_thread.join();
-    ASSERT_FALSE(svr.is_running());
-  });
-
-  svr.wait_until_ready();
-
-  Client cli(host, PORT);
-
-  // Warm-up request to establish connection and resolve DNS
-  auto warmup_res = cli.Get("/benchmark");
-  ASSERT_TRUE(warmup_res); // Ensure server is responding correctly
-
-  // Run multiple trials and collect timings
-  const int num_trials = 20;
-  std::vector<int64_t> timings;
-  timings.reserve(num_trials);
-
-  for (int i = 0; i < num_trials; i++) {
-    auto start = std::chrono::high_resolution_clock::now();
-    auto res = cli.Get("/benchmark");
-    auto end = std::chrono::high_resolution_clock::now();
-
-    auto elapsed =
-        std::chrono::duration_cast<std::chrono::milliseconds>(end - start)
-            .count();
-
-    // Assertions after timing measurement to avoid overhead
-    ASSERT_TRUE(res);
-    EXPECT_EQ(StatusCode::OK_200, res->status);
-
-    timings.push_back(elapsed);
-  }
-
-  // Calculate 25th percentile (lower quartile)
-  std::sort(timings.begin(), timings.end());
-  auto p25 = timings[num_trials / 4];
-
-  // Format timings for output
-  std::ostringstream timings_str;
-  timings_str << "[";
-  for (size_t i = 0; i < timings.size(); i++) {
-    if (i > 0) timings_str << ", ";
-    timings_str << timings[i];
-  }
-  timings_str << "]";
-
-  // Localhost HTTP GET should be fast even in CI environments
-  EXPECT_LE(p25, 5) << "25th percentile performance is too slow: " << p25
-                    << "ms (Issue #1777). Timings: " << timings_str.str();
-}
-
-TEST(BenchmarkTest, localhost) { performance_test("localhost"); }
-
-TEST(BenchmarkTest, v6) { performance_test("::1"); }
-
 class UnixSocketTest : public ::testing::Test {
 protected:
   void TearDown() override { std::remove(pathname_.c_str()); }

test/test_benchmark.cc

@@ -0,0 +1,78 @@
+#include <httplib.h>
+
+#include <gtest/gtest.h>
+
+#include <algorithm>
+#include <chrono>
+#include <sstream>
+#include <thread>
+#include <vector>
+
+using namespace httplib;
+
+static const int PORT = 11134;
+
+static void performance_test(const char *host) {
+  Server svr;
+
+  svr.Get("/benchmark", [&](const Request & /*req*/, Response &res) {
+    res.set_content("Benchmark Response", "text/plain");
+  });
+
+  auto listen_thread = std::thread([&]() { svr.listen(host, PORT); });
+  auto se = detail::scope_exit([&] {
+    svr.stop();
+    listen_thread.join();
+    ASSERT_FALSE(svr.is_running());
+  });
+
+  svr.wait_until_ready();
+
+  Client cli(host, PORT);
+
+  // Warm-up request to establish connection and resolve DNS
+  auto warmup_res = cli.Get("/benchmark");
+  ASSERT_TRUE(warmup_res); // Ensure server is responding correctly
+
+  // Run multiple trials and collect timings
+  const int num_trials = 20;
+  std::vector<int64_t> timings;
+  timings.reserve(num_trials);
+
+  for (int i = 0; i < num_trials; i++) {
+    auto start = std::chrono::high_resolution_clock::now();
+    auto res = cli.Get("/benchmark");
+    auto end = std::chrono::high_resolution_clock::now();
+
+    auto elapsed =
+        std::chrono::duration_cast<std::chrono::milliseconds>(end - start)
+            .count();
+
+    // Assertions after timing measurement to avoid overhead
+    ASSERT_TRUE(res);
+    EXPECT_EQ(StatusCode::OK_200, res->status);
+
+    timings.push_back(elapsed);
+  }
+
+  // Calculate 25th percentile (lower quartile)
+  std::sort(timings.begin(), timings.end());
+  auto p25 = timings[num_trials / 4];
+
+  // Format timings for output
+  std::ostringstream timings_str;
+  timings_str << "[";
+  for (size_t i = 0; i < timings.size(); i++) {
+    if (i > 0) timings_str << ", ";
+    timings_str << timings[i];
+  }
+  timings_str << "]";
+
+  // Localhost HTTP GET should be fast even in CI environments
+  EXPECT_LE(p25, 5) << "25th percentile performance is too slow: " << p25
+                    << "ms (Issue #1777). Timings: " << timings_str.str();
+}
+
+TEST(BenchmarkTest, localhost) { performance_test("localhost"); }
+
+TEST(BenchmarkTest, v6) { performance_test("::1"); }