Przeglądaj źródła

Add static-file, large-body and TLS workloads to the A/B benchmark

The harness had a single endpoint returning a 12-byte set_content() body.
That is the one case where the response line, the headers and the body
already share a single write(), so any change to the write path measured
as noise. Comparing a gather-write branch against its merge base reported
0.993x at p = 1.000 while the same branch moved static-file throughput by
a quarter and TLS throughput by nearly half in both directions.

The server now also serves a large set_content() body and small and large
files from a mount point, over HTTPS when a certificate is given, with
--path, --large-mib and --tls selecting the combination.

ab.sh now compiles the harness from the invoking worktree instead of each
ref's own copy, so both refs run an identical workload and a ref that
predates a harness change stays measurable. Only httplib.h varies, through
-I. --timeout is exposed because bombardier's 2s default aborts large TLS
responses, which then fails the non-2xx check.
yhirose 6 dni temu
rodzic
commit
6494edd8c0
2 zmienionych plików z 198 dodań i 12 usunięć
  1. 68 9
      benchmark/ab.sh
  2. 130 3
      benchmark/cpp-httplib/main.cpp

+ 68 - 9
benchmark/ab.sh

@@ -3,7 +3,25 @@
 # A/B throughput comparison between two git refs.
 #
 # Usage: ./ab.sh [--base REF] [--head REF] [--rounds N] [--duration S]
-#                [--connections N] [--threads N]
+#                [--connections N] [--threads N] [--path PATH] [--tls]
+#                [--large-mib N] [--timeout S]
+#
+# --path selects the workload. The harness serves:
+#   /                  small body via set_content(); the response line, the
+#                      headers and the body already share a single write(), so
+#                      this is the least sensitive case
+#   /large             large body via set_content()
+#   /static/small.js   1 KiB file from a mount point, where the headers and the
+#                      body are two separate writes
+#   /static/large.bin  same, with the body large enough to dominate
+#
+# --large-mib sizes the two large workloads (default 1).
+#
+# --tls runs the same workload over HTTPS, which writes through
+# SSLSocketStream instead of SocketStream.
+#
+# --timeout is bombardier's per-request timeout. Its 2s default aborts large
+# TLS responses, and the run then fails on the non-2xx check.
 #
 # Absolute numbers from a single run are meaningless: on a quiet 8-core laptop
 # the same binary varies by +/-20% run to run, and shared CI runners are worse.
@@ -21,6 +39,10 @@ DURATION="5s"
 CONNECTIONS=10
 THREADS=""
 PORT=8080
+REQ_PATH="/"
+TLS=0
+LARGE_MIB=1
+TIMEOUT="30s"
 
 while [ $# -gt 0 ]; do
   case "$1" in
@@ -30,6 +52,10 @@ while [ $# -gt 0 ]; do
     --duration) DURATION="$2"; shift 2 ;;
     --connections) CONNECTIONS="$2"; shift 2 ;;
     --threads) THREADS="$2"; shift 2 ;;
+    --path) REQ_PATH="$2"; shift 2 ;;
+    --large-mib) LARGE_MIB="$2"; shift 2 ;;
+    --timeout) TIMEOUT="$2"; shift 2 ;;
+    --tls) TLS=1; shift ;;
     *) echo "Unknown option: $1" >&2; exit 1 ;;
   esac
 done
@@ -62,6 +88,7 @@ HEAD_SHA=$(git -C "$REPO_ROOT" rev-parse --short "$HEAD_REF")
 echo "==> base: $BASE_REF ($BASE_SHA)"
 echo "==> head: $HEAD_REF ($HEAD_SHA)"
 echo "==> rounds=$ROUNDS duration=$DURATION connections=$CONNECTIONS threads=$THREADS"
+echo "==> path=$REQ_PATH tls=$TLS large=${LARGE_MIB}MiB"
 echo ""
 
 if [ "$BASE_SHA" = "$HEAD_SHA" ]; then
@@ -69,18 +96,49 @@ if [ "$BASE_SHA" = "$HEAD_SHA" ]; then
   echo ""
 fi
 
+# --- Toolchain bits that depend on --tls ---
+SCHEME="http"
+INSECURE=""
+TLS_CXXFLAGS=""
+TLS_LDFLAGS=""
+TLS_ARGS=""
+if [ "$TLS" = "1" ]; then
+  SCHEME="https"
+  INSECURE="-k"
+  TLS_CXXFLAGS="-DCPPHTTPLIB_OPENSSL_SUPPORT"
+  TLS_LDFLAGS="-lssl -lcrypto"
+  if command -v pkg-config >/dev/null 2>&1 && pkg-config --exists openssl; then
+    TLS_CXXFLAGS="$TLS_CXXFLAGS $(pkg-config --cflags openssl)"
+    TLS_LDFLAGS="$(pkg-config --libs openssl)"
+  elif command -v brew >/dev/null 2>&1 && brew --prefix openssl >/dev/null 2>&1; then
+    OPENSSL_PREFIX=$(brew --prefix openssl)
+    TLS_CXXFLAGS="$TLS_CXXFLAGS -I$OPENSSL_PREFIX/include"
+    TLS_LDFLAGS="-L$OPENSSL_PREFIX/lib -lssl -lcrypto"
+  fi
+  if [ "$(uname -s)" = "Darwin" ]; then
+    TLS_LDFLAGS="$TLS_LDFLAGS -framework CoreFoundation -framework Security"
+  fi
+  TLS_ARGS="--cert $REPO_ROOT/test/cert.pem --key $REPO_ROOT/test/key.pem"
+  for f in "$REPO_ROOT/test/cert.pem" "$REPO_ROOT/test/key.pem"; do
+    [ -f "$f" ] || { echo "Error: $f not found" >&2; exit 1; }
+  done
+fi
+
 # --- Build both refs ---
+# The harness source always comes from the invoking worktree, so both refs run
+# an identical workload and a ref that predates a harness change stays
+# measurable. Only httplib.h varies, through -I.
+HARNESS="$REPO_ROOT/benchmark/cpp-httplib/main.cpp"
+[ -f "$HARNESS" ] || { echo "Error: $HARNESS not found" >&2; exit 1; }
+
 build() {
   local name=$1 ref=$2
   git -C "$REPO_ROOT" worktree add --detach --quiet "$WORKDIR/$name" "$ref"
-  if [ ! -f "$WORKDIR/$name/benchmark/cpp-httplib/main.cpp" ]; then
-    echo "Error: benchmark/cpp-httplib/main.cpp missing in $ref" >&2
-    exit 1
-  fi
   "$CXX" -o "$WORKDIR/$name/server-ab" -O2 -std=c++11 \
     -I"$WORKDIR/$name" \
     -DCPPHTTPLIB_THREAD_POOL_COUNT="$THREADS" \
-    "$WORKDIR/$name/benchmark/cpp-httplib/main.cpp" -lpthread
+    $TLS_CXXFLAGS \
+    "$HARNESS" -lpthread $TLS_LDFLAGS
 }
 
 echo "==> Building..."
@@ -92,7 +150,8 @@ measure() {
   local name=$1
   local json rc
 
-  "$WORKDIR/$name/server-ab" >/dev/null 2>&1 &
+  "$WORKDIR/$name/server-ab" --port "$PORT" --dir "$WORKDIR/$name-www" \
+    --large-mib "$LARGE_MIB" $TLS_ARGS >/dev/null 2>&1 &
   local pid=$!
 
   # Wait for the listener (no dependency on nc)
@@ -103,8 +162,8 @@ measure() {
   done
 
   set +e
-  json=$(bombardier -c "$CONNECTIONS" -d "$DURATION" -o json -p r \
-    "http://127.0.0.1:$PORT/" 2>/dev/null)
+  json=$(bombardier -c "$CONNECTIONS" -d "$DURATION" -t "$TIMEOUT" -o json -p r $INSECURE \
+    "$SCHEME://127.0.0.1:$PORT$REQ_PATH" 2>/dev/null)
   rc=$?
   set -e
 

+ 130 - 3
benchmark/cpp-httplib/main.cpp

@@ -1,12 +1,139 @@
 #include "httplib.h"
+
+#include <cstdlib>
+#include <cstring>
+#include <fstream>
+#include <string>
+
+#ifndef _WIN32
+#include <sys/stat.h>
+#include <sys/types.h>
+#endif
+
 using namespace httplib;
 
-int main() {
-  Server svr;
+namespace {
 
+// The workloads differ in which part of the write path they exercise:
+//
+//   /                 small body set through set_content(); the response line,
+//                     the headers and the body already share a single write()
+//   /large            large body set through set_content(); the body is copied
+//                     into the header buffer before that single write().
+//                     --large-mib sets its size (and large.bin's).
+//   /static/small.js  small file served from a mount point, where the headers
+//                     and the body are two separate writes
+//   /static/large.bin same, with the body large enough to dominate
+//
+// Bodies are generated at startup so the repository carries no fixtures.
+
+const size_t SMALL_SIZE = 1024;
+const size_t LARGE_SIZE_DEFAULT_MIB = 1;
+
+std::string filler(size_t n) {
+  std::string s;
+  s.reserve(n);
+  while (s.size() < n) {
+    s += "0123456789abcdef";
+  }
+  s.resize(n);
+  return s;
+}
+
+bool write_file(const std::string &path, const std::string &content) {
+  std::ofstream f(path.c_str(), std::ios::binary);
+  f.write(content.data(), static_cast<std::streamsize>(content.size()));
+  return f.good();
+}
+
+std::string default_dir() {
+  const char *tmp = std::getenv("TMPDIR");
+  std::string base = tmp && *tmp ? tmp : "/tmp";
+  if (!base.empty() && base[base.size() - 1] == '/') {
+    base.erase(base.size() - 1);
+  }
+  return base + "/cpp-httplib-bench";
+}
+
+bool make_dir(const std::string &path) {
+#ifdef _WIN32
+  return _mkdir(path.c_str()) == 0 || errno == EEXIST;
+#else
+  return ::mkdir(path.c_str(), 0755) == 0 || errno == EEXIST;
+#endif
+}
+
+void setup(Server &svr, const std::string &large, const std::string &dir) {
   svr.Get("/", [](const Request &, Response &res) {
     res.set_content("Hello World!", "text/plain");
   });
 
-  svr.listen("0.0.0.0", 8080);
+  svr.Get("/large", [&large](const Request &, Response &res) {
+    res.set_content(large, "application/octet-stream");
+  });
+
+  svr.set_mount_point("/static", dir);
+}
+
+} // namespace
+
+int main(int argc, char *argv[]) {
+  int port = 8080;
+  std::string dir = default_dir();
+  std::string cert;
+  std::string key;
+  size_t large_mib = LARGE_SIZE_DEFAULT_MIB;
+
+  for (int i = 1; i < argc; i++) {
+    auto last = i + 1 < argc;
+    if (!std::strcmp(argv[i], "--port") && last) {
+      port = std::atoi(argv[++i]);
+    } else if (!std::strcmp(argv[i], "--large-mib") && last) {
+      large_mib = static_cast<size_t>(std::atoi(argv[++i]));
+    } else if (!std::strcmp(argv[i], "--dir") && last) {
+      dir = argv[++i];
+    } else if (!std::strcmp(argv[i], "--cert") && last) {
+      cert = argv[++i];
+    } else if (!std::strcmp(argv[i], "--key") && last) {
+      key = argv[++i];
+    } else {
+      std::fprintf(stderr,
+                   "usage: %s [--port N] [--dir PATH] [--large-mib N]"
+                   " [--cert PATH --key PATH]\n",
+                   argv[0]);
+      return 2;
+    }
+  }
+
+  if (!make_dir(dir)) {
+    std::fprintf(stderr, "cannot create %s\n", dir.c_str());
+    return 1;
+  }
+
+  auto large = filler(large_mib * 1024 * 1024);
+  if (!write_file(dir + "/small.js", filler(SMALL_SIZE)) ||
+      !write_file(dir + "/large.bin", large)) {
+    std::fprintf(stderr, "cannot write fixtures under %s\n", dir.c_str());
+    return 1;
+  }
+
+  if (!cert.empty()) {
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+    SSLServer svr(cert.c_str(), key.c_str());
+    if (!svr.is_valid()) {
+      std::fprintf(stderr, "cannot load %s / %s\n", cert.c_str(), key.c_str());
+      return 1;
+    }
+    setup(svr, large, dir);
+    svr.listen("0.0.0.0", port);
+    return 0;
+#else
+    std::fprintf(stderr, "built without CPPHTTPLIB_OPENSSL_SUPPORT\n");
+    return 1;
+#endif
+  }
+
+  Server svr;
+  setup(svr, large, dir);
+  svr.listen("0.0.0.0", port);
 }