#!/usr/bin/env bash set -euo pipefail # ============================================================================= # test_book.sh — LLM App Tutorial (Ch1–Ch5) E2E Test # # Code is extracted from the doc markdown files ( and # markers), so tests always stay in sync with the docs. # ============================================================================= SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" DOCS_DIR="$PROJECT_ROOT/docs-src/pages/ja/llm-app" WORKDIR=$(mktemp -d) MODEL_NAME="gemma-2-2b-it-Q4_K_M.gguf" MODEL_URL="https://huggingface.co/bartowski/gemma-2-2b-it-GGUF/resolve/main/${MODEL_NAME}" PORT=18080 GECKODRIVER_PORT=4444 SERVER_PID="" GECKODRIVER_PID="" PASS_COUNT=0 FAIL_COUNT=0 # --------------------------------------------------------------------------- # Cleanup # --------------------------------------------------------------------------- cleanup() { if [[ -n "$SERVER_PID" ]]; then kill "$SERVER_PID" 2>/dev/null || true wait "$SERVER_PID" 2>/dev/null || true fi if [[ -n "$GECKODRIVER_PID" ]]; then kill "$GECKODRIVER_PID" 2>/dev/null || true wait "$GECKODRIVER_PID" 2>/dev/null || true fi rm -rf "$WORKDIR" } trap cleanup EXIT # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- log() { echo "=== $*"; } pass() { echo " PASS: $*"; PASS_COUNT=$((PASS_COUNT + 1)); } fail() { echo " FAIL: $*"; FAIL_COUNT=$((FAIL_COUNT + 1)); } source "$SCRIPT_DIR/extract_code.sh" wait_for_server() { local max_wait=30 local i=0 while ! curl -s -o /dev/null "http://127.0.0.1:${PORT}/" 2>/dev/null; do sleep 1 i=$((i + 1)) if [[ $i -ge $max_wait ]]; then fail "Server did not start within ${max_wait}s" return 1 fi done } stop_server() { if [[ -n "$SERVER_PID" ]]; then kill "$SERVER_PID" 2>/dev/null || true wait "$SERVER_PID" 2>/dev/null || true SERVER_PID="" fi } # Make an HTTP request and capture status + body # Usage: http_request METHOD PATH [DATA] # Sets: HTTP_STATUS, HTTP_BODY http_request() { local method="$1" path="$2" data="${3:-}" local tmp tmp=$(mktemp) if [[ -n "$data" ]]; then HTTP_STATUS=$(curl -s -o "$tmp" -w '%{http_code}' \ -X "$method" "http://127.0.0.1:${PORT}${path}" \ -H "Content-Type: application/json" \ -d "$data") else HTTP_STATUS=$(curl -s -o "$tmp" -w '%{http_code}' \ -X "$method" "http://127.0.0.1:${PORT}${path}") fi HTTP_BODY=$(cat "$tmp") rm -f "$tmp" } # Make an SSE request and capture the raw stream # Usage: http_sse PATH DATA # Sets: HTTP_STATUS, HTTP_BODY http_sse() { local path="$1" data="$2" HTTP_SSE_FILE=$(mktemp) HTTP_STATUS=$(curl -s -N -o "$HTTP_SSE_FILE" -w '%{http_code}' \ -X POST "http://127.0.0.1:${PORT}${path}" \ -H "Content-Type: application/json" \ -d "$data") HTTP_BODY=$(cat "$HTTP_SSE_FILE") rm -f "$HTTP_SSE_FILE" } assert_status() { local expected="$1" label="$2" if [[ "$HTTP_STATUS" == "$expected" ]]; then pass "$label (status=$HTTP_STATUS)" else fail "$label (expected=$expected, got=$HTTP_STATUS)" echo " body: $HTTP_BODY" fi } assert_json_field() { local field="$1" label="$2" if echo "$HTTP_BODY" | python3 -c "import sys,json; d=json.load(sys.stdin); assert '$field' in d" 2>/dev/null; then pass "$label (field '$field' exists)" else fail "$label (field '$field' missing in response)" echo " body: $HTTP_BODY" fi } assert_json_value() { local field="$1" expected="$2" label="$3" local actual actual=$(echo "$HTTP_BODY" | python3 -c "import sys,json; print(json.load(sys.stdin)['$field'])" 2>/dev/null || echo "") if [[ "$actual" == "$expected" ]]; then pass "$label ($field='$actual')" else fail "$label (expected $field='$expected', got='$actual')" fi } assert_json_nonempty() { local field="$1" label="$2" local val val=$(echo "$HTTP_BODY" | python3 -c "import sys,json; v=json.load(sys.stdin)['$field']; assert len(str(v))>0; print(v)" 2>/dev/null || echo "") if [[ -n "$val" ]]; then pass "$label ($field is non-empty)" else fail "$label ($field is empty or missing)" echo " body: $HTTP_BODY" fi } # Patch port number in extracted source code (8080 -> test port) patch_port() { sed "s/127\.0\.0\.1\", 8080/127.0.0.1\", ${PORT}/g; s/127\.0\.0\.1:8080/127.0.0.1:${PORT}/g" } # Patch model path in extracted source code patch_model() { sed "s|models/gemma-2-2b-it-Q4_K_M.gguf|models/${MODEL_NAME}|g" } # ============================================================================= # Ch1: Skeleton Server # ============================================================================= test_ch1() { log "Ch1: Project Setup & Skeleton Server" local APP_DIR="$WORKDIR/translate-app" mkdir -p "$APP_DIR/src" "$APP_DIR/models" cd "$APP_DIR" # Copy httplib.h from project root (test current version) cp "$PROJECT_ROOT/httplib.h" . # Download json.hpp into nlohmann/ directory to match #include mkdir -p nlohmann curl -sL -o nlohmann/json.hpp \ https://github.com/nlohmann/json/releases/latest/download/json.hpp # CMakeLists.txt — ch1 doesn't need llama.cpp, so use a minimal version # (the doc's cmake includes llama.cpp which isn't cloned yet in ch1) cat > CMakeLists.txt << 'CMAKE_EOF' cmake_minimum_required(VERSION 3.16) project(translate-server LANGUAGES CXX) set(CMAKE_CXX_STANDARD 17) add_executable(translate-server src/main.cpp) target_include_directories(translate-server PRIVATE ${CMAKE_SOURCE_DIR}) CMAKE_EOF # Extract main.cpp from ch1 doc and patch port extract_code "$DOCS_DIR/ch01-setup.md" "main.cpp" | patch_port > src/main.cpp # Build log "Ch1: Building..." cmake -B build -DCMAKE_BUILD_TYPE=Release 2>&1 | tail -1 cmake --build build -j 2>&1 | tail -3 # Start server ./build/translate-server & SERVER_PID=$! wait_for_server # Tests http_request POST /translate '{"text":"hello","target_lang":"ja"}' assert_status 200 "Ch1 POST /translate" assert_json_value translation "TODO" "Ch1 POST /translate returns TODO" http_request GET /models assert_status 200 "Ch1 GET /models" assert_json_field models "Ch1 GET /models" http_request POST /models/select '{"model":"test"}' assert_status 200 "Ch1 POST /models/select" assert_json_value status "TODO" "Ch1 POST /models/select returns TODO" stop_server log "Ch1: Done" } # ============================================================================= # Ch2: REST API with llama.cpp # ============================================================================= test_ch2() { log "Ch2: REST API with llama.cpp" local APP_DIR="$WORKDIR/translate-app" cd "$APP_DIR" # Clone llama.cpp if [[ ! -d llama.cpp ]]; then log "Ch2: Cloning llama.cpp..." git clone --depth 1 https://github.com/ggml-org/llama.cpp.git 2>&1 | tail -1 fi # Download cpp-llamalib.h if [[ ! -f cpp-llamalib.h ]]; then curl -sL -o cpp-llamalib.h \ https://raw.githubusercontent.com/yhirose/cpp-llamalib/main/cpp-llamalib.h fi # Download model if [[ ! -f "models/$MODEL_NAME" ]]; then log "Ch2: Downloading model ${MODEL_NAME} (~1.6GB)..." curl -L -o "models/$MODEL_NAME" "$MODEL_URL" fi # CMakeLists.txt from ch1 doc (includes llama.cpp) extract_code "$DOCS_DIR/ch01-setup.md" "CMakeLists.txt" > CMakeLists.txt # Extract main.cpp from ch2 doc and patch port + model path extract_code "$DOCS_DIR/ch02-rest-api.md" "main.cpp" | patch_port | patch_model > src/main.cpp # Build (clean rebuild needed — cmake config changed) log "Ch2: Building (this may take a while for llama.cpp)..." rm -rf build cmake -B build -DCMAKE_BUILD_TYPE=Release 2>&1 | tail -1 cmake --build build -j 2>&1 | tail -3 # Start server ./build/translate-server & SERVER_PID=$! wait_for_server # Tests — normal request http_request POST /translate \ '{"text":"I had a great time visiting Tokyo last spring. The cherry blossoms were beautiful.","target_lang":"ja"}' assert_status 200 "Ch2 POST /translate normal" assert_json_nonempty translation "Ch2 POST /translate has translation" # Tests — invalid JSON http_request POST /translate 'not json' assert_status 400 "Ch2 POST /translate invalid JSON" # Tests — missing text http_request POST /translate '{"target_lang":"ja"}' assert_status 400 "Ch2 POST /translate missing text" # Tests — empty text http_request POST /translate '{"text":""}' assert_status 400 "Ch2 POST /translate empty text" stop_server log "Ch2: Done" } # ============================================================================= # Ch3: SSE Streaming # ============================================================================= test_ch3() { log "Ch3: SSE Streaming" local APP_DIR="$WORKDIR/translate-app" cd "$APP_DIR" # Extract main.cpp from ch3 doc and patch port + model path extract_code "$DOCS_DIR/ch03-sse-streaming.md" "main.cpp" | patch_port | patch_model > src/main.cpp # Build (incremental — only main.cpp changed) log "Ch3: Building..." cmake --build build -j 2>&1 | tail -3 # Start server ./build/translate-server & SERVER_PID=$! wait_for_server # Tests — /translate still works http_request POST /translate \ '{"text":"Hello world","target_lang":"ja"}' assert_status 200 "Ch3 POST /translate still works" # Tests — SSE streaming http_sse /translate/stream \ '{"text":"I had a great time visiting Tokyo last spring. The cherry blossoms were beautiful.","target_lang":"ja"}' assert_status 200 "Ch3 POST /translate/stream status" # Check SSE format: has data: lines and ends with [DONE] local data_lines data_lines=$(echo "$HTTP_BODY" | grep -c '^data: ' || true) if [[ $data_lines -ge 2 ]]; then pass "Ch3 SSE has multiple data: lines ($data_lines)" else fail "Ch3 SSE expected multiple data: lines, got $data_lines" echo " body: $HTTP_BODY" fi if echo "$HTTP_BODY" | grep -q 'data: \[DONE\]'; then pass "Ch3 SSE ends with data: [DONE]" else fail "Ch3 SSE missing data: [DONE]" echo " body: $HTTP_BODY" fi # Tests — SSE invalid JSON http_sse /translate/stream 'not json' assert_status 400 "Ch3 POST /translate/stream invalid JSON" stop_server log "Ch3: Done" } # ============================================================================= # Ch4: Model Management # ============================================================================= test_ch4() { log "Ch4: Model Management" local APP_DIR="$WORKDIR/translate-app" cd "$APP_DIR" # Ch4+ uses ~/.translate-app/models/ — symlink model there local MODELS_HOME="$HOME/.translate-app/models" mkdir -p "$MODELS_HOME" ln -sf "$APP_DIR/models/$MODEL_NAME" "$MODELS_HOME/$MODEL_NAME" # CMakeLists.txt from ch4 (adds OpenSSL) extract_code "$DOCS_DIR/ch04-model-management.md" "CMakeLists.txt" > CMakeLists.txt # Extract main.cpp from ch4 doc extract_code "$DOCS_DIR/ch04-model-management.md" "main.cpp" | patch_port > src/main.cpp # Build (reconfigure for OpenSSL, incremental — reuses llama.cpp objects) log "Ch4: Building..." cmake -B build -DCMAKE_BUILD_TYPE=Release 2>&1 | tail -1 cmake --build build -j 2>&1 | tail -3 # Start server ./build/translate-server & SERVER_PID=$! wait_for_server # Tests — GET /models http_request GET /models assert_status 200 "Ch4 GET /models" assert_json_field models "Ch4 GET /models has models array" # デフォルトモデルがdownloaded+selectedであること local selected selected=$(echo "$HTTP_BODY" | python3 -c " import sys, json models = json.load(sys.stdin)['models'] sel = [m for m in models if m['selected']] print(sel[0]['downloaded'] if sel else '') " 2>/dev/null || echo "") if [[ "$selected" == "True" ]]; then pass "Ch4 GET /models default model is downloaded and selected" else fail "Ch4 GET /models default model state unexpected" echo " body: $HTTP_BODY" fi # Tests — POST /models/select with already-downloaded model (SSE) http_sse /models/select '{"model": "gemma-2-2b-it"}' assert_status 200 "Ch4 POST /models/select already downloaded" if echo "$HTTP_BODY" | grep -q '"ready"'; then pass "Ch4 POST /models/select returns ready" else fail "Ch4 POST /models/select missing ready status" echo " body: $HTTP_BODY" fi # Tests — POST /models/select unknown model http_request POST /models/select '{"model": "nonexistent"}' assert_status 404 "Ch4 POST /models/select unknown model" # Tests — POST /models/select missing model field http_request POST /models/select '{"foo": "bar"}' assert_status 400 "Ch4 POST /models/select missing model field" # Tests — /translate still works after model select http_request POST /translate '{"text": "Hello", "target_lang": "ja"}' assert_status 200 "Ch4 POST /translate still works" assert_json_nonempty translation "Ch4 POST /translate has translation" # Tests — switch model via symlink (avoids downloading a second model) # Place a symlink so the server sees Llama-3.1-8B-Instruct as "downloaded" ln -sf "$MODELS_HOME/$MODEL_NAME" "$MODELS_HOME/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf" http_sse /models/select '{"model": "Llama-3.1-8B-Instruct"}' assert_status 200 "Ch4 POST /models/select switch model" if echo "$HTTP_BODY" | grep -q '"ready"'; then pass "Ch4 model switch returns ready" else fail "Ch4 model switch missing ready" echo " body: $HTTP_BODY" fi # Translate with the switched model http_request POST /translate \ '{"text": "The weather is nice today.", "target_lang": "ja"}' assert_status 200 "Ch4 POST /translate after model switch" assert_json_nonempty translation "Ch4 POST /translate switched model has translation" # Verify model list reflects the switch http_request GET /models local new_selected new_selected=$(echo "$HTTP_BODY" | python3 -c " import sys, json models = json.load(sys.stdin)['models'] sel = [m for m in models if m['selected']] print(sel[0]['name'] if sel else '') " 2>/dev/null || echo "") if [[ "$new_selected" == "Llama-3.1-8B-Instruct" ]]; then pass "Ch4 GET /models reflects model switch" else fail "Ch4 GET /models expected Llama-3.1-8B-Instruct selected, got '$new_selected'" fi stop_server log "Ch4: Done" } # ============================================================================= # Ch5: Web UI (browser tests via geckodriver + webdriver.h) # ============================================================================= start_geckodriver() { geckodriver --port "$GECKODRIVER_PORT" &>/dev/null & GECKODRIVER_PID=$! # Wait for geckodriver to be ready local i=0 while ! curl -s -o /dev/null "http://127.0.0.1:${GECKODRIVER_PORT}/status" 2>/dev/null; do sleep 0.5 i=$((i + 1)) if [[ $i -ge 20 ]]; then fail "geckodriver did not start within 10s" return 1 fi done } stop_geckodriver() { if [[ -n "$GECKODRIVER_PID" ]]; then kill "$GECKODRIVER_PID" 2>/dev/null || true wait "$GECKODRIVER_PID" 2>/dev/null || true GECKODRIVER_PID="" fi } test_ch5() { log "Ch5: Web UI (browser tests)" # Check for geckodriver if ! command -v geckodriver &>/dev/null; then log "Ch5: Skipping browser tests (geckodriver not found)" log "Ch5: Install with: brew install geckodriver" return 0 fi local APP_DIR="$WORKDIR/translate-app" cd "$APP_DIR" # Extract source files from ch05 extract_code "$DOCS_DIR/ch05-web-ui.md" "main.cpp" \ | patch_port > src/main.cpp mkdir -p public extract_code "$DOCS_DIR/ch05-web-ui.md" "index.html" > public/index.html extract_code "$DOCS_DIR/ch05-web-ui.md" "style.css" > public/style.css extract_code "$DOCS_DIR/ch05-web-ui.md" "script.js" > public/script.js # Build (incremental — only main.cpp changed) log "Ch5: Building server..." cmake --build build -j 2>&1 | tail -3 # Build browser test program log "Ch5: Building browser test..." g++ -std=c++17 \ -I"$APP_DIR" \ -I"$SCRIPT_DIR" \ -o "$APP_DIR/build/test_webui" \ "$SCRIPT_DIR/test_webui.cpp" \ -pthread # Start server ./build/translate-server & SERVER_PID=$! wait_for_server # Start geckodriver start_geckodriver # Run browser tests log "Ch5: Running browser tests..." local test_exit=0 "$APP_DIR/build/test_webui" "$PORT" || test_exit=$? # Parse pass/fail from test output and add to totals # (test_webui prints its own pass/fail, but we track via exit code) if [[ $test_exit -ne 0 ]]; then fail "Ch5 browser tests had failures" else pass "Ch5 browser tests all passed" fi stop_geckodriver stop_server log "Ch5: Done" } # ============================================================================= # Main # ============================================================================= log "LLM App Tutorial E2E Test" log "Working directory: $WORKDIR" echo "" test_ch1 echo "" test_ch2 echo "" test_ch3 echo "" test_ch4 echo "" test_ch5 log "Results: $PASS_COUNT passed, $FAIL_COUNT failed" if [[ $FAIL_COUNT -gt 0 ]]; then exit 1 fi