volt-vmm/benchmarks/compare.sh

#!/bin/bash
# Volt Network Benchmark - Backend Comparison
# Generates side-by-side comparison of all backends

set -e

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
RESULTS_BASE="${1:-${SCRIPT_DIR}/results}"

echo "╔══════════════════════════════════════════════════════════════╗"
echo "║         Volt Backend Comparison Report                   ║"
echo "╚══════════════════════════════════════════════════════════════╝"
echo ""
echo "Results directory: $RESULTS_BASE"
echo "Generated: $(date)"
echo ""

# Find all backends with results
BACKENDS=()
for dir in "${RESULTS_BASE}"/*/; do
    if [ -d "$dir" ]; then
        backend=$(basename "$dir")
        BACKENDS+=("$backend")
    fi
done

if [ ${#BACKENDS[@]} -eq 0 ]; then
    echo "ERROR: No results found in $RESULTS_BASE"
    echo "Run benchmarks first with: ./run-all.sh <server-ip> <backend-name>"
    exit 1
fi

echo "Found backends: ${BACKENDS[*]}"
echo ""

# Function to get latest result directory for a backend
get_latest_result() {
    local backend="$1"
    ls -td "${RESULTS_BASE}/${backend}"/*/ 2>/dev/null | head -1
}

# Function to extract metric from JSON
get_json_metric() {
    local file="$1"
    local path="$2"
    local default="${3:-N/A}"
    
    if [ -f "$file" ] && command -v jq &> /dev/null; then
        result=$(jq -r "$path // \"$default\"" "$file" 2>/dev/null)
        echo "${result:-$default}"
    else
        echo "$default"
    fi
}

# Function to format Gbps
format_gbps() {
    local bps="$1"
    if [ "$bps" = "N/A" ] || [ -z "$bps" ] || [ "$bps" = "0" ]; then
        echo "N/A"
    else
        printf "%.2f" $(echo "$bps / 1000000000" | bc -l 2>/dev/null || echo "0")
    fi
}

# Collect data for comparison
declare -A TCP_SINGLE TCP_MULTI UDP_MAX ICMP_P50 ICMP_P99 PPS_64

for backend in "${BACKENDS[@]}"; do
    result_dir=$(get_latest_result "$backend")
    if [ -z "$result_dir" ]; then
        continue
    fi
    
    # Throughput
    tcp_single_bps=$(get_json_metric "${result_dir}/tcp-single.json" '.end.sum_sent.bits_per_second')
    TCP_SINGLE[$backend]=$(format_gbps "$tcp_single_bps")
    
    tcp_multi_bps=$(get_json_metric "${result_dir}/tcp-multi-8.json" '.end.sum_sent.bits_per_second')
    TCP_MULTI[$backend]=$(format_gbps "$tcp_multi_bps")
    
    udp_max_bps=$(get_json_metric "${result_dir}/udp-max.json" '.end.sum.bits_per_second')
    UDP_MAX[$backend]=$(format_gbps "$udp_max_bps")
    
    # Latency
    if [ -f "${result_dir}/ping-summary.env" ]; then
        source "${result_dir}/ping-summary.env"
        ICMP_P50[$backend]="${ICMP_P50_US:-N/A}"
        ICMP_P99[$backend]="${ICMP_P99_US:-N/A}"
    else
        ICMP_P50[$backend]="N/A"
        ICMP_P99[$backend]="N/A"
    fi
    
    # PPS
    if [ -f "${result_dir}/udp-64byte.json" ]; then
        packets=$(get_json_metric "${result_dir}/udp-64byte.json" '.end.sum.packets')
        # Assume 30s duration if not specified
        if [ "$packets" != "N/A" ] && [ -n "$packets" ]; then
            pps=$(echo "$packets / 30" | bc 2>/dev/null || echo "N/A")
            PPS_64[$backend]="$pps"
        else
            PPS_64[$backend]="N/A"
        fi
    else
        PPS_64[$backend]="N/A"
    fi
done

# Print comparison tables
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "                     THROUGHPUT COMPARISON (Gbps)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

# Header
printf "%-15s" "Backend"
printf "%15s" "TCP Single"
printf "%15s" "TCP Multi-8"
printf "%15s" "UDP Max"
echo ""

printf "%-15s" "-------"
printf "%15s" "----------"
printf "%15s" "-----------"
printf "%15s" "-------"
echo ""

for backend in "${BACKENDS[@]}"; do
    printf "%-15s" "$backend"
    printf "%15s" "${TCP_SINGLE[$backend]:-N/A}"
    printf "%15s" "${TCP_MULTI[$backend]:-N/A}"
    printf "%15s" "${UDP_MAX[$backend]:-N/A}"
    echo ""
done

echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "                      LATENCY COMPARISON (µs)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

printf "%-15s" "Backend"
printf "%15s" "ICMP P50"
printf "%15s" "ICMP P99"
echo ""

printf "%-15s" "-------"
printf "%15s" "--------"
printf "%15s" "--------"
echo ""

for backend in "${BACKENDS[@]}"; do
    printf "%-15s" "$backend"
    printf "%15s" "${ICMP_P50[$backend]:-N/A}"
    printf "%15s" "${ICMP_P99[$backend]:-N/A}"
    echo ""
done

echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "                    PPS COMPARISON (packets/sec)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

printf "%-15s" "Backend"
printf "%15s" "64-byte UDP"
echo ""

printf "%-15s" "-------"
printf "%15s" "-----------"
echo ""

for backend in "${BACKENDS[@]}"; do
    printf "%-15s" "$backend"
    printf "%15s" "${PPS_64[$backend]:-N/A}"
    echo ""
done

# Generate markdown report
REPORT_FILE="${RESULTS_BASE}/COMPARISON.md"
{
    echo "# Volt Backend Comparison"
    echo ""
    echo "Generated: $(date)"
    echo ""
    echo "## Throughput (Gbps)"
    echo ""
    echo "| Backend | TCP Single | TCP Multi-8 | UDP Max |"
    echo "|---------|------------|-------------|---------|"
    for backend in "${BACKENDS[@]}"; do
        echo "| $backend | ${TCP_SINGLE[$backend]:-N/A} | ${TCP_MULTI[$backend]:-N/A} | ${UDP_MAX[$backend]:-N/A} |"
    done
    echo ""
    echo "## Latency (µs)"
    echo ""
    echo "| Backend | ICMP P50 | ICMP P99 |"
    echo "|---------|----------|----------|"
    for backend in "${BACKENDS[@]}"; do
        echo "| $backend | ${ICMP_P50[$backend]:-N/A} | ${ICMP_P99[$backend]:-N/A} |"
    done
    echo ""
    echo "## Packets Per Second"
    echo ""
    echo "| Backend | 64-byte UDP PPS |"
    echo "|---------|-----------------|"
    for backend in "${BACKENDS[@]}"; do
        echo "| $backend | ${PPS_64[$backend]:-N/A} |"
    done
    echo ""
    echo "## Analysis"
    echo ""
    echo "### Expected Performance Hierarchy"
    echo ""
    echo "1. **macvtap** - Direct host NIC passthrough, near line-rate"
    echo "2. **vhost-net** - Kernel datapath, 2-3x virtio throughput"
    echo "3. **virtio** - QEMU userspace, baseline performance"
    echo ""
    echo "### Key Observations"
    echo ""
    echo "- TCP Multi-stream shows aggregate bandwidth capability"
    echo "- P99 latency reveals worst-case jitter"
    echo "- 64-byte PPS shows raw packet processing overhead"
    echo ""
} > "$REPORT_FILE"

echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
echo "Comparison report saved to: $REPORT_FILE"
echo ""
echo "Performance Hierarchy (expected):"
echo "  macvtap > vhost-net > virtio"
echo ""
echo "Key insight: If vhost-net isn't 2-3x faster than virtio,"
echo "check that vhost_net kernel module is loaded and in use."