Volt VMM (Neutron Stardust): source-available under AGPSL v5.0
KVM-based microVMM for the Volt platform: - Sub-second VM boot times - Minimal memory footprint - Landlock LSM + seccomp security - Virtio device support - Custom kernel management Copyright (c) Armored Gates LLC. All rights reserved. Licensed under AGPSL v5.0
This commit is contained in:
Karl Clinger
236
benchmarks/compare.sh
Executable file
236
benchmarks/compare.sh
Executable file
@@ -0,0 +1,236 @@
|
||||
#!/bin/bash
|
||||
# Volt Network Benchmark - Backend Comparison
|
||||
# Generates side-by-side comparison of all backends
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
RESULTS_BASE="${1:-${SCRIPT_DIR}/results}"
|
||||
|
||||
echo "╔══════════════════════════════════════════════════════════════╗"
|
||||
echo "║ Volt Backend Comparison Report ║"
|
||||
echo "╚══════════════════════════════════════════════════════════════╝"
|
||||
echo ""
|
||||
echo "Results directory: $RESULTS_BASE"
|
||||
echo "Generated: $(date)"
|
||||
echo ""
|
||||
|
||||
# Find all backends with results
|
||||
BACKENDS=()
|
||||
for dir in "${RESULTS_BASE}"/*/; do
|
||||
if [ -d "$dir" ]; then
|
||||
backend=$(basename "$dir")
|
||||
BACKENDS+=("$backend")
|
||||
fi
|
||||
done
|
||||
|
||||
if [ ${#BACKENDS[@]} -eq 0 ]; then
|
||||
echo "ERROR: No results found in $RESULTS_BASE"
|
||||
echo "Run benchmarks first with: ./run-all.sh <server-ip> <backend-name>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Found backends: ${BACKENDS[*]}"
|
||||
echo ""
|
||||
|
||||
# Function to get latest result directory for a backend
|
||||
get_latest_result() {
|
||||
local backend="$1"
|
||||
ls -td "${RESULTS_BASE}/${backend}"/*/ 2>/dev/null | head -1
|
||||
}
|
||||
|
||||
# Function to extract metric from JSON
|
||||
get_json_metric() {
|
||||
local file="$1"
|
||||
local path="$2"
|
||||
local default="${3:-N/A}"
|
||||
|
||||
if [ -f "$file" ] && command -v jq &> /dev/null; then
|
||||
result=$(jq -r "$path // \"$default\"" "$file" 2>/dev/null)
|
||||
echo "${result:-$default}"
|
||||
else
|
||||
echo "$default"
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to format Gbps
|
||||
format_gbps() {
|
||||
local bps="$1"
|
||||
if [ "$bps" = "N/A" ] || [ -z "$bps" ] || [ "$bps" = "0" ]; then
|
||||
echo "N/A"
|
||||
else
|
||||
printf "%.2f" $(echo "$bps / 1000000000" | bc -l 2>/dev/null || echo "0")
|
||||
fi
|
||||
}
|
||||
|
||||
# Collect data for comparison
|
||||
declare -A TCP_SINGLE TCP_MULTI UDP_MAX ICMP_P50 ICMP_P99 PPS_64
|
||||
|
||||
for backend in "${BACKENDS[@]}"; do
|
||||
result_dir=$(get_latest_result "$backend")
|
||||
if [ -z "$result_dir" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# Throughput
|
||||
tcp_single_bps=$(get_json_metric "${result_dir}/tcp-single.json" '.end.sum_sent.bits_per_second')
|
||||
TCP_SINGLE[$backend]=$(format_gbps "$tcp_single_bps")
|
||||
|
||||
tcp_multi_bps=$(get_json_metric "${result_dir}/tcp-multi-8.json" '.end.sum_sent.bits_per_second')
|
||||
TCP_MULTI[$backend]=$(format_gbps "$tcp_multi_bps")
|
||||
|
||||
udp_max_bps=$(get_json_metric "${result_dir}/udp-max.json" '.end.sum.bits_per_second')
|
||||
UDP_MAX[$backend]=$(format_gbps "$udp_max_bps")
|
||||
|
||||
# Latency
|
||||
if [ -f "${result_dir}/ping-summary.env" ]; then
|
||||
source "${result_dir}/ping-summary.env"
|
||||
ICMP_P50[$backend]="${ICMP_P50_US:-N/A}"
|
||||
ICMP_P99[$backend]="${ICMP_P99_US:-N/A}"
|
||||
else
|
||||
ICMP_P50[$backend]="N/A"
|
||||
ICMP_P99[$backend]="N/A"
|
||||
fi
|
||||
|
||||
# PPS
|
||||
if [ -f "${result_dir}/udp-64byte.json" ]; then
|
||||
packets=$(get_json_metric "${result_dir}/udp-64byte.json" '.end.sum.packets')
|
||||
# Assume 30s duration if not specified
|
||||
if [ "$packets" != "N/A" ] && [ -n "$packets" ]; then
|
||||
pps=$(echo "$packets / 30" | bc 2>/dev/null || echo "N/A")
|
||||
PPS_64[$backend]="$pps"
|
||||
else
|
||||
PPS_64[$backend]="N/A"
|
||||
fi
|
||||
else
|
||||
PPS_64[$backend]="N/A"
|
||||
fi
|
||||
done
|
||||
|
||||
# Print comparison tables
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " THROUGHPUT COMPARISON (Gbps)"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo ""
|
||||
|
||||
# Header
|
||||
printf "%-15s" "Backend"
|
||||
printf "%15s" "TCP Single"
|
||||
printf "%15s" "TCP Multi-8"
|
||||
printf "%15s" "UDP Max"
|
||||
echo ""
|
||||
|
||||
printf "%-15s" "-------"
|
||||
printf "%15s" "----------"
|
||||
printf "%15s" "-----------"
|
||||
printf "%15s" "-------"
|
||||
echo ""
|
||||
|
||||
for backend in "${BACKENDS[@]}"; do
|
||||
printf "%-15s" "$backend"
|
||||
printf "%15s" "${TCP_SINGLE[$backend]:-N/A}"
|
||||
printf "%15s" "${TCP_MULTI[$backend]:-N/A}"
|
||||
printf "%15s" "${UDP_MAX[$backend]:-N/A}"
|
||||
echo ""
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " LATENCY COMPARISON (µs)"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo ""
|
||||
|
||||
printf "%-15s" "Backend"
|
||||
printf "%15s" "ICMP P50"
|
||||
printf "%15s" "ICMP P99"
|
||||
echo ""
|
||||
|
||||
printf "%-15s" "-------"
|
||||
printf "%15s" "--------"
|
||||
printf "%15s" "--------"
|
||||
echo ""
|
||||
|
||||
for backend in "${BACKENDS[@]}"; do
|
||||
printf "%-15s" "$backend"
|
||||
printf "%15s" "${ICMP_P50[$backend]:-N/A}"
|
||||
printf "%15s" "${ICMP_P99[$backend]:-N/A}"
|
||||
echo ""
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " PPS COMPARISON (packets/sec)"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo ""
|
||||
|
||||
printf "%-15s" "Backend"
|
||||
printf "%15s" "64-byte UDP"
|
||||
echo ""
|
||||
|
||||
printf "%-15s" "-------"
|
||||
printf "%15s" "-----------"
|
||||
echo ""
|
||||
|
||||
for backend in "${BACKENDS[@]}"; do
|
||||
printf "%-15s" "$backend"
|
||||
printf "%15s" "${PPS_64[$backend]:-N/A}"
|
||||
echo ""
|
||||
done
|
||||
|
||||
# Generate markdown report
|
||||
REPORT_FILE="${RESULTS_BASE}/COMPARISON.md"
|
||||
{
|
||||
echo "# Volt Backend Comparison"
|
||||
echo ""
|
||||
echo "Generated: $(date)"
|
||||
echo ""
|
||||
echo "## Throughput (Gbps)"
|
||||
echo ""
|
||||
echo "| Backend | TCP Single | TCP Multi-8 | UDP Max |"
|
||||
echo "|---------|------------|-------------|---------|"
|
||||
for backend in "${BACKENDS[@]}"; do
|
||||
echo "| $backend | ${TCP_SINGLE[$backend]:-N/A} | ${TCP_MULTI[$backend]:-N/A} | ${UDP_MAX[$backend]:-N/A} |"
|
||||
done
|
||||
echo ""
|
||||
echo "## Latency (µs)"
|
||||
echo ""
|
||||
echo "| Backend | ICMP P50 | ICMP P99 |"
|
||||
echo "|---------|----------|----------|"
|
||||
for backend in "${BACKENDS[@]}"; do
|
||||
echo "| $backend | ${ICMP_P50[$backend]:-N/A} | ${ICMP_P99[$backend]:-N/A} |"
|
||||
done
|
||||
echo ""
|
||||
echo "## Packets Per Second"
|
||||
echo ""
|
||||
echo "| Backend | 64-byte UDP PPS |"
|
||||
echo "|---------|-----------------|"
|
||||
for backend in "${BACKENDS[@]}"; do
|
||||
echo "| $backend | ${PPS_64[$backend]:-N/A} |"
|
||||
done
|
||||
echo ""
|
||||
echo "## Analysis"
|
||||
echo ""
|
||||
echo "### Expected Performance Hierarchy"
|
||||
echo ""
|
||||
echo "1. **macvtap** - Direct host NIC passthrough, near line-rate"
|
||||
echo "2. **vhost-net** - Kernel datapath, 2-3x virtio throughput"
|
||||
echo "3. **virtio** - QEMU userspace, baseline performance"
|
||||
echo ""
|
||||
echo "### Key Observations"
|
||||
echo ""
|
||||
echo "- TCP Multi-stream shows aggregate bandwidth capability"
|
||||
echo "- P99 latency reveals worst-case jitter"
|
||||
echo "- 64-byte PPS shows raw packet processing overhead"
|
||||
echo ""
|
||||
} > "$REPORT_FILE"
|
||||
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo ""
|
||||
echo "Comparison report saved to: $REPORT_FILE"
|
||||
echo ""
|
||||
echo "Performance Hierarchy (expected):"
|
||||
echo " macvtap > vhost-net > virtio"
|
||||
echo ""
|
||||
echo "Key insight: If vhost-net isn't 2-3x faster than virtio,"
|
||||
echo "check that vhost_net kernel module is loaded and in use."
|
||||
Reference in New Issue
Block a user