Volt CLI: source-available under AGPSL v5.0
Complete infrastructure platform CLI: - Container runtime (systemd-nspawn) - VoltVisor VMs (Neutron Stardust / QEMU) - Stellarium CAS (content-addressed storage) - ORAS Registry - GitOps integration - Landlock LSM security - Compose orchestration - Mesh networking Copyright (c) Armored Gates LLC. All rights reserved. Licensed under AGPSL v5.0
This commit is contained in:
Karl Clinger
348
pkg/cdn/client.go
Normal file
348
pkg/cdn/client.go
Normal file
@@ -0,0 +1,348 @@
|
||||
/*
|
||||
CDN Client — BunnyCDN blob and manifest operations for Volt CAS.
|
||||
|
||||
Handles pull (public, unauthenticated) and push (authenticated via AccessKey)
|
||||
to the BunnyCDN storage and pull-zone endpoints that back Stellarium.
|
||||
|
||||
Copyright (c) Armored Gates LLC. All rights reserved.
|
||||
*/
|
||||
package cdn
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// ── Defaults ─────────────────────────────────────────────────────────────────
|
||||
|
||||
const (
|
||||
DefaultBlobsURL = "https://blobs.3kb.io"
|
||||
DefaultManifestsURL = "https://manifests.3kb.io"
|
||||
DefaultRegion = "ny"
|
||||
)
|
||||
|
||||
// ── Manifest ─────────────────────────────────────────────────────────────────
|
||||
|
||||
// Manifest represents a CAS build manifest as stored on the CDN.
|
||||
type Manifest struct {
|
||||
Name string `json:"name"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
Objects map[string]string `json:"objects"` // relative path → sha256 hash
|
||||
}
|
||||
|
||||
// ── Client ───────────────────────────────────────────────────────────────────
|
||||
|
||||
// Client handles blob upload/download to BunnyCDN.
|
||||
type Client struct {
|
||||
BlobsBaseURL string // pull-zone URL for blobs, e.g. https://blobs.3kb.io
|
||||
ManifestsBaseURL string // pull-zone URL for manifests, e.g. https://manifests.3kb.io
|
||||
StorageAPIKey string // BunnyCDN storage zone API key
|
||||
StorageZoneName string // BunnyCDN storage zone name
|
||||
Region string // BunnyCDN region, e.g. "ny"
|
||||
HTTPClient *http.Client
|
||||
}
|
||||
|
||||
// ── CDN Config (from config.yaml) ────────────────────────────────────────────
|
||||
|
||||
// CDNConfig represents the cdn section of /etc/volt/config.yaml.
|
||||
type CDNConfig struct {
|
||||
BlobsURL string `yaml:"blobs_url"`
|
||||
ManifestsURL string `yaml:"manifests_url"`
|
||||
StorageAPIKey string `yaml:"storage_api_key"`
|
||||
StorageZone string `yaml:"storage_zone"`
|
||||
Region string `yaml:"region"`
|
||||
}
|
||||
|
||||
// voltConfig is a minimal representation of the config file, just enough to
|
||||
// extract the cdn block.
|
||||
type voltConfig struct {
|
||||
CDN CDNConfig `yaml:"cdn"`
|
||||
}
|
||||
|
||||
// ── Constructors ─────────────────────────────────────────────────────────────
|
||||
|
||||
// NewClient creates a CDN client by reading config from /etc/volt/config.yaml
|
||||
// (if present) and falling back to environment variables.
|
||||
func NewClient() (*Client, error) {
|
||||
return NewClientFromConfigFile("")
|
||||
}
|
||||
|
||||
// NewClientFromConfigFile creates a CDN client from a specific config file
|
||||
// path. If configPath is empty, it tries /etc/volt/config.yaml.
|
||||
func NewClientFromConfigFile(configPath string) (*Client, error) {
|
||||
var cfg CDNConfig
|
||||
|
||||
// Try to load from config file.
|
||||
if configPath == "" {
|
||||
configPath = "/etc/volt/config.yaml"
|
||||
}
|
||||
if data, err := os.ReadFile(configPath); err == nil {
|
||||
var vc voltConfig
|
||||
if err := yaml.Unmarshal(data, &vc); err == nil {
|
||||
cfg = vc.CDN
|
||||
}
|
||||
}
|
||||
|
||||
// Expand environment variable references in config values (e.g. "${BUNNY_API_KEY}").
|
||||
cfg.BlobsURL = expandEnv(cfg.BlobsURL)
|
||||
cfg.ManifestsURL = expandEnv(cfg.ManifestsURL)
|
||||
cfg.StorageAPIKey = expandEnv(cfg.StorageAPIKey)
|
||||
cfg.StorageZone = expandEnv(cfg.StorageZone)
|
||||
cfg.Region = expandEnv(cfg.Region)
|
||||
|
||||
// Override with environment variables if config values are empty.
|
||||
if cfg.BlobsURL == "" {
|
||||
cfg.BlobsURL = os.Getenv("VOLT_CDN_BLOBS_URL")
|
||||
}
|
||||
if cfg.ManifestsURL == "" {
|
||||
cfg.ManifestsURL = os.Getenv("VOLT_CDN_MANIFESTS_URL")
|
||||
}
|
||||
if cfg.StorageAPIKey == "" {
|
||||
cfg.StorageAPIKey = os.Getenv("BUNNY_API_KEY")
|
||||
}
|
||||
if cfg.StorageZone == "" {
|
||||
cfg.StorageZone = os.Getenv("BUNNY_STORAGE_ZONE")
|
||||
}
|
||||
if cfg.Region == "" {
|
||||
cfg.Region = os.Getenv("BUNNY_REGION")
|
||||
}
|
||||
|
||||
// Apply defaults.
|
||||
if cfg.BlobsURL == "" {
|
||||
cfg.BlobsURL = DefaultBlobsURL
|
||||
}
|
||||
if cfg.ManifestsURL == "" {
|
||||
cfg.ManifestsURL = DefaultManifestsURL
|
||||
}
|
||||
if cfg.Region == "" {
|
||||
cfg.Region = DefaultRegion
|
||||
}
|
||||
|
||||
return &Client{
|
||||
BlobsBaseURL: strings.TrimRight(cfg.BlobsURL, "/"),
|
||||
ManifestsBaseURL: strings.TrimRight(cfg.ManifestsURL, "/"),
|
||||
StorageAPIKey: cfg.StorageAPIKey,
|
||||
StorageZoneName: cfg.StorageZone,
|
||||
Region: cfg.Region,
|
||||
HTTPClient: &http.Client{
|
||||
Timeout: 5 * time.Minute,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// NewClientFromConfig creates a CDN client from explicit parameters.
|
||||
func NewClientFromConfig(blobsURL, manifestsURL, apiKey, zoneName string) *Client {
|
||||
if blobsURL == "" {
|
||||
blobsURL = DefaultBlobsURL
|
||||
}
|
||||
if manifestsURL == "" {
|
||||
manifestsURL = DefaultManifestsURL
|
||||
}
|
||||
return &Client{
|
||||
BlobsBaseURL: strings.TrimRight(blobsURL, "/"),
|
||||
ManifestsBaseURL: strings.TrimRight(manifestsURL, "/"),
|
||||
StorageAPIKey: apiKey,
|
||||
StorageZoneName: zoneName,
|
||||
Region: DefaultRegion,
|
||||
HTTPClient: &http.Client{
|
||||
Timeout: 5 * time.Minute,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ── Pull Operations (public, no auth) ────────────────────────────────────────
|
||||
|
||||
// PullBlob downloads a blob by hash from the CDN pull zone and verifies its
|
||||
// SHA-256 integrity. Returns the raw content.
|
||||
func (c *Client) PullBlob(hash string) ([]byte, error) {
|
||||
url := fmt.Sprintf("%s/sha256:%s", c.BlobsBaseURL, hash)
|
||||
|
||||
resp, err := c.HTTPClient.Get(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cdn pull blob %s: %w", hash[:12], err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("cdn pull blob %s: HTTP %d", hash[:12], resp.StatusCode)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cdn pull blob %s: read body: %w", hash[:12], err)
|
||||
}
|
||||
|
||||
// Verify integrity.
|
||||
actualHash := sha256Hex(data)
|
||||
if actualHash != hash {
|
||||
return nil, fmt.Errorf("cdn pull blob %s: integrity check failed (got %s)", hash[:12], actualHash[:12])
|
||||
}
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// PullManifest downloads a manifest by name from the CDN manifests pull zone.
|
||||
func (c *Client) PullManifest(name string) (*Manifest, error) {
|
||||
url := fmt.Sprintf("%s/v2/public/%s/latest.json", c.ManifestsBaseURL, name)
|
||||
|
||||
resp, err := c.HTTPClient.Get(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cdn pull manifest %s: %w", name, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
return nil, fmt.Errorf("cdn pull manifest %s: not found", name)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("cdn pull manifest %s: HTTP %d", name, resp.StatusCode)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cdn pull manifest %s: read body: %w", name, err)
|
||||
}
|
||||
|
||||
var m Manifest
|
||||
if err := json.Unmarshal(data, &m); err != nil {
|
||||
return nil, fmt.Errorf("cdn pull manifest %s: unmarshal: %w", name, err)
|
||||
}
|
||||
|
||||
return &m, nil
|
||||
}
|
||||
|
||||
// BlobExists checks whether a blob exists on the CDN using a HEAD request.
|
||||
func (c *Client) BlobExists(hash string) (bool, error) {
|
||||
url := fmt.Sprintf("%s/sha256:%s", c.BlobsBaseURL, hash)
|
||||
|
||||
req, err := http.NewRequest(http.MethodHead, url, nil)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("cdn blob exists %s: %w", hash[:12], err)
|
||||
}
|
||||
|
||||
resp, err := c.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("cdn blob exists %s: %w", hash[:12], err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
switch resp.StatusCode {
|
||||
case http.StatusOK:
|
||||
return true, nil
|
||||
case http.StatusNotFound:
|
||||
return false, nil
|
||||
default:
|
||||
return false, fmt.Errorf("cdn blob exists %s: HTTP %d", hash[:12], resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// ── Push Operations (authenticated) ──────────────────────────────────────────
|
||||
|
||||
// PushBlob uploads a blob to BunnyCDN storage. The hash must match the SHA-256
|
||||
// of the data. Requires StorageAPIKey and StorageZoneName to be set.
|
||||
func (c *Client) PushBlob(hash string, data []byte) error {
|
||||
if c.StorageAPIKey == "" {
|
||||
return fmt.Errorf("cdn push blob: StorageAPIKey not configured")
|
||||
}
|
||||
if c.StorageZoneName == "" {
|
||||
return fmt.Errorf("cdn push blob: StorageZoneName not configured")
|
||||
}
|
||||
|
||||
// Verify the hash matches the data.
|
||||
actualHash := sha256Hex(data)
|
||||
if actualHash != hash {
|
||||
return fmt.Errorf("cdn push blob: hash mismatch (expected %s, got %s)", hash[:12], actualHash[:12])
|
||||
}
|
||||
|
||||
// BunnyCDN storage upload endpoint.
|
||||
url := fmt.Sprintf("https://%s.storage.bunnycdn.com/%s/sha256:%s",
|
||||
c.Region, c.StorageZoneName, hash)
|
||||
|
||||
req, err := http.NewRequest(http.MethodPut, url, strings.NewReader(string(data)))
|
||||
if err != nil {
|
||||
return fmt.Errorf("cdn push blob %s: create request: %w", hash[:12], err)
|
||||
}
|
||||
req.Header.Set("AccessKey", c.StorageAPIKey)
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
req.ContentLength = int64(len(data))
|
||||
|
||||
resp, err := c.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cdn push blob %s: %w", hash[:12], err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("cdn push blob %s: HTTP %d: %s", hash[:12], resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// PushManifest uploads a manifest to BunnyCDN storage under the conventional
|
||||
// path: v2/public/{name}/latest.json
|
||||
func (c *Client) PushManifest(name string, manifest *Manifest) error {
|
||||
if c.StorageAPIKey == "" {
|
||||
return fmt.Errorf("cdn push manifest: StorageAPIKey not configured")
|
||||
}
|
||||
if c.StorageZoneName == "" {
|
||||
return fmt.Errorf("cdn push manifest: StorageZoneName not configured")
|
||||
}
|
||||
|
||||
data, err := json.MarshalIndent(manifest, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("cdn push manifest %s: marshal: %w", name, err)
|
||||
}
|
||||
|
||||
// Upload to manifests storage zone path.
|
||||
url := fmt.Sprintf("https://%s.storage.bunnycdn.com/%s/v2/public/%s/latest.json",
|
||||
c.Region, c.StorageZoneName, name)
|
||||
|
||||
req, err := http.NewRequest(http.MethodPut, url, strings.NewReader(string(data)))
|
||||
if err != nil {
|
||||
return fmt.Errorf("cdn push manifest %s: create request: %w", name, err)
|
||||
}
|
||||
req.Header.Set("AccessKey", c.StorageAPIKey)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.ContentLength = int64(len(data))
|
||||
|
||||
resp, err := c.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cdn push manifest %s: %w", name, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("cdn push manifest %s: HTTP %d: %s", name, resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
// sha256Hex computes the SHA-256 hex digest of data.
|
||||
func sha256Hex(data []byte) string {
|
||||
h := sha256.Sum256(data)
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
|
||||
// expandEnv expands "${VAR}" patterns in a string. Only the ${VAR} form is
|
||||
// expanded (not $VAR) to avoid accidental substitution.
|
||||
func expandEnv(s string) string {
|
||||
if !strings.Contains(s, "${") {
|
||||
return s
|
||||
}
|
||||
return os.Expand(s, os.Getenv)
|
||||
}
|
||||
487
pkg/cdn/client_test.go
Normal file
487
pkg/cdn/client_test.go
Normal file
@@ -0,0 +1,487 @@
|
||||
package cdn
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
func testHash(data []byte) string {
|
||||
h := sha256.Sum256(data)
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
|
||||
// ── TestNewClientFromEnv ─────────────────────────────────────────────────────
|
||||
|
||||
func TestNewClientFromEnv(t *testing.T) {
|
||||
// Set env vars.
|
||||
os.Setenv("VOLT_CDN_BLOBS_URL", "https://blobs.example.com")
|
||||
os.Setenv("VOLT_CDN_MANIFESTS_URL", "https://manifests.example.com")
|
||||
os.Setenv("BUNNY_API_KEY", "test-api-key-123")
|
||||
os.Setenv("BUNNY_STORAGE_ZONE", "test-zone")
|
||||
os.Setenv("BUNNY_REGION", "la")
|
||||
defer func() {
|
||||
os.Unsetenv("VOLT_CDN_BLOBS_URL")
|
||||
os.Unsetenv("VOLT_CDN_MANIFESTS_URL")
|
||||
os.Unsetenv("BUNNY_API_KEY")
|
||||
os.Unsetenv("BUNNY_STORAGE_ZONE")
|
||||
os.Unsetenv("BUNNY_REGION")
|
||||
}()
|
||||
|
||||
// Use a non-existent config file so we rely purely on env.
|
||||
c, err := NewClientFromConfigFile("/nonexistent/config.yaml")
|
||||
if err != nil {
|
||||
t.Fatalf("NewClientFromConfigFile: %v", err)
|
||||
}
|
||||
|
||||
if c.BlobsBaseURL != "https://blobs.example.com" {
|
||||
t.Errorf("BlobsBaseURL = %q, want %q", c.BlobsBaseURL, "https://blobs.example.com")
|
||||
}
|
||||
if c.ManifestsBaseURL != "https://manifests.example.com" {
|
||||
t.Errorf("ManifestsBaseURL = %q, want %q", c.ManifestsBaseURL, "https://manifests.example.com")
|
||||
}
|
||||
if c.StorageAPIKey != "test-api-key-123" {
|
||||
t.Errorf("StorageAPIKey = %q, want %q", c.StorageAPIKey, "test-api-key-123")
|
||||
}
|
||||
if c.StorageZoneName != "test-zone" {
|
||||
t.Errorf("StorageZoneName = %q, want %q", c.StorageZoneName, "test-zone")
|
||||
}
|
||||
if c.Region != "la" {
|
||||
t.Errorf("Region = %q, want %q", c.Region, "la")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewClientDefaults(t *testing.T) {
|
||||
// Clear all relevant env vars.
|
||||
for _, key := range []string{
|
||||
"VOLT_CDN_BLOBS_URL", "VOLT_CDN_MANIFESTS_URL",
|
||||
"BUNNY_API_KEY", "BUNNY_STORAGE_ZONE", "BUNNY_REGION",
|
||||
} {
|
||||
os.Unsetenv(key)
|
||||
}
|
||||
|
||||
c, err := NewClientFromConfigFile("/nonexistent/config.yaml")
|
||||
if err != nil {
|
||||
t.Fatalf("NewClientFromConfigFile: %v", err)
|
||||
}
|
||||
|
||||
if c.BlobsBaseURL != DefaultBlobsURL {
|
||||
t.Errorf("BlobsBaseURL = %q, want default %q", c.BlobsBaseURL, DefaultBlobsURL)
|
||||
}
|
||||
if c.ManifestsBaseURL != DefaultManifestsURL {
|
||||
t.Errorf("ManifestsBaseURL = %q, want default %q", c.ManifestsBaseURL, DefaultManifestsURL)
|
||||
}
|
||||
if c.Region != DefaultRegion {
|
||||
t.Errorf("Region = %q, want default %q", c.Region, DefaultRegion)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewClientFromConfig(t *testing.T) {
|
||||
c := NewClientFromConfig("https://b.example.com", "https://m.example.com", "key", "zone")
|
||||
if c.BlobsBaseURL != "https://b.example.com" {
|
||||
t.Errorf("BlobsBaseURL = %q", c.BlobsBaseURL)
|
||||
}
|
||||
if c.StorageAPIKey != "key" {
|
||||
t.Errorf("StorageAPIKey = %q", c.StorageAPIKey)
|
||||
}
|
||||
}
|
||||
|
||||
// ── TestPullBlob (integrity) ─────────────────────────────────────────────────
|
||||
|
||||
func TestPullBlobIntegrity(t *testing.T) {
|
||||
content := []byte("hello stellarium blob")
|
||||
hash := testHash(content)
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
expectedPath := "/sha256:" + hash
|
||||
if r.URL.Path != expectedPath {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(content)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := NewClientFromConfig(srv.URL, "", "", "")
|
||||
c.HTTPClient = srv.Client()
|
||||
|
||||
data, err := c.PullBlob(hash)
|
||||
if err != nil {
|
||||
t.Fatalf("PullBlob: %v", err)
|
||||
}
|
||||
if string(data) != string(content) {
|
||||
t.Errorf("PullBlob data = %q, want %q", data, content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPullBlobHashVerification(t *testing.T) {
|
||||
content := []byte("original content")
|
||||
hash := testHash(content)
|
||||
|
||||
// Serve tampered content that doesn't match the hash.
|
||||
tampered := []byte("tampered content!!!")
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(tampered)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := NewClientFromConfig(srv.URL, "", "", "")
|
||||
c.HTTPClient = srv.Client()
|
||||
|
||||
_, err := c.PullBlob(hash)
|
||||
if err == nil {
|
||||
t.Fatal("PullBlob should fail on tampered content, got nil error")
|
||||
}
|
||||
if !contains(err.Error(), "integrity check failed") {
|
||||
t.Errorf("expected integrity error, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPullBlobNotFound(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.NotFound(w, r)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := NewClientFromConfig(srv.URL, "", "", "")
|
||||
c.HTTPClient = srv.Client()
|
||||
|
||||
_, err := c.PullBlob("abcdef123456abcdef123456abcdef123456abcdef123456abcdef123456abcd")
|
||||
if err == nil {
|
||||
t.Fatal("PullBlob should fail on 404")
|
||||
}
|
||||
if !contains(err.Error(), "HTTP 404") {
|
||||
t.Errorf("expected HTTP 404 error, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ── TestPullManifest ─────────────────────────────────────────────────────────
|
||||
|
||||
func TestPullManifest(t *testing.T) {
|
||||
manifest := Manifest{
|
||||
Name: "test-image",
|
||||
CreatedAt: "2024-01-01T00:00:00Z",
|
||||
Objects: map[string]string{
|
||||
"usr/bin/hello": "aabbccdd",
|
||||
"etc/config": "eeff0011",
|
||||
},
|
||||
}
|
||||
manifestJSON, _ := json.Marshal(manifest)
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/v2/public/test-image/latest.json" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(manifestJSON)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := NewClientFromConfig("", srv.URL, "", "")
|
||||
c.HTTPClient = srv.Client()
|
||||
|
||||
m, err := c.PullManifest("test-image")
|
||||
if err != nil {
|
||||
t.Fatalf("PullManifest: %v", err)
|
||||
}
|
||||
if m.Name != "test-image" {
|
||||
t.Errorf("Name = %q, want %q", m.Name, "test-image")
|
||||
}
|
||||
if len(m.Objects) != 2 {
|
||||
t.Errorf("Objects count = %d, want 2", len(m.Objects))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPullManifestNotFound(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.NotFound(w, r)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := NewClientFromConfig("", srv.URL, "", "")
|
||||
c.HTTPClient = srv.Client()
|
||||
|
||||
_, err := c.PullManifest("nonexistent")
|
||||
if err == nil {
|
||||
t.Fatal("PullManifest should fail on 404")
|
||||
}
|
||||
if !contains(err.Error(), "not found") {
|
||||
t.Errorf("expected 'not found' error, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ── TestBlobExists ───────────────────────────────────────────────────────────
|
||||
|
||||
func TestBlobExists(t *testing.T) {
|
||||
existingHash := "aabbccddee112233aabbccddee112233aabbccddee112233aabbccddee112233"
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodHead {
|
||||
t.Errorf("expected HEAD, got %s", r.Method)
|
||||
}
|
||||
if r.URL.Path == "/sha256:"+existingHash {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
} else {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := NewClientFromConfig(srv.URL, "", "", "")
|
||||
c.HTTPClient = srv.Client()
|
||||
|
||||
exists, err := c.BlobExists(existingHash)
|
||||
if err != nil {
|
||||
t.Fatalf("BlobExists: %v", err)
|
||||
}
|
||||
if !exists {
|
||||
t.Error("BlobExists = false, want true")
|
||||
}
|
||||
|
||||
exists, err = c.BlobExists("0000000000000000000000000000000000000000000000000000000000000000")
|
||||
if err != nil {
|
||||
t.Fatalf("BlobExists: %v", err)
|
||||
}
|
||||
if exists {
|
||||
t.Error("BlobExists = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
// ── TestPushBlob ─────────────────────────────────────────────────────────────
|
||||
|
||||
func TestPushBlob(t *testing.T) {
|
||||
content := []byte("push me to CDN")
|
||||
hash := testHash(content)
|
||||
|
||||
var receivedKey string
|
||||
var receivedBody []byte
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPut {
|
||||
t.Errorf("expected PUT, got %s", r.Method)
|
||||
}
|
||||
receivedKey = r.Header.Get("AccessKey")
|
||||
var err error
|
||||
receivedBody, err = readAll(r.Body)
|
||||
if err != nil {
|
||||
t.Errorf("read body: %v", err)
|
||||
}
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
// Override the storage URL by setting region to a dummy value and using
|
||||
// the test server URL directly. We'll need to construct the client manually.
|
||||
c := &Client{
|
||||
BlobsBaseURL: srv.URL,
|
||||
StorageAPIKey: "test-key-456",
|
||||
StorageZoneName: "test-zone",
|
||||
Region: "ny",
|
||||
HTTPClient: srv.Client(),
|
||||
}
|
||||
|
||||
// Override the storage endpoint to use our test server.
|
||||
// We need to monkeypatch the push URL. Since the real URL uses bunnycdn.com,
|
||||
// we'll create a custom roundtripper.
|
||||
c.HTTPClient.Transport = &rewriteTransport{
|
||||
inner: srv.Client().Transport,
|
||||
targetURL: srv.URL,
|
||||
}
|
||||
|
||||
err := c.PushBlob(hash, content)
|
||||
if err != nil {
|
||||
t.Fatalf("PushBlob: %v", err)
|
||||
}
|
||||
|
||||
if receivedKey != "test-key-456" {
|
||||
t.Errorf("AccessKey header = %q, want %q", receivedKey, "test-key-456")
|
||||
}
|
||||
if string(receivedBody) != string(content) {
|
||||
t.Errorf("body = %q, want %q", receivedBody, content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPushBlobHashMismatch(t *testing.T) {
|
||||
content := []byte("some content")
|
||||
wrongHash := "0000000000000000000000000000000000000000000000000000000000000000"
|
||||
|
||||
c := &Client{
|
||||
StorageAPIKey: "key",
|
||||
StorageZoneName: "zone",
|
||||
HTTPClient: &http.Client{},
|
||||
}
|
||||
|
||||
err := c.PushBlob(wrongHash, content)
|
||||
if err == nil {
|
||||
t.Fatal("PushBlob should fail on hash mismatch")
|
||||
}
|
||||
if !contains(err.Error(), "hash mismatch") {
|
||||
t.Errorf("expected hash mismatch error, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPushBlobNoAPIKey(t *testing.T) {
|
||||
c := &Client{
|
||||
StorageAPIKey: "",
|
||||
StorageZoneName: "zone",
|
||||
HTTPClient: &http.Client{},
|
||||
}
|
||||
|
||||
err := c.PushBlob("abc", []byte("data"))
|
||||
if err == nil {
|
||||
t.Fatal("PushBlob should fail without API key")
|
||||
}
|
||||
if !contains(err.Error(), "StorageAPIKey not configured") {
|
||||
t.Errorf("expected 'not configured' error, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ── TestExpandEnv ────────────────────────────────────────────────────────────
|
||||
|
||||
func TestExpandEnv(t *testing.T) {
|
||||
os.Setenv("TEST_CDN_VAR", "expanded-value")
|
||||
defer os.Unsetenv("TEST_CDN_VAR")
|
||||
|
||||
result := expandEnv("${TEST_CDN_VAR}")
|
||||
if result != "expanded-value" {
|
||||
t.Errorf("expandEnv = %q, want %q", result, "expanded-value")
|
||||
}
|
||||
|
||||
// No expansion when no pattern.
|
||||
result = expandEnv("plain-string")
|
||||
if result != "plain-string" {
|
||||
t.Errorf("expandEnv = %q, want %q", result, "plain-string")
|
||||
}
|
||||
}
|
||||
|
||||
// ── TestConfigFile ───────────────────────────────────────────────────────────
|
||||
|
||||
func TestConfigFileLoading(t *testing.T) {
|
||||
// Clear env vars so config file values are used.
|
||||
for _, key := range []string{
|
||||
"VOLT_CDN_BLOBS_URL", "VOLT_CDN_MANIFESTS_URL",
|
||||
"BUNNY_API_KEY", "BUNNY_STORAGE_ZONE", "BUNNY_REGION",
|
||||
} {
|
||||
os.Unsetenv(key)
|
||||
}
|
||||
|
||||
os.Setenv("MY_API_KEY", "from-env-ref")
|
||||
defer os.Unsetenv("MY_API_KEY")
|
||||
|
||||
// Write a temp config file.
|
||||
configContent := `cdn:
|
||||
blobs_url: "https://custom-blobs.example.com"
|
||||
manifests_url: "https://custom-manifests.example.com"
|
||||
storage_api_key: "${MY_API_KEY}"
|
||||
storage_zone: "my-zone"
|
||||
region: "sg"
|
||||
`
|
||||
tmpFile, err := os.CreateTemp("", "volt-config-*.yaml")
|
||||
if err != nil {
|
||||
t.Fatalf("create temp: %v", err)
|
||||
}
|
||||
defer os.Remove(tmpFile.Name())
|
||||
|
||||
if _, err := tmpFile.WriteString(configContent); err != nil {
|
||||
t.Fatalf("write temp: %v", err)
|
||||
}
|
||||
tmpFile.Close()
|
||||
|
||||
c, err := NewClientFromConfigFile(tmpFile.Name())
|
||||
if err != nil {
|
||||
t.Fatalf("NewClientFromConfigFile: %v", err)
|
||||
}
|
||||
|
||||
if c.BlobsBaseURL != "https://custom-blobs.example.com" {
|
||||
t.Errorf("BlobsBaseURL = %q", c.BlobsBaseURL)
|
||||
}
|
||||
if c.ManifestsBaseURL != "https://custom-manifests.example.com" {
|
||||
t.Errorf("ManifestsBaseURL = %q", c.ManifestsBaseURL)
|
||||
}
|
||||
if c.StorageAPIKey != "from-env-ref" {
|
||||
t.Errorf("StorageAPIKey = %q, want %q", c.StorageAPIKey, "from-env-ref")
|
||||
}
|
||||
if c.StorageZoneName != "my-zone" {
|
||||
t.Errorf("StorageZoneName = %q", c.StorageZoneName)
|
||||
}
|
||||
if c.Region != "sg" {
|
||||
t.Errorf("Region = %q", c.Region)
|
||||
}
|
||||
}
|
||||
|
||||
// ── Test Helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
func contains(s, substr string) bool {
|
||||
return len(s) >= len(substr) && searchString(s, substr)
|
||||
}
|
||||
|
||||
func searchString(s, substr string) bool {
|
||||
for i := 0; i <= len(s)-len(substr); i++ {
|
||||
if s[i:i+len(substr)] == substr {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func readAll(r interface{ Read([]byte) (int, error) }) ([]byte, error) {
|
||||
var buf []byte
|
||||
tmp := make([]byte, 4096)
|
||||
for {
|
||||
n, err := r.Read(tmp)
|
||||
if n > 0 {
|
||||
buf = append(buf, tmp[:n]...)
|
||||
}
|
||||
if err != nil {
|
||||
if err.Error() == "EOF" {
|
||||
break
|
||||
}
|
||||
return buf, err
|
||||
}
|
||||
}
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
// rewriteTransport rewrites all requests to point at a test server.
|
||||
type rewriteTransport struct {
|
||||
inner http.RoundTripper
|
||||
targetURL string
|
||||
}
|
||||
|
||||
func (t *rewriteTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
// Replace the host with our test server.
|
||||
req.URL.Scheme = "http"
|
||||
req.URL.Host = stripScheme(t.targetURL)
|
||||
transport := t.inner
|
||||
if transport == nil {
|
||||
transport = http.DefaultTransport
|
||||
}
|
||||
return transport.RoundTrip(req)
|
||||
}
|
||||
|
||||
func stripScheme(url string) string {
|
||||
if idx := findIndex(url, "://"); idx >= 0 {
|
||||
return url[idx+3:]
|
||||
}
|
||||
return url
|
||||
}
|
||||
|
||||
func findIndex(s, substr string) int {
|
||||
for i := 0; i <= len(s)-len(substr); i++ {
|
||||
if s[i:i+len(substr)] == substr {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
196
pkg/cdn/encrypted_client.go
Normal file
196
pkg/cdn/encrypted_client.go
Normal file
@@ -0,0 +1,196 @@
|
||||
/*
|
||||
Encrypted CDN Client — Transparent AGE encryption layer over CDN operations.
|
||||
|
||||
Wraps the standard CDN Client to encrypt blobs before upload and decrypt
|
||||
on download. The encryption is transparent to callers — they push/pull
|
||||
plaintext and the encryption happens automatically.
|
||||
|
||||
Architecture:
|
||||
- PushBlob: plaintext → AGE encrypt → upload ciphertext
|
||||
- PullBlob: download ciphertext → AGE decrypt → return plaintext
|
||||
- Hash verification: hash is of PLAINTEXT (preserves CAS dedup)
|
||||
- Manifests are NOT encrypted (they contain only hashes, no sensitive data)
|
||||
|
||||
Copyright (c) Armored Gates LLC. All rights reserved.
|
||||
*/
|
||||
package cdn
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/armoredgate/volt/pkg/encryption"
|
||||
)
|
||||
|
||||
// ── Encrypted Client ─────────────────────────────────────────────────────────
|
||||
|
||||
// EncryptedClient wraps a CDN Client with transparent AGE encryption.
|
||||
type EncryptedClient struct {
|
||||
// Inner is the underlying CDN client that handles HTTP operations.
|
||||
Inner *Client
|
||||
|
||||
// Recipients are the AGE public keys to encrypt to.
|
||||
// Populated from encryption.BuildRecipients() on creation.
|
||||
Recipients []string
|
||||
|
||||
// IdentityPath is the path to the AGE private key for decryption.
|
||||
IdentityPath string
|
||||
}
|
||||
|
||||
// NewEncryptedClient creates a CDN client with transparent encryption.
|
||||
// It reads encryption keys from the standard locations.
|
||||
func NewEncryptedClient() (*EncryptedClient, error) {
|
||||
inner, err := NewClient()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encrypted cdn client: %w", err)
|
||||
}
|
||||
|
||||
return NewEncryptedClientFromInner(inner)
|
||||
}
|
||||
|
||||
// NewEncryptedClientFromInner wraps an existing CDN client with encryption.
|
||||
func NewEncryptedClientFromInner(inner *Client) (*EncryptedClient, error) {
|
||||
recipients, err := encryption.BuildRecipients()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encrypted cdn client: %w", err)
|
||||
}
|
||||
|
||||
return &EncryptedClient{
|
||||
Inner: inner,
|
||||
Recipients: recipients,
|
||||
IdentityPath: encryption.CDNIdentityPath(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ── Encrypted Push/Pull ──────────────────────────────────────────────────────
|
||||
|
||||
// PushBlob encrypts plaintext data and uploads the ciphertext to the CDN.
|
||||
// The hash parameter is the SHA-256 of the PLAINTEXT (for CAS addressing).
|
||||
// The CDN stores the ciphertext keyed by the plaintext hash.
|
||||
func (ec *EncryptedClient) PushBlob(hash string, plaintext []byte) error {
|
||||
// Verify plaintext hash matches
|
||||
actualHash := encSha256Hex(plaintext)
|
||||
if actualHash != hash {
|
||||
return fmt.Errorf("encrypted push: hash mismatch (expected %s, got %s)", hash[:12], actualHash[:12])
|
||||
}
|
||||
|
||||
// Encrypt
|
||||
ciphertext, err := encryption.Encrypt(plaintext, ec.Recipients)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encrypted push %s: %w", hash[:12], err)
|
||||
}
|
||||
|
||||
// Upload ciphertext — we bypass the inner client's hash check since the
|
||||
// ciphertext hash won't match the plaintext hash. We use the raw HTTP upload.
|
||||
return ec.pushRawBlob(hash, ciphertext)
|
||||
}
|
||||
|
||||
// PullBlob downloads ciphertext from the CDN, decrypts it, and returns plaintext.
|
||||
// The hash is verified against the decrypted plaintext.
|
||||
func (ec *EncryptedClient) PullBlob(hash string) ([]byte, error) {
|
||||
// Download raw (skip inner client's integrity check since it's ciphertext)
|
||||
ciphertext, err := ec.pullRawBlob(hash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Decrypt
|
||||
plaintext, err := encryption.Decrypt(ciphertext, ec.IdentityPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encrypted pull %s: %w", hash[:12], err)
|
||||
}
|
||||
|
||||
// Verify plaintext integrity
|
||||
actualHash := encSha256Hex(plaintext)
|
||||
if actualHash != hash {
|
||||
return nil, fmt.Errorf("encrypted pull %s: plaintext integrity check failed (got %s)", hash[:12], actualHash[:12])
|
||||
}
|
||||
|
||||
return plaintext, nil
|
||||
}
|
||||
|
||||
// BlobExists checks if a blob exists on the CDN (delegates to inner client).
|
||||
func (ec *EncryptedClient) BlobExists(hash string) (bool, error) {
|
||||
return ec.Inner.BlobExists(hash)
|
||||
}
|
||||
|
||||
// PullManifest downloads a manifest (NOT encrypted — manifests contain only hashes).
|
||||
func (ec *EncryptedClient) PullManifest(name string) (*Manifest, error) {
|
||||
return ec.Inner.PullManifest(name)
|
||||
}
|
||||
|
||||
// PushManifest uploads a manifest (NOT encrypted).
|
||||
func (ec *EncryptedClient) PushManifest(name string, manifest *Manifest) error {
|
||||
return ec.Inner.PushManifest(name, manifest)
|
||||
}
|
||||
|
||||
// ── Raw HTTP Operations ──────────────────────────────────────────────────────
|
||||
|
||||
// pushRawBlob uploads raw bytes to the CDN without hash verification.
|
||||
// Used for ciphertext upload where the hash is of the plaintext.
|
||||
func (ec *EncryptedClient) pushRawBlob(hash string, data []byte) error {
|
||||
if ec.Inner.StorageAPIKey == "" {
|
||||
return fmt.Errorf("cdn push blob: StorageAPIKey not configured")
|
||||
}
|
||||
if ec.Inner.StorageZoneName == "" {
|
||||
return fmt.Errorf("cdn push blob: StorageZoneName not configured")
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("https://%s.storage.bunnycdn.com/%s/sha256:%s",
|
||||
ec.Inner.Region, ec.Inner.StorageZoneName, hash)
|
||||
|
||||
req, err := http.NewRequest(http.MethodPut, url, strings.NewReader(string(data)))
|
||||
if err != nil {
|
||||
return fmt.Errorf("cdn push blob %s: create request: %w", hash[:12], err)
|
||||
}
|
||||
req.Header.Set("AccessKey", ec.Inner.StorageAPIKey)
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
req.ContentLength = int64(len(data))
|
||||
|
||||
resp, err := ec.Inner.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cdn push blob %s: %w", hash[:12], err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("cdn push blob %s: HTTP %d: %s", hash[:12], resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// pullRawBlob downloads raw bytes from the CDN without hash verification.
|
||||
// Used for ciphertext download where the hash is of the plaintext.
|
||||
func (ec *EncryptedClient) pullRawBlob(hash string) ([]byte, error) {
|
||||
url := fmt.Sprintf("%s/sha256:%s", ec.Inner.BlobsBaseURL, hash)
|
||||
|
||||
resp, err := ec.Inner.HTTPClient.Get(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cdn pull blob %s: %w", hash[:12], err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("cdn pull blob %s: HTTP %d", hash[:12], resp.StatusCode)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cdn pull blob %s: read body: %w", hash[:12], err)
|
||||
}
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
func encSha256Hex(data []byte) string {
|
||||
h := sha256.Sum256(data)
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
Reference in New Issue
Block a user