commit ac5e6f926d19f91c911ddb035f83ed9d3fa77378 Author: Stephen Schuresko Date: Sun Sep 13 21:01:42 2026 -0400 initial diff --git a/Dockerfile.systemd b/Dockerfile.systemd new file mode 100644 index 0000000..f7cf74d --- /dev/null +++ b/Dockerfile.systemd @@ -0,0 +1,37 @@ +FROM debian:bookworm-slim + +# Install systemd, dbus, curl, ca-certificates, xz-utils, and basic utilities +RUN apt-get update && apt-get install -y \ + systemd \ + systemd-sysv \ + dbus \ + curl \ + ca-certificates \ + xz-utils \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Install wasmtime WASI WebAssembly runtime +RUN curl -L https://github.com/bytecodealliance/wasmtime/releases/download/v23.0.2/wasmtime-v23.0.2-x86_64-linux.tar.xz | tar -xJ -C /usr/local/bin --strip-components=1 wasmtime-v23.0.2-x86_64-linux/wasmtime + +# Clean up default systemd console gettys +RUN rm -f /lib/systemd/system/multi-user.target.wants/getty.target \ + && rm -f /lib/systemd/system/multi-user.target.wants/systemd-logind.service + +# Create directories +RUN mkdir -p /opt/apps + +# Copy binaries +COPY gateway /usr/local/bin/gateway +COPY mock-app /usr/local/bin/mock-app + +# Copy systemd unit files +COPY services/ /etc/systemd/system/ + +# Enable gateway service on boot +RUN systemctl enable gateway.service + +# Tell systemd it is running in docker +ENV container docker +STOPSIGNAL SIGRTMIN+3 +ENTRYPOINT ["/lib/systemd/systemd"] diff --git a/gateway b/gateway new file mode 100755 index 0000000..e768576 Binary files /dev/null and b/gateway differ diff --git a/gateway.go b/gateway.go new file mode 100644 index 0000000..9ff6193 --- /dev/null +++ b/gateway.go @@ -0,0 +1,464 @@ +package main + +import ( + "crypto/tls" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "log" + "net" + "net/http" + "net/http/httputil" + "net/url" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "time" +) + +var ( + appsDir = "/opt/apps" + configPath = "/etc/zero-scale/artifacts.json" + allocatedPorts = make(map[string]int) + lastActive = make(map[string]time.Time) + mu sync.Mutex + + idleTimeout = 15 * time.Minute + checkPeriod = 30 * time.Second +) + +type ArtifactItem struct { + URL string `json:"url"` + Type string `json:"type"` +} + +type ConfigMapData struct { + Artifacts map[string]ArtifactItem `json:"artifacts"` +} + +func main() { + if dir := os.Getenv("APPS_DIR"); dir != "" { + appsDir = dir + } + os.MkdirAll(appsDir, 0755) + + go startReaper() + go startArtifactSync() + + http.HandleFunc("/", handleRequest) + + log.Println("Starting Zero-Scale Gateway on :80...") + if err := http.ListenAndServe(":80", nil); err != nil { + log.Fatalf("Server error: %v", err) + } +} + +func handleRequest(w http.ResponseWriter, r *http.Request) { + host := r.Host + if strings.Contains(host, ":") { + host = strings.Split(host, ":")[0] + } + log.Printf("Received %s request for host: %s (path: %s)", r.Method, host, r.URL.Path) + + binaryPath := filepath.Join(appsDir, host) + if !fileExists(binaryPath) { + // Fallback lookup: match shortName (e.g. bdash2 -> bdash2.kube.fairfaxmedia.net) + shortName := strings.Split(host, ".")[0] + entries, err := os.ReadDir(appsDir) + if err == nil { + for _, entry := range entries { + if !entry.IsDir() && (strings.HasPrefix(entry.Name(), shortName+".") || entry.Name() == shortName) { + binaryPath = filepath.Join(appsDir, entry.Name()) + log.Printf("Resolved host %s to binary %s", host, binaryPath) + break + } + } + } + } + if !fileExists(binaryPath) { + http.Error(w, fmt.Sprintf("Application %s not found", host), http.StatusNotFound) + return + } + + unitName := fmt.Sprintf("app-%s", host) + + mu.Lock() + lastActive[host] = time.Now() + + active, err := isServiceActive(unitName) + if err != nil { + mu.Unlock() + log.Printf("Error checking status of unit %s: %v", unitName, err) + http.Error(w, "Failed to check service status", http.StatusInternalServerError) + return + } + + port, exists := allocatedPorts[host] + if active && !exists { + // Discover active port from systemd unit Environment if gateway restarted + if activePort := getActiveServicePort(unitName); activePort > 0 { + allocatedPorts[host] = activePort + port = activePort + exists = true + log.Printf("Discovered active service %s running on port %d", host, port) + } + } + + if !active || !exists { + // Service is stopped or unallocated; allocate fresh free port and start + if active { + stopServiceLocked(host) + } + freePort, err := getFreePort() + if err != nil { + mu.Unlock() + log.Printf("Failed to get free port for %s: %v", host, err) + http.Error(w, "Internal Server Error: No available ports", http.StatusInternalServerError) + return + } + allocatedPorts[host] = freePort + port = freePort + + log.Printf("Host %s matches binary %s. Starting sandboxed systemd unit on port %d...", host, binaryPath, port) + err = startTransientService(host, port, binaryPath) + if err != nil { + delete(allocatedPorts, host) + mu.Unlock() + log.Printf("Failed to start transient systemd unit for %s: %v", host, err) + http.Error(w, "Failed to start application process", http.StatusInternalServerError) + return + } + mu.Unlock() + + // Wait for the app socket to bind and respond + targetAddr := fmt.Sprintf("127.0.0.1:%d", port) + if err := waitPortReady(targetAddr, 30*time.Second); err != nil { + log.Printf("Service %s on port %d did not bind in time: %v", host, port, err) + stopService(host) + http.Error(w, "Application startup timeout", http.StatusGatewayTimeout) + return + } + log.Printf("Service %s successfully scaled up on port %d!", host, port) + } else { + mu.Unlock() + } + + // Reverse proxy request + targetURL, _ := url.Parse(fmt.Sprintf("http://127.0.0.1:%d", port)) + proxy := httputil.NewSingleHostReverseProxy(targetURL) + proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) { + log.Printf("Proxy error for %s on port %d: %v. Cleaning up service...", host, port, err) + stopService(host) + http.Error(w, "Bad Gateway", http.StatusBadGateway) + } + proxy.ServeHTTP(w, r) +} + +// getFreePort queries kernel for free ephemeral port +func getFreePort() (int, error) { + addr, err := net.ResolveTCPAddr("tcp", "127.0.0.1:0") + if err != nil { + return 0, err + } + l, err := net.ListenTCP("tcp", addr) + if err != nil { + return 0, err + } + defer l.Close() + return l.Addr().(*net.TCPAddr).Port, nil +} + +// isServiceActive checks if the dynamic systemd unit is running +func isServiceActive(unit string) (bool, error) { + cmd := exec.Command("systemctl", "is-active", unit) + err := cmd.Run() + if err != nil { + if _, ok := err.(*exec.ExitError); ok { + return false, nil + } + return false, err + } + return true, nil +} + +// getActiveServicePort reads PORT from running systemd service environment +func getActiveServicePort(unit string) int { + out, err := exec.Command("systemctl", "show", "-p", "Environment", unit).Output() + if err != nil { + return 0 + } + str := string(out) + for _, env := range strings.Fields(str) { + if strings.HasPrefix(env, "PORT=") || strings.HasPrefix(env, "Environment=PORT=") { + parts := strings.Split(env, "=") + if len(parts) >= 2 { + val := parts[len(parts)-1] + if p, err := strconv.Atoi(val); err == nil && p > 0 { + return p + } + } + } + } + return 0 +} + +// fetchK8sResource dynamically retrieves a ConfigMap or Secret from the in-cluster Kubernetes API +func fetchK8sResource(resourceType, name string) (map[string]string, error) { + tokenBytes, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token") + if err != nil { + return nil, err + } + token := strings.TrimSpace(string(tokenBytes)) + + k8sHost := os.Getenv("KUBERNETES_SERVICE_HOST") + if k8sHost == "" { + k8sHost = "kubernetes.default.svc" + } + k8sPort := os.Getenv("KUBERNETES_SERVICE_PORT") + if k8sPort == "" { + k8sPort = "443" + } + + reqURL := fmt.Sprintf("https://%s:%s/api/v1/namespaces/default/%s/%s", k8sHost, k8sPort, resourceType, name) + req, err := http.NewRequest("GET", reqURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+token) + + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr, Timeout: 5 * time.Second} + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("k8s API returned status %d for %s/%s", resp.StatusCode, resourceType, name) + } + + var res struct { + Data map[string]string `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&res); err != nil { + return nil, err + } + + if resourceType == "secrets" { + decoded := make(map[string]string) + for k, v := range res.Data { + b, err := base64.StdEncoding.DecodeString(v) + if err == nil { + decoded[k] = string(b) + } else { + decoded[k] = v + } + } + return decoded, nil + } + + return res.Data, nil +} + +// startTransientService executes systemd-run with dynamic environment injection +func startTransientService(host string, port int, binaryPath string) error { + unitName := fmt.Sprintf("app-%s", host) + typePath := binaryPath + ".type" + + runType := "elf" + if fileExists(typePath) { + tBytes, err := os.ReadFile(typePath) + if err == nil { + runType = strings.TrimSpace(string(tBytes)) + } + } + + baseArgs := []string{ + fmt.Sprintf("--unit=%s", unitName), + "-p", "DynamicUser=yes", + "-p", "PrivateTmp=yes", + "-p", "ProtectSystem=strict", + fmt.Sprintf("--setenv=PORT=%d", port), + } + + if _, err := os.Stat("/app/uploads"); err == nil { + baseArgs = append(baseArgs, "-p", "BindPaths=/app/uploads:/app/uploads") + } + + envKeys := []string{ + "DATABASE_URL", "DATABASE_HOST", "DATABASE_PORT", "DATABASE_USER", "DATABASE_PASSWORD", "DATABASE_NAME", + "PGHOST", "PGPORT", "PGUSER", "PGPASSWORD", "PGDATABASE", + } + for _, k := range envKeys { + if v := os.Getenv(k); v != "" { + baseArgs = append(baseArgs, fmt.Sprintf("--setenv=%s=%s", k, v)) + } + } + + shortName := strings.Split(host, ".")[0] + appCMName := "zero-scale-app-" + shortName + if appData, err := fetchK8sResource("configmaps", appCMName); err == nil { + for k, v := range appData { + if k != "host" && k != "url" && k != "type" && k != "mounts" && k != "scale-to-zero" && k != "database_configmap" && k != "database_secret" { + baseArgs = append(baseArgs, fmt.Sprintf("--setenv=%s=%s", k, v)) + } + } + if dbCM := appData["database_configmap"]; dbCM != "" { + if cmData, err := fetchK8sResource("configmaps", dbCM); err == nil { + for k, v := range cmData { + baseArgs = append(baseArgs, fmt.Sprintf("--setenv=%s=%s", k, v)) + } + } + } + if dbSec := appData["database_secret"]; dbSec != "" { + if secData, err := fetchK8sResource("secrets", dbSec); err == nil { + for k, v := range secData { + baseArgs = append(baseArgs, fmt.Sprintf("--setenv=%s=%s", k, v)) + } + } + } + } + + var cmd *exec.Cmd + if runType == "wasm" { + args := append(baseArgs, + "/usr/local/bin/wasmtime", "run", + "--tcplisten", fmt.Sprintf("127.0.0.1:%d", port), + binaryPath, + ) + cmd = exec.Command("systemd-run", args...) + } else { + args := append(baseArgs, binaryPath, "-port", strconv.Itoa(port)) + cmd = exec.Command("systemd-run", args...) + } + + log.Printf("Executing: %s", cmd.String()) + return cmd.Run() +} + +// stopServiceLocked shuts down dynamic systemd service unit and clears port state (assumes mu is held) +func stopServiceLocked(host string) error { + unit := fmt.Sprintf("app-%s", host) + cmd := exec.Command("systemctl", "stop", unit) + _ = cmd.Run() + cmdReset := exec.Command("systemctl", "reset-failed", unit) + _ = cmdReset.Run() + + delete(allocatedPorts, host) + return nil +} + +// stopService safely locks mu and calls stopServiceLocked +func stopService(host string) error { + mu.Lock() + defer mu.Unlock() + return stopServiceLocked(host) +} + +// waitPortReady checks port every 200ms +func waitPortReady(addr string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + conn, err := net.DialTimeout("tcp", addr, 100*time.Millisecond) + if err == nil { + conn.Close() + return nil + } + time.Sleep(200 * time.Millisecond) + } + return fmt.Errorf("timeout waiting for %s", addr) +} + +// startReaper checks active services and stops idle ones +func startReaper() { + ticker := time.NewTicker(checkPeriod) + for range ticker.C { + mu.Lock() + now := time.Now() + for host, lastTime := range lastActive { + unitName := fmt.Sprintf("app-%s", host) + active, err := isServiceActive(unitName) + if err == nil && active && now.Sub(lastTime) > idleTimeout { + log.Printf("Service %s has been idle for %v. Scaling to zero...", host, idleTimeout) + if err := stopServiceLocked(host); err != nil { + log.Printf("Failed to stop idle unit %s: %v", unitName, err) + } + } + } + mu.Unlock() + } +} + +// startArtifactSync polls artifacts config map and downloads binaries dynamically +func startArtifactSync() { + ticker := time.NewTicker(15 * time.Second) + for range ticker.C { + if !fileExists(configPath) { + continue + } + data, err := os.ReadFile(configPath) + if err != nil { + log.Printf("Failed to read artifacts config: %v", err) + continue + } + var cfg ConfigMapData + if err := json.Unmarshal(data, &cfg); err != nil { + log.Printf("Failed to parse artifacts config JSON: %v", err) + continue + } + + for host, item := range cfg.Artifacts { + targetPath := filepath.Join(appsDir, host) + typePath := targetPath + ".type" + + _ = os.WriteFile(typePath, []byte(item.Type), 0644) + + if !fileExists(targetPath) { + log.Printf("Downloading artifact for %s from %s...", host, item.URL) + err := downloadFile(item.URL, targetPath) + if err != nil { + log.Printf("Failed to download %s: %v", item.URL, err) + continue + } + os.Chmod(targetPath, 0755) + log.Printf("Successfully deployed %s to %s", host, targetPath) + } + } + } +} + +func fileExists(path string) bool { + info, err := os.Stat(path) + if os.IsNotExist(err) { + return false + } + return err == nil && !info.IsDir() +} + +func downloadFile(url, targetPath string) error { + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("bad status: %s", resp.Status) + } + + out, err := os.Create(targetPath) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, resp.Body) + return err +} diff --git a/mock-app b/mock-app new file mode 100755 index 0000000..64e8b28 Binary files /dev/null and b/mock-app differ diff --git a/mock-app.go b/mock-app.go new file mode 100644 index 0000000..fc98fa8 --- /dev/null +++ b/mock-app.go @@ -0,0 +1,27 @@ +package main + +import ( + "flag" + "fmt" + "log" + "net/http" +) + +func main() { + port := flag.Int("port", 8080, "Port to listen on") + name := flag.String("name", "mock-service", "Name of the service") + flag.Parse() + + log.Printf("Starting mock service %s on port %d...", *name, *port) + + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + log.Printf("Received request: %s %s from %s", r.Method, r.URL.Path, r.RemoteAddr) + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{"status": "healthy", "service": "%s", "message": "Hello from the zero-scale consolidated %s!"}`, *name, *name) + }) + + addr := fmt.Sprintf(":%d", *port) + if err := http.ListenAndServe(addr, nil); err != nil { + log.Fatalf("Mock service failed: %v", err) + } +} diff --git a/services/bdash.service b/services/bdash.service new file mode 100644 index 0000000..991c5b0 --- /dev/null +++ b/services/bdash.service @@ -0,0 +1,14 @@ +[Unit] +Description=Builds-Dash Mock Service +After=network.target + +[Service] +Type=simple +ExecStart=/opt/apps/mock-app -port 8934 -name bdash +Restart=always +DynamicUser=yes +PrivateTmp=yes +ProtectSystem=strict + +[Install] +WantedBy=multi-user.target diff --git a/services/gateway.service b/services/gateway.service new file mode 100644 index 0000000..5d9c761 --- /dev/null +++ b/services/gateway.service @@ -0,0 +1,11 @@ +[Unit] +Description=Zero-Scale Gateway Proxy +After=network.target + +[Service] +Type=simple +ExecStart=/usr/local/bin/gateway +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/services/gitea.service b/services/gitea.service new file mode 100644 index 0000000..455e6d1 --- /dev/null +++ b/services/gitea.service @@ -0,0 +1,14 @@ +[Unit] +Description=Gitea Mock Service +After=network.target + +[Service] +Type=simple +ExecStart=/opt/apps/mock-app -port 3000 -name gitea +Restart=always +DynamicUser=yes +PrivateTmp=yes +ProtectSystem=strict + +[Install] +WantedBy=multi-user.target diff --git a/services/hoppscotch-web.service b/services/hoppscotch-web.service new file mode 100644 index 0000000..4d9e8ed --- /dev/null +++ b/services/hoppscotch-web.service @@ -0,0 +1,14 @@ +[Unit] +Description=Hoppscotch-Web Mock Service +After=network.target + +[Service] +Type=simple +ExecStart=/opt/apps/mock-app -port 8080 -name hoppscotch-web +Restart=always +DynamicUser=yes +PrivateTmp=yes +ProtectSystem=strict + +[Install] +WantedBy=multi-user.target diff --git a/services/papi-prod.service b/services/papi-prod.service new file mode 100644 index 0000000..cb9c31a --- /dev/null +++ b/services/papi-prod.service @@ -0,0 +1,14 @@ +[Unit] +Description=Papi-Prod Mock Service +After=network.target + +[Service] +Type=simple +ExecStart=/opt/apps/mock-app -port 8912 -name papi-prod +Restart=always +DynamicUser=yes +PrivateTmp=yes +ProtectSystem=strict + +[Install] +WantedBy=multi-user.target diff --git a/systemd-pod.yaml b/systemd-pod.yaml new file mode 100644 index 0000000..f45225c --- /dev/null +++ b/systemd-pod.yaml @@ -0,0 +1,135 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: zero-scale-artifacts + namespace: default +data: + artifacts.json: | + { + "artifacts": { + "bdash2.kube.fairfaxmedia.net": { + "url": "https://raw.githubusercontent.com/bytecodealliance/wasmtime/main/README.md", + "type": "elf" + }, + "hopscotch.kube.fairfaxmedia.net": { + "url": "https://raw.githubusercontent.com/bytecodealliance/wasmtime/main/README.md", + "type": "elf" + } + } + } +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: zero-scale-binaries-pvc + namespace: default +spec: + accessModes: + - ReadWriteMany + storageClassName: longhorn-2-replicas + resources: + requests: + storage: 5Gi +--- +apiVersion: v1 +kind: Pod +metadata: + name: zero-scale-gateway + namespace: default + labels: + app: zero-scale-gateway +spec: + nodeSelector: + kubernetes.io/hostname: didi-macbookpro14-3 + containers: + - name: systemd-runner + image: localhost:32000/systemd-runner:latest + securityContext: + privileged: true + ports: + - containerPort: 80 + name: http + volumeMounts: + - name: run + mountPath: /run + - name: run-lock + mountPath: /run/lock + - name: apps-volume + mountPath: /opt/apps + - name: bdash-uploads + mountPath: /app/uploads + - name: config-volume + mountPath: /etc/zero-scale + volumes: + - name: run + emptyDir: + medium: Memory + - name: run-lock + emptyDir: + medium: Memory + - name: apps-volume + persistentVolumeClaim: + claimName: zero-scale-binaries-pvc + - name: bdash-uploads + persistentVolumeClaim: + claimName: builds-dash-uploads-longhorn-rwx + - name: config-volume + configMap: + name: zero-scale-artifacts +--- +apiVersion: v1 +kind: Service +metadata: + name: zero-scale-gateway + namespace: default +spec: + ports: + - port: 80 + targetPort: 80 + protocol: TCP + name: http + selector: + app: zero-scale-gateway + type: ClusterIP +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: zero-scale-gateway + namespace: default + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/ssl-redirect: "false" +spec: + ingressClassName: public + rules: + - host: bdash2.fairfaxmedia.net + http: + paths: + - backend: + service: + name: zero-scale-gateway + port: + number: 80 + path: / + pathType: ImplementationSpecific + - host: bdash2.kube.fairfaxmedia.net + http: + paths: + - backend: + service: + name: zero-scale-gateway + port: + number: 80 + path: / + pathType: ImplementationSpecific + - host: hopscotch.kube.fairfaxmedia.net + http: + paths: + - backend: + service: + name: zero-scale-gateway + port: + number: 80 + path: / + pathType: ImplementationSpecific