Add LiteLLM gateway (skyliner) via ArgoCD/Helm

- cluster/applications/litellm: multi-source Application (oci litellm-helm 1.106.1),
  values, ExternalSecrets (1Password wdw-labs), HTTPRoute, ExternalName services
  for fantasmic/wishes, namespace
- gateway: https-skyliner listener + skyliner.wdw-labs.cloud certificate,
  ReferenceGrant for litellm namespace
- cert-manager: add wdw-labs.cloud to ClusterIssuer dnsZones
- external-dns: add wdw-labs.cloud zone (Technitium RFC2136)
- onepassword-connect: add wdw-labs vault to ClusterSecretStore

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0118eUSMQitZMHN7qaCvynA3

cluster/applications/litellm/application.yaml

@@ -0,0 +1,84 @@
+# ============================================================================
+# ArgoCD Application: LiteLLM proxy (skyliner)
+# Wave: 8 (same tier as hindsight / openviking)
+# Deployment method: GitOps (Gitea -> ArgoCD)
+# ============================================================================
+#
+# Multi-source: upstream OCI Helm chart (pinned) + local values + manifests
+# from repo (namespace, ExternalSecrets, HTTPRoute, ExternalName Services,
+# ServiceMonitor). Follows the openviking / hindsight pattern.
+#
+# Chart: oci://ghcr.io/berriai/litellm-helm  (version == appVersion)
+# Includes bundled Bitnami PostgreSQL (db.deployStandalone: true).
+#
+# NOTE (ArgoCD + OCI): repoURL has NO "oci://" prefix. If the OCI registry is not
+# already usable by ArgoCD, register it once (public repo, no creds needed):
+#   argocd repo add ghcr.io/berriai --type helm --name berriai-oci --enable-oci
+#
+# Platform changes required BEFORE this syncs (see DEPLOYMENT GATE below).
+# ============================================================================
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: litellm
+  namespace: argocd
+  labels:
+    app.kubernetes.io/name: litellm
+    app.kubernetes.io/part-of: mk-labs
+  annotations:
+    argocd.argoproj.io/sync-wave: "8"
+    description: |
+      LiteLLM OpenAI-compatible gateway for Paperclip/OpenCode and Hermes agents.
+      Host: skyliner.wdw-labs.cloud (fastpass-gateway, HTTPRoute). Admin UI at /ui.
+      Backends: illuminations vLLM (Gemma-4-26B); Mac Studio (fantasmic) and
+      Mac Mini (wishes) as placeholders.
+spec:
+  project: default
+
+  sources:
+    # Source 1: upstream OCI Helm chart, pinned
+    - repoURL: ghcr.io/berriai
+      chart: litellm-helm
+      targetRevision: 1.106.1
+      helm:
+        releaseName: litellm
+        valueFiles:
+          - $values/cluster/applications/litellm/values.yaml
+
+    # Source 2: homelab repo — values ref + manifests
+    - repoURL: https://gitea.mk-labs.cloud/rblundon/homelab.git
+      targetRevision: main
+      path: cluster/applications/litellm
+      ref: values
+      directory:
+        # Not Kubernetes manifests
+        exclude: "{application.yaml,values.yaml}"
+
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: litellm
+
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true
+
+# ============================================================================
+# DEPLOYMENT GATE: DO NOT MERGE/SYNC UNTIL
+# ============================================================================
+# 1. 1Password vault wdw-labs has item "litellm" (master-key, salt-key, db-password,
+#    db-admin-password) and items "illuminations", "fantasmic",
+#    "wishes" (field "api-key"); the onepassword-connect token can read
+#    wdw-labs and the ClusterSecretStore lists it (platform/onepassword-connect).
+#    (master-key must start with "sk-"; salt-key is PERMANENT — never rotate it
+#    after models/keys are stored, or stored credentials become undecryptable).
+# 2. Platform changes are live: cert-manager dnsZones (wdw-labs.cloud), gateway
+#    listener + skyliner Certificate, ReferenceGrant for litellm, and external-dns
+#    (Technitium) can write to the wdw-labs.cloud zone (zone exists + TSIG key
+#    authorized for it).
+# 3. DNS names fantasmic.wdw-labs.cloud / wishes.wdw-labs.cloud resolve from pods.
+# 4. Verified with `helm template` (see README / hand-off notes).
+# ============================================================================

cluster/applications/litellm/externalname-services.yaml

@@ -0,0 +1,26 @@
+# ExternalName Services for Mac inference endpoints (DNS names, not raw IPs).
+# Referenced from proxy_config api_base in values.yaml.
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: fantasmic
+  namespace: litellm
+  labels:
+    app.kubernetes.io/component: llm-inference
+    app.kubernetes.io/name: mac-studio
+spec:
+  type: ExternalName
+  externalName: fantasmic.wdw-labs.cloud   # Mac Studio (Tier 1 Hermes / Gemma-4 VLM)
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: wishes
+  namespace: litellm
+  labels:
+    app.kubernetes.io/component: llm-inference
+    app.kubernetes.io/name: mac-mini
+spec:
+  type: ExternalName
+  externalName: wishes.wdw-labs.cloud      # Mac Mini M4 (Tier 3 / oMLX)

cluster/applications/litellm/externalsecret.yaml

@@ -0,0 +1,82 @@
+# ExternalSecrets - LiteLLM
+# Store: onepassword-connect ClusterSecretStore (vaults: mk-labs, wdw-labs)
+#
+# 1Password, vault wdw-labs:
+#   Item "litellm" (CONCEALED fields):
+#     master-key         LiteLLM admin/master key (must start with "sk-")
+#     salt-key           LITELLM_SALT_KEY — encrypts stored provider creds. NEVER ROTATE.
+#     db-password        Postgres password for user "litellm" (use alphanumerics,
+#                        e.g. `openssl rand -hex 24`; it is embedded in a URL)
+#     db-admin-password  Postgres "postgres" superuser password
+#   Items "illuminations", "fantasmic", "wishes" — field "api-key"
+#
+# Wave -1: secrets exist before the Deployment / migration job / Postgres.
+---
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: litellm-credentials
+  namespace: litellm
+  annotations:
+    argocd.argoproj.io/sync-wave: "-1"
+spec:
+  refreshInterval: "1h"
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: onepassword-connect
+  target:
+    name: litellm-credentials
+    creationPolicy: Owner
+    template:
+      engineVersion: v2
+      data:
+        # Chart masterkeySecretName/Key
+        master-key: "{{ .MASTER_KEY }}"
+        # Chart db.secret + bundled Postgres existingSecret
+        username: litellm
+        password: "{{ .DB_PASSWORD }}"
+        postgres-password: "{{ .DB_ADMIN_PASSWORD }}"
+  data:
+    - secretKey: MASTER_KEY
+      remoteRef: { key: litellm, property: master-key }
+    - secretKey: DB_PASSWORD
+      remoteRef: { key: litellm, property: db-password }
+    - secretKey: DB_ADMIN_PASSWORD
+      remoteRef: { key: litellm, property: db-admin-password }
+---
+# Exported wholesale as env vars via chart `environmentSecrets`
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: litellm-env
+  namespace: litellm
+  annotations:
+    argocd.argoproj.io/sync-wave: "-1"
+spec:
+  refreshInterval: "1h"
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: onepassword-connect
+  target:
+    name: litellm-env
+    creationPolicy: Owner
+    template:
+      engineVersion: v2
+      data:
+        LITELLM_SALT_KEY: "{{ .SALT_KEY }}"
+        # Full URL for the proxy Deployment (see values.yaml environmentSecrets note)
+        DATABASE_URL: "postgresql://litellm:{{ .DB_PASSWORD | urlquery }}@litellm-postgresql:5432/litellm"
+        ILLUMINATIONS_API_KEY: "{{ .ILLUMINATIONS_API_KEY }}"
+        FANTASMIC_API_KEY: "{{ .FANTASMIC_API_KEY }}"
+        WISHES_API_KEY: "{{ .WISHES_API_KEY }}"
+  data:
+    - secretKey: DB_PASSWORD
+      remoteRef: { key: litellm, property: db-password }
+    - secretKey: SALT_KEY
+      remoteRef: { key: litellm, property: salt-key }
+    - secretKey: ILLUMINATIONS_API_KEY
+      remoteRef: { key: illuminations, property: api-key }
+    - secretKey: FANTASMIC_API_KEY
+      remoteRef: { key: fantasmic, property: api-key }
+    - secretKey: WISHES_API_KEY
+      remoteRef: { key: wishes, property: api-key }

cluster/applications/litellm/httproute.yaml

@@ -0,0 +1,31 @@
+# HTTPRoute — LiteLLM (skyliner) via Cilium fastpass-gateway
+# Certificate + dedicated listener live in cluster/platform/gateway
+# (certificate-skyliner.yaml, gateway.yaml listener "https-skyliner").
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: skyliner
+  namespace: litellm
+  annotations:
+    external-dns.alpha.kubernetes.io/hostname: skyliner.wdw-labs.cloud
+    external-dns.alpha.kubernetes.io/target: "10.1.71.90"
+spec:
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: fastpass-gateway
+      namespace: gateway
+      sectionName: https-skyliner
+  hostnames:
+    - skyliner.wdw-labs.cloud
+  rules:
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /
+      backendRefs:
+        - group: ""
+          kind: Service
+          name: litellm
+          port: 4000
+          weight: 1

cluster/applications/litellm/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: litellm

cluster/applications/litellm/values.yaml

@@ -0,0 +1,170 @@
+# ============================================================================
+# LiteLLM Helm values — overrides for oci://ghcr.io/berriai/litellm-helm 1.106.1
+# Cluster: fastpass (Talos). Host: skyliner.wdw-labs.cloud. Wave 8.
+# Upstream defaults for reference: values.upstream.yaml (same dir; not applied).
+# ============================================================================
+
+replicaCount: 1
+
+image:
+  repository: ghcr.io/berriai/litellm
+  # Empty tag => chart appVersion (1.106.1). Version is pinned via chart version
+  # in application.yaml; bump both together.
+  tag: ""
+  pullPolicy: IfNotPresent
+
+# Exposure is via Gateway API HTTPRoute (httproute.yaml), not the chart ingress.
+ingress:
+  enabled: false
+
+service:
+  type: ClusterIP
+  port: 4000
+
+resources:
+  requests:
+    cpu: 250m
+    memory: 512Mi
+  limits:
+    memory: 2Gi
+
+# --- Secrets -----------------------------------------------------------------
+# Master key from ExternalSecret litellm-credentials (key: master-key) -> PROXY_MASTER_KEY
+masterkeySecretName: litellm-credentials
+masterkeySecretKey: master-key
+
+# All keys of litellm-env exported as env vars (LITELLM_SALT_KEY, ILLUMINATIONS_/FANTASMIC_/WISHES_API_KEY)
+# (includes DATABASE_URL: the Deployment template ignores db.useExisting and would
+#  otherwise use the chart's placeholder DB password; envFrom supplies the real URL.
+#  Not extraEnvVars: the chart also injects those into the migration Job -> duplicate env.)
+environmentSecrets:
+  - litellm-env
+
+# --- Database: bundled Bitnami PostgreSQL ------------------------------------
+db:
+  # Bundled Postgres is still deployed (deployStandalone), but the app and the
+  # migration job are pointed at it as an "existing" DB so credentials come from
+  # litellm-credentials (ExternalSecret) instead of the chart's placeholder secret.
+  deployStandalone: true
+  useExisting: true
+  endpoint: litellm-postgresql
+  database: litellm
+  secret:
+    name: litellm-credentials
+    usernameKey: username
+    passwordKey: password
+
+postgresql:
+  architecture: standalone
+  auth:
+    username: litellm
+    database: litellm
+    existingSecret: litellm-credentials
+    secretKeys:
+      adminPasswordKey: postgres-password
+      userPasswordKey: password
+  primary:
+    persistence:
+      enabled: true
+      storageClass: px-fa-direct-access   # Portworx direct access to FlashArray (as openviking)
+      size: 10Gi
+    resources:
+      requests:
+        cpu: 100m
+        memory: 256Mi
+      limits:
+        memory: 1Gi
+
+redis:
+  enabled: false
+
+# Schema migrations. The chart's default is an ArgoCD PreSync hook, which runs
+# BEFORE Postgres/ExternalSecrets exist and would block first install. Instead run
+# it as a Sync-phase hook ordered by wave: ExternalSecrets (-1) -> Postgres (0)
+# -> migration Job (1) -> Deployment (2).
+migrationJob:
+  enabled: true
+  hooks:
+    argocd:
+      enabled: false
+    helm:
+      enabled: false
+  jobAnnotations:
+    argocd.argoproj.io/hook: Sync
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+    argocd.argoproj.io/sync-wave: "1"
+
+deploymentAnnotations:
+  argocd.argoproj.io/sync-wave: "2"
+
+logLevel: INFO
+
+# --- Prometheus ----------------------------------------------------------------
+serviceMonitor:
+  enabled: true
+  labels:
+    release: monitoring        # kube-prometheus-stack selector (matches Harbor/external-dns)
+  interval: 30s
+  scrapeTimeout: 10s
+  namespaceSelector:
+    matchNames:
+      - litellm
+
+# --- LiteLLM proxy config (rendered to ConfigMap) ------------------------------
+proxy_config:
+  model_list:
+    # --- illuminations (RTX 3090, vLLM) --------------------------------------
+    - model_name: Gemma-4-26B-A4B-it-AWQ
+      litellm_params:
+        model: openai/Gemma-4-26B-A4B-it-AWQ
+        api_base: http://illuminations.wdw-labs.cloud:8000/v1
+        api_key: os.environ/ILLUMINATIONS_API_KEY
+
+    # --- Mac Studio (fantasmic, oMLX) — Tier 1 Hermes ------------------------
+    - model_name: Gemma-4-26B-A4B-Hermes-VLM-MLX
+      litellm_params:
+        model: openai/Gemma-4-26B-A4B-Hermes-VLM-MLX
+        api_base: http://fantasmic.litellm.svc.cluster.local:8000/v1
+        api_key: os.environ/FANTASMIC_API_KEY
+    - model_name: Qwen3.8-27B-4bit
+      litellm_params:
+        model: openai/mlx-community--Qwen3.8-27B-4bit
+        api_base: http://fantasmic.litellm.svc.cluster.local:8000/v1
+        api_key: os.environ/FANTASMIC_API_KEY
+    # Also served, not exposed: incoai--Qwen3.8-27B-DFlash2, ukisai-Swift-Qwen3.8-27b-MTPLX
+
+    # --- Mac Mini M4 (wishes, oMLX) — Tier 3 aux tasks -----------------------
+    - model_name: Qwen2.5-7B-Instruct-4bit
+      litellm_params:
+        model: openai/Qwen2.5-7B-Instruct-4bit
+        api_base: http://wishes.litellm.svc.cluster.local:8000/v1
+        api_key: os.environ/WISHES_API_KEY
+    - model_name: Phi-3.5-mini-instruct-4bit
+      litellm_params:
+        model: openai/Phi-3.5-mini-instruct-4bit
+        api_base: http://wishes.litellm.svc.cluster.local:8000/v1
+        api_key: os.environ/WISHES_API_KEY
+    - model_name: Phi-3.5-mini-instruct-bf16
+      litellm_params:
+        model: openai/Phi-3.5-mini-instruct-bf16
+        api_base: http://wishes.litellm.svc.cluster.local:8000/v1
+        api_key: os.environ/WISHES_API_KEY
+    - model_name: Phi-3.5-vision-instruct-4bit
+      litellm_params:
+        model: openai/Phi-3.5-vision-instruct-4bit
+        api_base: http://wishes.litellm.svc.cluster.local:8000/v1
+        api_key: os.environ/WISHES_API_KEY
+    - model_name: gemma-2-2b-it-4bit
+      litellm_params:
+        model: openai/gemma-2-2b-it-4bit
+        api_base: http://wishes.litellm.svc.cluster.local:8000/v1
+        api_key: os.environ/WISHES_API_KEY
+
+  litellm_settings:
+    drop_params: true
+    callbacks: ["prometheus"]     # exposes /metrics for the ServiceMonitor
+
+  general_settings:
+    master_key: os.environ/PROXY_MASTER_KEY
+    # Allow adding models / virtual keys from the Admin UI (/ui), stored in Postgres
+    store_model_in_db: true

cluster/platform/cert-manager/clusterissuer.yaml

@@ -25,6 +25,7 @@ spec:
         selector:
           dnsZones:
             - mk-labs.cloud
+            - wdw-labs.cloud
 
 ---
 # Production issuer — switch to this once staging certs are working
@@ -47,3 +48,4 @@ spec:
         selector:
           dnsZones:
             - mk-labs.cloud
+            - wdw-labs.cloud

cluster/platform/external-dns/values.yaml

@@ -16,8 +16,6 @@ env:
     value: "10.1.71.32"
   - name: EXTERNAL_DNS_RFC2136_PORT
     value: "53"
-  - name: EXTERNAL_DNS_RFC2136_ZONE
-    value: "local.mk-labs.cloud"
   - name: EXTERNAL_DNS_RFC2136_TSIG_AXFR
     value: "true"
   # TSIG secret key name — must match what's configured in Technitium
@@ -34,6 +32,14 @@ env:
 # Only manage records in this zone
 domainFilters:
   - local.mk-labs.cloud
+  - wdw-labs.cloud
+
+# Multiple Technitium zones (the env var form only supports one zone).
+# wdw-labs.cloud zone must exist in Technitium and the "external-dns" TSIG key
+# must be authorised for dynamic updates + zone transfer on it (added for skyliner).
+extraArgs:
+  - --rfc2136-zone=local.mk-labs.cloud
+  - --rfc2136-zone=wdw-labs.cloud
 
 # Sources: ingress, service, and Gateway API HTTPRoutes
 sources:

cluster/platform/gateway/certificate-skyliner.yaml

@@ -0,0 +1,17 @@
+# ------------------------------------------------------------------------------
+# Certificate for skyliner.wdw-labs.cloud (LiteLLM)
+# Single-host cert; referenced by listener "https-skyliner" in gateway.yaml.
+# Requires letsencrypt-prod dnsZones to include wdw-labs.cloud (cert-manager).
+# ------------------------------------------------------------------------------
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: skyliner-wdw-labs-tls
+  namespace: gateway
+spec:
+  secretName: skyliner-wdw-labs-tls
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+  dnsNames:
+    - skyliner.wdw-labs.cloud

cluster/platform/gateway/gateway.yaml

@@ -36,3 +36,18 @@ spec:
       allowedRoutes:
         namespaces:
           from: All
+    # Dedicated listener for LiteLLM (skyliner) — wdw-labs.cloud is not covered
+    # by the wildcard cert above. Cert: certificate-skyliner.yaml
+    - name: https-skyliner
+      port: 443
+      protocol: HTTPS
+      hostname: skyliner.wdw-labs.cloud
+      tls:
+        mode: Terminate
+        certificateRefs:
+          - group: ""
+            kind: Secret
+            name: skyliner-wdw-labs-tls
+      allowedRoutes:
+        namespaces:
+          from: All

cluster/platform/gateway/referencegrant.yaml

@@ -16,6 +16,7 @@
 #   - monitoring    (Grafana)
 #   - authentik     (Authentik SSO — future)
 #   - gitea         (Gitea — future)
+#   - litellm       (LiteLLM / skyliner)
 # ------------------------------------------------------------------------------
 apiVersion: gateway.networking.k8s.io/v1beta1
 kind: ReferenceGrant
@@ -45,6 +46,9 @@ spec:
     - group: gateway.networking.k8s.io
       kind: HTTPRoute
       namespace: firecrawl
+    - group: gateway.networking.k8s.io
+      kind: HTTPRoute
+      namespace: litellm
   to:
     - group: gateway.networking.k8s.io
       kind: Gateway

cluster/platform/onepassword-connect/cluster-secret-store.yaml

@@ -24,6 +24,7 @@ spec:
       connectHost: http://onepassword-connect.onepassword-connect.svc.cluster.local:8080
       vaults:
         mk-labs: 1          # vault name in 1Password : priority
+        wdw-labs: 2         # WDW Labs hosts/agents (LiteLLM, host api-keys); Connect token needs read access
       auth:
         secretRef:
           connectTokenSecretRef: