From d95477fc3b5da0df412c69f854f595ebaec8ca7e Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Tue, 1 Sep 2026 13:14:03 -0500 Subject: [PATCH 01/13] Fix Open WebUI connectivity to astro-orbiter - Add ExternalName Service for astro-orbiter (10.1.71.130:8000) - Remove Ollama-specific configuration (OLLAMA_BASE_URLS) - Keep only OpenAI-compatible configuration (OPENAI_BASE_URL) Fixes: Open WebUI pod was attempting to use Ollama endpoints (/api/tags) against vLLM which only supports OpenAI-compatible API (/v1/models). Pod now reaches astro-orbiter via Kubernetes Service DNS name. Resolves: HTTP 404 errors from Upstream ollama request failed --- .../applications/open-webui/deployment.yaml | 5 +-- .../externalname-service.yaml | 37 +++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 cluster/platform/astro-orbiter-gateway/externalname-service.yaml diff --git a/cluster/applications/open-webui/deployment.yaml b/cluster/applications/open-webui/deployment.yaml index d0f1087..241a59c 100644 --- a/cluster/applications/open-webui/deployment.yaml +++ b/cluster/applications/open-webui/deployment.yaml @@ -39,9 +39,8 @@ spec: name: body-wars-secret key: vllm-api-key optional: false - # Backend configuration for vLLM - - name: OLLAMA_BASE_URLS - value: "http://astro-orbiter:8000" + # Backend configuration for vLLM (OpenAI-compatible, NOT Ollama) + # OLLAMA_BASE_URLS removed: astro-orbiter runs vLLM, not Ollama - name: OPENAI_BASE_URL value: "http://astro-orbiter:8000/v1" - name: OPENAI_API_KEY diff --git a/cluster/platform/astro-orbiter-gateway/externalname-service.yaml b/cluster/platform/astro-orbiter-gateway/externalname-service.yaml new file mode 100644 index 0000000..e4d44b1 --- /dev/null +++ b/cluster/platform/astro-orbiter-gateway/externalname-service.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: Service +metadata: + name: astro-orbiter + namespace: default + labels: + app.kubernetes.io/name: astro-orbiter-gateway + app.kubernetes.io/component: llm-inference +spec: + type: ExternalName + externalName: 10.1.71.130 + ports: + - name: vllm-8000 + port: 8000 + protocol: TCP + - name: llama-cpp-8001 + port: 8001 + protocol: TCP + - name: router-8002 + port: 8002 + protocol: TCP +--- +apiVersion: v1 +kind: Service +metadata: + name: astro-orbiter-vllm + namespace: default + labels: + app.kubernetes.io/name: astro-orbiter-gateway + app.kubernetes.io/component: llm-inference +spec: + type: ExternalName + externalName: 10.1.71.130 + ports: + - name: vllm + port: 8000 + protocol: TCP From 3dc58cf644e1b9180e5ac8b279545abc42760e3f Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Tue, 1 Sep 2026 13:18:55 -0500 Subject: [PATCH 02/13] Fix Open WebUI auth: correct WEBUI_SECRET_KEY and remove Ollama config - WEBUI_SECRET_KEY now uses dedicated webui-secret-key (not vllm-api-key) - ExternalSecret updated to fetch webui_secret_key from 1Password - Remove duplicate OLLAMA_BASE_URLS configuration (vLLM only) --- cluster/applications/open-webui/deployment.yaml | 2 +- cluster/applications/open-webui/externalsecret.yaml | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cluster/applications/open-webui/deployment.yaml b/cluster/applications/open-webui/deployment.yaml index 241a59c..2978a5a 100644 --- a/cluster/applications/open-webui/deployment.yaml +++ b/cluster/applications/open-webui/deployment.yaml @@ -37,7 +37,7 @@ spec: valueFrom: secretKeyRef: name: body-wars-secret - key: vllm-api-key + key: webui-secret-key optional: false # Backend configuration for vLLM (OpenAI-compatible, NOT Ollama) # OLLAMA_BASE_URLS removed: astro-orbiter runs vLLM, not Ollama diff --git a/cluster/applications/open-webui/externalsecret.yaml b/cluster/applications/open-webui/externalsecret.yaml index 81260d1..93afbf5 100644 --- a/cluster/applications/open-webui/externalsecret.yaml +++ b/cluster/applications/open-webui/externalsecret.yaml @@ -18,9 +18,15 @@ spec: engineVersion: v2 data: vllm-api-key: "{{ .vllm_api_key }}" + webui-secret-key: "{{ .webui_secret_key }}" data: # vLLM API key from 1Password (mk-labs vault, vllm item, api-key field) - secretKey: vllm_api_key remoteRef: key: vllm property: api-key + # WebUI JWT signing secret from 1Password (mk-labs vault, open-webui item, secret-key field) + - secretKey: webui_secret_key + remoteRef: + key: open-webui + property: secret-key From e80a1dc08884d4e6d85d5f6fbaceeb6f928271c0 Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Tue, 1 Sep 2026 13:19:18 -0500 Subject: [PATCH 03/13] Temporarily disable webui-secret-key from ExternalSecret The 'open-webui' item doesn't exist in 1Password yet. webui-secret-key is manually patched in the secret for now. Re-enable ExternalSecret pull once the 1Password item is created. --- .../open-webui/externalsecret.yaml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/cluster/applications/open-webui/externalsecret.yaml b/cluster/applications/open-webui/externalsecret.yaml index 93afbf5..d0f332e 100644 --- a/cluster/applications/open-webui/externalsecret.yaml +++ b/cluster/applications/open-webui/externalsecret.yaml @@ -18,15 +18,22 @@ spec: engineVersion: v2 data: vllm-api-key: "{{ .vllm_api_key }}" - webui-secret-key: "{{ .webui_secret_key }}" + # NOTE: webui-secret-key is manually managed as a Kubernetes secret patch + # until the 'open-webui' item exists in 1Password with a 'secret-key' field. + # To re-enable automatic sync: + # 1. Add open-webui item to 1Password mk-labs vault + # 2. Add secret-key field with a strong random value + # 3. Uncomment the data section below and re-apply this manifest + webui-secret-key: "{{ .webui_secret_key | default \"PLACEHOLDER\" }}" data: # vLLM API key from 1Password (mk-labs vault, vllm item, api-key field) - secretKey: vllm_api_key remoteRef: key: vllm property: api-key - # WebUI JWT signing secret from 1Password (mk-labs vault, open-webui item, secret-key field) - - secretKey: webui_secret_key - remoteRef: - key: open-webui - property: secret-key + # WebUI JWT signing secret - TEMPORARILY DISABLED + # Uncomment and configure once open-webui item exists in 1Password: + # - secretKey: webui_secret_key + # remoteRef: + # key: open-webui + # property: secret-key From f51d1c16ac6e425eb1f8672107deeb02a7147a50 Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Tue, 1 Sep 2026 13:28:21 -0500 Subject: [PATCH 04/13] Fix Open WebUI storage and model detection - Change emptyDir to persistent volume (open-webui-data PVC) Fixes data loss on pod restart (admin user disappears) - Add ENABLE_OLLAMA_API=false Disable broken Ollama endpoint probing vLLM uses OpenAI-compatible API, not Ollama - Create pvc.yaml for persistent data storage 10Gi NFS-backed storage via nfs-emporium StorageClass --- cluster/applications/open-webui/deployment.yaml | 7 +++++-- cluster/applications/open-webui/pvc.yaml | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 cluster/applications/open-webui/pvc.yaml diff --git a/cluster/applications/open-webui/deployment.yaml b/cluster/applications/open-webui/deployment.yaml index 2978a5a..0b4542d 100644 --- a/cluster/applications/open-webui/deployment.yaml +++ b/cluster/applications/open-webui/deployment.yaml @@ -59,6 +59,9 @@ spec: value: "true" - name: ENABLE_MODEL_FILTER value: "false" + # Disable Ollama model detection (we use vLLM/OpenAI-compatible only) + - name: ENABLE_OLLAMA_API + value: "false" # Storage - name: DATA_DIR value: "/app/backend/data" @@ -82,5 +85,5 @@ spec: mountPath: /app/backend/data volumes: - name: data - emptyDir: - sizeLimit: 1Gi + persistentVolumeClaim: + claimName: open-webui-data diff --git a/cluster/applications/open-webui/pvc.yaml b/cluster/applications/open-webui/pvc.yaml new file mode 100644 index 0000000..f8af8f5 --- /dev/null +++ b/cluster/applications/open-webui/pvc.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: open-webui-data + namespace: open-webui + labels: + app.kubernetes.io/name: open-webui + app.kubernetes.io/component: data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi From f4b1fc9e7117d8af8f89c19316a6d5f8758d2b08 Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Tue, 1 Sep 2026 13:30:24 -0500 Subject: [PATCH 05/13] Fix ArgoCD Application destination namespace Change namespace from 'body-wars' to 'open-webui' to match actual deployment --- cluster/applications/open-webui/application.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/applications/open-webui/application.yaml b/cluster/applications/open-webui/application.yaml index aae8c2c..ee13c9d 100644 --- a/cluster/applications/open-webui/application.yaml +++ b/cluster/applications/open-webui/application.yaml @@ -38,7 +38,7 @@ spec: destination: server: https://kubernetes.default.svc - namespace: body-wars + namespace: open-webui syncPolicy: automated: From 1e537cf5f70e4ebb9fb5cb239df10a3519feb92f Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Tue, 1 Sep 2026 13:31:18 -0500 Subject: [PATCH 06/13] Add storageClassName to PVC manifest for explicit NFS declaration --- cluster/applications/open-webui/pvc.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster/applications/open-webui/pvc.yaml b/cluster/applications/open-webui/pvc.yaml index f8af8f5..519ed04 100644 --- a/cluster/applications/open-webui/pvc.yaml +++ b/cluster/applications/open-webui/pvc.yaml @@ -9,6 +9,7 @@ metadata: spec: accessModes: - ReadWriteOnce + storageClassName: nfs-emporium resources: requests: storage: 10Gi From dd5ce10910e89ada0b1383ecab06e7a15364111f Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Tue, 1 Sep 2026 13:38:11 -0500 Subject: [PATCH 07/13] Fix ExternalSecret template: remove webui-secret-key reference The open-webui item doesn't exist in 1Password yet. webui-secret-key is manually managed via Kubernetes secret patch. Template now only pulls vllm-api-key from 1Password. --- cluster/applications/open-webui/externalsecret.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cluster/applications/open-webui/externalsecret.yaml b/cluster/applications/open-webui/externalsecret.yaml index d0f332e..57be4ad 100644 --- a/cluster/applications/open-webui/externalsecret.yaml +++ b/cluster/applications/open-webui/externalsecret.yaml @@ -18,13 +18,6 @@ spec: engineVersion: v2 data: vllm-api-key: "{{ .vllm_api_key }}" - # NOTE: webui-secret-key is manually managed as a Kubernetes secret patch - # until the 'open-webui' item exists in 1Password with a 'secret-key' field. - # To re-enable automatic sync: - # 1. Add open-webui item to 1Password mk-labs vault - # 2. Add secret-key field with a strong random value - # 3. Uncomment the data section below and re-apply this manifest - webui-secret-key: "{{ .webui_secret_key | default \"PLACEHOLDER\" }}" data: # vLLM API key from 1Password (mk-labs vault, vllm item, api-key field) - secretKey: vllm_api_key From f0387c1033885e80bf96eaca704a3e68a326950a Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Tue, 1 Sep 2026 13:39:14 -0500 Subject: [PATCH 08/13] Simplify: use vllm-api-key for WEBUI_SECRET_KEY ExternalSecret only provides vllm-api-key from 1Password. Using same key for both purposes is acceptable for now. Avoids need for manual secret patching. --- cluster/applications/open-webui/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/applications/open-webui/deployment.yaml b/cluster/applications/open-webui/deployment.yaml index 0b4542d..87dcd02 100644 --- a/cluster/applications/open-webui/deployment.yaml +++ b/cluster/applications/open-webui/deployment.yaml @@ -37,7 +37,7 @@ spec: valueFrom: secretKeyRef: name: body-wars-secret - key: webui-secret-key + key: vllm-api-key optional: false # Backend configuration for vLLM (OpenAI-compatible, NOT Ollama) # OLLAMA_BASE_URLS removed: astro-orbiter runs vLLM, not Ollama From 56db1b94ee20b1e61ccb91e820b015fffb2e17d3 Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Tue, 1 Sep 2026 13:44:41 -0500 Subject: [PATCH 09/13] Disable OpenAI API key validation for local vLLM endpoint Open WebUI was validating API keys against OpenAI's servers. For local vLLM backend, disable this validation. --- cluster/applications/open-webui/deployment.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cluster/applications/open-webui/deployment.yaml b/cluster/applications/open-webui/deployment.yaml index 87dcd02..99ffb55 100644 --- a/cluster/applications/open-webui/deployment.yaml +++ b/cluster/applications/open-webui/deployment.yaml @@ -51,6 +51,9 @@ spec: optional: false - name: OPENAI_MODEL_FALLBACK value: "Gemma-4-26B-A4B-it-AWQ" + # Disable OpenAI API key validation (we're using vLLM, not OpenAI) + - name: OPENAI_VALIDATE_ENDPOINT + value: "false" # Allow access from ingress - name: WEBUI_URL value: "https://body-wars.local.mk-labs.cloud" From 3c6f6dfe2eaeb42b21c6a581cf85ef34c54d2be2 Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Tue, 1 Sep 2026 13:48:37 -0500 Subject: [PATCH 10/13] Fix: Use local astro-orbiter endpoint, not OpenAI API Set OPENAI_API_BASE_URL (config.py default) to our local vLLM endpoint. Open WebUI was using hardcoded 'https://api.openai.com/v1' for model discovery. --- cluster/applications/open-webui/deployment.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cluster/applications/open-webui/deployment.yaml b/cluster/applications/open-webui/deployment.yaml index 99ffb55..58edfef 100644 --- a/cluster/applications/open-webui/deployment.yaml +++ b/cluster/applications/open-webui/deployment.yaml @@ -43,6 +43,8 @@ spec: # OLLAMA_BASE_URLS removed: astro-orbiter runs vLLM, not Ollama - name: OPENAI_BASE_URL value: "http://astro-orbiter:8000/v1" + - name: OPENAI_API_BASE_URL + value: "http://astro-orbiter:8000/v1" - name: OPENAI_API_KEY valueFrom: secretKeyRef: From 404ff3d91d1d35beeeeeebcf81f2d00948211f84 Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Tue, 1 Sep 2026 13:49:20 -0500 Subject: [PATCH 11/13] Set OPENAI_API_BASE_URLS (plural) for model discovery --- cluster/applications/open-webui/deployment.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cluster/applications/open-webui/deployment.yaml b/cluster/applications/open-webui/deployment.yaml index 58edfef..df783c1 100644 --- a/cluster/applications/open-webui/deployment.yaml +++ b/cluster/applications/open-webui/deployment.yaml @@ -43,6 +43,8 @@ spec: # OLLAMA_BASE_URLS removed: astro-orbiter runs vLLM, not Ollama - name: OPENAI_BASE_URL value: "http://astro-orbiter:8000/v1" + - name: OPENAI_API_BASE_URLS + value: "http://astro-orbiter:8000/v1" - name: OPENAI_API_BASE_URL value: "http://astro-orbiter:8000/v1" - name: OPENAI_API_KEY From 7e4b103e68b9703c32441ef01bbc5f64a9eb9cc3 Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Tue, 1 Sep 2026 13:58:44 -0500 Subject: [PATCH 12/13] Clean up duplicate OPENAI_BASE_URL env vars; keep only one --- cluster/applications/open-webui/deployment.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cluster/applications/open-webui/deployment.yaml b/cluster/applications/open-webui/deployment.yaml index df783c1..99ffb55 100644 --- a/cluster/applications/open-webui/deployment.yaml +++ b/cluster/applications/open-webui/deployment.yaml @@ -43,10 +43,6 @@ spec: # OLLAMA_BASE_URLS removed: astro-orbiter runs vLLM, not Ollama - name: OPENAI_BASE_URL value: "http://astro-orbiter:8000/v1" - - name: OPENAI_API_BASE_URLS - value: "http://astro-orbiter:8000/v1" - - name: OPENAI_API_BASE_URL - value: "http://astro-orbiter:8000/v1" - name: OPENAI_API_KEY valueFrom: secretKeyRef: From 49989b1757941b6671ebd024657521f8b43e7a11 Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Tue, 1 Sep 2026 09:45:03 -0500 Subject: [PATCH 13/13] Migrate Body Wars to Helm chart --- .../body-wars-helm/application.yaml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 cluster/applications/body-wars-helm/application.yaml diff --git a/cluster/applications/body-wars-helm/application.yaml b/cluster/applications/body-wars-helm/application.yaml new file mode 100644 index 0000000..1fa3c24 --- /dev/null +++ b/cluster/applications/body-wars-helm/application.yaml @@ -0,0 +1,39 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: body-wars + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "9" +spec: + project: default + source: + repoURL: https://helm-charts.open-webui.com + chart: open-webui + targetRevision: 16.3.0 + helm: + values: | + ingress: + enabled: true + className: nginx + hosts: + - host: body-wars.local.mk-labs.cloud + paths: + - path: / + pathType: Prefix + tls: + - secretName: body-wars-tls + hosts: + - body-wars.local.mk-labs.cloud + openwebui: + backend: + url: http://astro-orbiter:8000/v1 + destination: + server: https://kubernetes.default.svc + namespace: open-webui + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true