repo cleanup

cluster/archive/argocd/application.yaml

@@ -0,0 +1,46 @@
+# ------------------------------------------------------------------------------
+# Platform: ArgoCD Configuration (post-bootstrap)
+# Wave 2 — after ESO (wave 1), needs 1Password for Authentik client secret
+#
+# This is NOT the ArgoCD install (that's a manual bootstrap step).
+# This Application manages ArgoCD's own configuration declaratively:
+#   - argocd-cm       (URL, Dex/OIDC config, repo credentials)
+#   - argocd-rbac-cm  (RBAC policies mapped from Authentik groups)
+#   - argocd-secret   (Authentik client secret via ESO)
+#
+# ArgoCD managing its own config is intentional and works cleanly —
+# it's the "self-managed ArgoCD" pattern.
+# ------------------------------------------------------------------------------
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: argocd-config
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
+spec:
+  project: default
+  source:
+    repoURL: https://gitea.mk-labs.cloud/rblundon/homelab.git
+    targetRevision: main
+    path: cluster/platform/argocd
+    directory:
+        exclude: "application.yaml"
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: argocd
+  syncPolicy:
+    automated:
+      prune: false
+      selfHeal: true
+    syncOptions:
+      - ServerSideApply=true
+  ignoreDifferences:
+    # ArgoCD mutates its own secret — ignore those fields
+    - group: ""
+      kind: Secret
+      name: argocd-secret
+      jsonPointers:
+        - /data/admin.password
+        - /data/admin.passwordMtime
+        - /data/server.secretkey

cluster/archive/argocd/argocd-cm.yaml

@@ -0,0 +1,69 @@
+# ------------------------------------------------------------------------------
+# argocd-cm — ArgoCD Core Configuration
+#
+# Configures:
+#   - ArgoCD public URL (used in Dex redirect URIs)
+#   - Dex connector to Authentik OIDC
+#   - Gitea repo credentials reference
+#
+# Authentik side setup (manual — do this in guest-relations before bootstrapping):
+#   1. Admin UI → Applications → Create New Application
+#   2. Provider type: OAuth2/OpenID Connect
+#   3. Name: argocd
+#   4. Authorization flow: default-provider-authorization-explicit-consent
+#   5. Client type: Confidential
+#   6. Strict redirect URIs:
+#        https://argocd.local.mk-labs.cloud/api/dex/callback
+#        http://localhost:8085/auth/callback        (ArgoCD CLI)
+#   7. Scopes: openid, profile, email, groups
+#   8. Note the Client ID and Client Secret → store in 1Password as:
+#        Item: "argocd-oidc"
+#        Fields: client-id, client-secret
+#   9. Create two groups in Authentik:
+#        - argocd-admins  (add your user)
+#        - argocd-viewers (read-only users)
+# ------------------------------------------------------------------------------
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: argocd-cm
+  namespace: argocd
+data:
+  # Public URL — must match the HTTPRoute hostname
+  url: https://argocd.local.mk-labs.cloud
+
+  # Dex connector to Authentik
+  # Using Dex (not native OIDC) to preserve ArgoCD CLI functionality
+  dex.config: |
+    connectors:
+      - type: oidc
+        id: authentik
+        name: Authentik
+        config:
+          # Authentik OIDC issuer — slug must match the Application slug in Authentik
+          # Slug is auto-generated from the app name, usually lowercase with hyphens
+          issuer: https://authentik.local.mk-labs.cloud/application/o/argocd/
+          clientID: $dex.authentik.clientID
+          clientSecret: $dex.authentik.clientSecret
+          redirectURI: https://argocd.local.mk-labs.cloud/api/dex/callback
+          scopes:
+            - openid
+            - profile
+            - email
+            - groups
+          insecureEnableGroups: true
+          groupsKey: groups
+          userIDKey: sub
+          userNameKey: preferred_username
+          emailKey: email
+
+  # Gitea repo — ArgoCD needs credentials to pull from your private repo
+  # Credentials are stored as a Repository secret (see repository-secret.yaml)
+  # This tells ArgoCD to use SSH for the repo
+  repositories: |
+    - url: git@gitea.mk-labs.cloud:rblundon/homelab.git
+      type: git
+      name: homelab
+
+  # Resource tracking via annotation (recommended over label tracking)
+  application.resourceTrackingMethod: annotation

cluster/archive/argocd/argocd-params.yaml

@@ -0,0 +1,11 @@
+# ------------------------------------------------------------------------------
+# ArgoCD Server Parameters
+# Sets server.insecure=true so TLS is terminated at ingress, not argocd-server
+# ------------------------------------------------------------------------------
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: argocd-cmd-params-cm
+  namespace: argocd
+data:
+  server.insecure: "true"

cluster/archive/argocd/argocd-rbac-cm.yaml

@@ -0,0 +1,30 @@
+# ------------------------------------------------------------------------------
+# argocd-rbac-cm — RBAC Policy
+#
+# Maps Authentik groups to ArgoCD roles.
+# Groups must exist in Authentik and be assigned to users there.
+#
+# Roles:
+#   role:admin    — full access, all operations
+#   role:readonly — read-only, can view but not sync/modify
+#
+# Single-user homelab: your user should be in argocd-admins in Authentik.
+# Add argocd-viewers later when you have guests/collaborators.
+# ------------------------------------------------------------------------------
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: argocd-rbac-cm
+  namespace: argocd
+data:
+  # Default policy for any authenticated user not in a mapped group
+  # readonly is safe — they can see but not touch
+  policy.default: role:readonly
+
+  policy.csv: |
+    # Authentik group → ArgoCD role mappings
+    g, argocd-admins, role:admin
+    g, argocd-viewers, role:readonly
+
+  # Tell ArgoCD which OIDC claims to use for group mapping
+  scopes: '[groups, email]'

cluster/archive/argocd/certificate.yaml

@@ -0,0 +1,41 @@
+# ------------------------------------------------------------------------------
+# Certificate — ArgoCD TLS
+# Issued by cert-manager via letsencrypt-prod + Cloudflare DNS-01
+#
+# The secret produced (argocd-tls) is referenced by the Gateway listener
+# in cluster/platform/gateway-api/gateway.yaml via ReferenceGrant.
+#
+# Per the thematic naming convention: argocd.local.mk-labs.cloud is the
+# service DNS name. No VM hostname alias needed here.
+# ------------------------------------------------------------------------------
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: argocd-tls
+  namespace: argocd
+spec:
+  secretName: argocd-tls
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+  dnsNames:
+    - argocd.local.mk-labs.cloud
+
+---
+# ReferenceGrant — allows the Gateway in gateway-system to reference
+# the TLS secret in the argocd namespace
+# Without this, cross-namespace secret references are denied
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: ReferenceGrant
+metadata:
+  name: argocd-tls-grant
+  namespace: argocd
+spec:
+  from:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      namespace: gateway-system
+  to:
+    - group: ""
+      kind: Secret
+      name: argocd-tls

cluster/archive/argocd/externalsecret.yaml

@@ -0,0 +1,49 @@
+# ------------------------------------------------------------------------------
+# ExternalSecret — ArgoCD Secrets
+#
+# Pulls two secrets from 1Password:
+#   1. Authentik OIDC client ID + secret (for Dex connector)
+#   2. Gitea personal access token (for repo access)
+#
+# In 1Password (mk-labs vault), create:
+#   Item: "argocd-oidc"
+#     Field: client-id     → Authentik OAuth2 client ID
+#     Field: client-secret → Authentik OAuth2 client secret
+#
+#   Item: "gitea-argocd-token"
+#     Field: token → Gitea personal access token with repo read permissions
+#     (Create in Gitea: Settings → Applications → Generate Token)
+#
+# These materialize into argocd-secret, which ArgoCD reads natively.
+# The $dex.authentik.clientID / $dex.authentik.clientSecret references
+# in argocd-cm.yaml are resolved by ArgoCD from this secret automatically.
+# ------------------------------------------------------------------------------
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: argocd-secrets
+  namespace: argocd
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: onepassword-connect
+  refreshInterval: "1h"
+  target:
+    # Must be named argocd-secret — ArgoCD reads this specific secret name
+    name: argocd-secret
+    creationPolicy: Merge    # Merge into existing argocd-secret, don't replace it
+  data:
+    # Authentik OIDC credentials — referenced as $dex.authentik.* in argocd-cm
+    - secretKey: dex.authentik.clientID
+      remoteRef:
+        key: argocd-oidc
+        property: client-id
+    - secretKey: dex.authentik.clientSecret
+      remoteRef:
+        key: argocd-oidc
+        property: client-secret
+    # Gitea token — used by the Repository credential below
+    - secretKey: sshPrivateKey
+      remoteRef:
+        key: gitea-argocd-ssh
+        property: private-key

cluster/archive/argocd/ingress.yaml

@@ -0,0 +1,31 @@
+# ------------------------------------------------------------------------------
+# Ingress — ArgoCD via ingress-nginx
+# Temporary until Cilium Gateway API is stable
+# TLS terminated at ingress, argocd-server runs in insecure mode
+# ------------------------------------------------------------------------------
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: argocd
+  namespace: argocd
+  annotations:
+    nginx.ingress.kubernetes.io/ssl-passthrough: "false"
+    nginx.ingress.kubernetes.io/backend-protocol: "HTTP"
+    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
+spec:
+  ingressClassName: nginx
+  tls:
+    - hosts:
+        - argocd.local.mk-labs.cloud
+      secretName: argocd-tls
+  rules:
+    - host: argocd.local.mk-labs.cloud
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: argocd-server
+                port:
+                  number: 80

cluster/archive/argocd/repository-secret.yaml

@@ -0,0 +1,27 @@
+# ------------------------------------------------------------------------------
+# Repository Credential Secret — Gitea homelab repo
+#
+# ArgoCD discovers repository credentials via secrets with the label
+# argocd.argoproj.io/secret-type: repository
+#
+# The actual token value is injected by the ExternalSecret (externalsecret.yaml)
+# which merges into argocd-secret. This secret references that value.
+#
+# NOTE: The token field here uses a placeholder. The ExternalSecret populates
+# argocd-secret with gitea.token, and ArgoCD resolves it via the
+# stringData reference below.
+# ------------------------------------------------------------------------------
+apiVersion: v1
+kind: Secret
+metadata:
+  name: repo-homelab-gitea
+  namespace: argocd
+  labels:
+    argocd.argoproj.io/secret-type: repository
+stringData:
+  type: git
+  url: git@gitea.mk-labs.cloud:rblundon/homelab.git
+  sshPrivateKey: |
+    -----BEGIN OPENSSH PRIVATE KEY-----
+    # populated by ESO
+    -----END OPENSSH PRIVATE KEY-----

cluster/archive/cilium-config/application.yaml

@@ -0,0 +1,43 @@
+# ------------------------------------------------------------------------------
+# Platform: Cilium — Helm upgrade + LB IPAM + Gateway API
+# Wave 4 — no dependencies, Cilium is pre-installed via Talos bootstrap
+#
+# This Application does two things:
+#   1. Manages Cilium itself via Helm (enables Gateway API, keeps config in git)
+#   2. Applies LB pool and L2 announcement manifests from repo
+#
+# IP Allocation:
+#   10.1.71.80-.89  — ingress-nginx LB pool
+#   10.1.71.90-.99  — Cilium Gateway LB pool
+# ------------------------------------------------------------------------------
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: cilium-config
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "4"
+spec:
+  project: default
+  sources:
+    # Source 1: Cilium Helm chart — manages Cilium itself
+    - repoURL: https://helm.cilium.io
+      chart: cilium
+      targetRevision: 1.17.4
+      helm:
+        valueFiles:
+          - $values/cluster/platform/cilium-config/values.yaml
+    # Source 2: Repo — values ref + lb-pool.yaml + GatewayClass manifests
+    - repoURL: https://gitea.mk-labs.cloud/rblundon/homelab.git
+      targetRevision: main
+      path: cluster/platform/cilium-config/manifests
+      ref: values
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: kube-system
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - ServerSideApply=true

cluster/archive/cilium-config/manifests/lb-pool.yaml

@@ -0,0 +1,41 @@
+# Pool for ingress-nginx
+apiVersion: cilium.io/v2
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: ingress-nginx-pool
+  namespace: kube-system
+spec:
+  blocks:
+    - start: 10.1.71.80
+      stop: 10.1.71.89
+  serviceSelector:
+    matchLabels:
+      app.kubernetes.io/name: ingress-nginx
+
+---
+# Pool for Cilium Gateway API
+apiVersion: cilium.io/v2
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: cilium-gateway-pool
+  namespace: kube-system
+spec:
+  blocks:
+    - start: 10.1.71.90
+      stop: 10.1.71.99
+  serviceSelector:
+    matchExpressions:
+      - key: io.cilium.gateway/owning-gateway
+        operator: Exists
+
+---
+# L2 Announcement Policy
+apiVersion: cilium.io/v2alpha1
+kind: CiliumL2AnnouncementPolicy
+metadata:
+  name: fastpass-l2-policy
+  namespace: kube-system
+spec:
+  loadBalancerIPs: true
+  interfaces:
+    - ^eth[0-9]+

cluster/archive/cilium-config/values.yaml

@@ -0,0 +1,41 @@
+# ------------------------------------------------------------------------------
+# Cilium Helm Values — fastpass cluster
+# Chart: https://helm.cilium.io
+# Version: 1.17.4
+# ------------------------------------------------------------------------------
+
+# Bootstrap values
+kubeProxyReplacement: true
+k8sServiceHost: 10.1.71.65
+k8sServicePort: 6443
+ipam:
+  mode: kubernetes
+
+# Routing mode — replaces deprecated tunnel value
+routingMode: tunnel
+tunnelProtocol: vxlan
+
+# L2 announcements
+l2announcements:
+  enabled: true
+
+k8sClientRateLimit:
+  qps: 20
+  burst: 40
+
+externalIPs:
+  enabled: true
+
+# Hubble observability
+hubble:
+  enabled: true
+  relay:
+    enabled: true
+  ui:
+    enabled: true
+
+# Prometheus metrics
+prometheus:
+  enabled: true
+  serviceMonitor:
+    enabled: false

cluster/argocd/BOOTSTRAP.md

@@ -9,36 +9,23 @@ This document covers the manual steps required to bootstrap ArgoCD on the
 - IP pools applied (`kubectl get ciliumloadbalancerippool`)
 - `city-hall` has `helm`, `kubectl`, and `talosctl` installed
 - SSH keypair for ArgoCD → Gitea exists at `/home/wed/.ssh/argocd_gitea`
+- 1Password credentials file at `/home/wed/1password-credentials.json`
+- 1Password Connect token at `/home/wed/connect-token` (no trailing newline — use `echo -n`)
 
 ---
 
 ## Step 1 — Create the argocd namespace
 
+Must be created first — the Gitea repo secret targets this namespace before
+Helm creates it.
+
 ```bash
 kubectl apply -f cluster/argocd/namespace.yaml
 ```
 
 ---
 
-## Step 2 — Add Gitea to ArgoCD known hosts
-
-ArgoCD refuses SSH connections to unknown hosts. Pre-populate the known hosts
-configmap before ArgoCD starts:
-
-```bash
-ssh-keyscan -t ed25519 gitea.mk-labs.cloud | \
-  kubectl create configmap argocd-ssh-known-hosts-cm \
-  --namespace argocd \
-  --from-file=ssh_known_hosts=/dev/stdin \
-  --dry-run=client -o yaml | kubectl apply -f -
-```
-
----
-
-## Step 3 — Create the Gitea repo secret
-
-This is the only credential that must be manually created. Once ESO is running,
-secrets will be managed via 1Password. For now, create it directly:
+## Step 2 — Create the Gitea repo secret
 
 ```bash
 kubectl create secret generic gitea-repo \
@@ -58,7 +45,7 @@ kubectl -n argocd get secret gitea-repo
 
 ---
 
-## Step 4 — Install ArgoCD via Helm
+## Step 3 — Install ArgoCD via Helm
 
 ```bash
 helm repo add argo https://argoproj.github.io/argo-helm
@@ -80,6 +67,19 @@ kubectl -n argocd get pods --watch
 
 ---
 
+## Step 4 — Add Gitea to ArgoCD known hosts
+
+The known hosts configmap is created by Helm — patch it after install:
+
+```bash
+ssh-keyscan -t ed25519 gitea.mk-labs.cloud 2>/dev/null | \
+  kubectl patch configmap argocd-ssh-known-hosts-cm -n argocd \
+  --type merge \
+  -p "{\"data\":{\"ssh_known_hosts\": \"$(ssh-keyscan -t ed25519 gitea.mk-labs.cloud 2>/dev/null)\"}}"
+```
+
+---
+
 ## Step 5 — Get the initial admin password
 
 ```bash
@@ -90,7 +90,7 @@ kubectl -n argocd get secret argocd-initial-admin-secret \
 Access the UI via port-forward (ingress-nginx isn't installed yet at this point):
 
 ```bash
-kubectl port-forward svc/argocd-server -n argocd 8080:80
+kubectl port-forward svc/argocd-server -n argocd 8080:80 > /tmp/pf.log 2>&1 &
 ```
 
 Browse to http://localhost:8080 — login with `admin` and the password above.
@@ -127,6 +127,49 @@ Run this periodically until no pending CSRs remain.
 
 ---
 
+## Step 8 — Bootstrap 1Password Connect secrets
+
+Once the `onepassword-connect` namespace exists (created by ArgoCD sync),
+create the two secrets the Connect server requires.
+
+**Important notes:**
+- The credentials file must be stored **base64-encoded** in the secret
+- The token file must have **no trailing newline** — use `echo -n` when creating it
+- Both secrets go in the `onepassword-connect` namespace, not `external-secrets`
+- Secret names must match exactly: `op-credentials` and `connect-token`
+
+```bash
+# Verify the namespace exists before proceeding
+kubectl get namespace onepassword-connect
+
+# Create credentials secret — base64-encode the file contents
+kubectl create secret generic op-credentials \
+  --namespace onepassword-connect \
+  --from-literal=1password-credentials.json=$(base64 -w 0 /home/wed/1password-credentials.json)
+
+# Create token secret — token file must have no trailing newline
+kubectl create secret generic connect-token \
+  --namespace onepassword-connect \
+  --from-file=token=/home/wed/connect-token
+```
+
+Restart the Connect deployment to pick up the secrets:
+
+```bash
+kubectl -n onepassword-connect rollout restart deployment onepassword-connect
+kubectl -n onepassword-connect rollout status deployment onepassword-connect
+```
+
+Verify Connect is healthy:
+
+```bash
+kubectl -n onepassword-connect logs deployment/onepassword-connect -c connect-api 2>&1 | tail -10
+```
+
+You should see only `GET /health` and `GET /heartbeat` 200 responses — no errors.
+
+---
+
 ## Expected platform sync order
 
 Apps sync in wave order via `argocd.argoproj.io/sync-wave` annotations:
@@ -156,7 +199,7 @@ Check that the `gitea-repo` secret exists and the known hosts configmap has
 kubectl -n argocd get application <name> -o yaml | grep -A10 "conditions"
 ```
 
-**self-reference loop (app syncing itself):**
+**Self-reference loop (app syncing itself):**
 Ensure `apps-of-apps.yaml` uses `include: "*/application.yaml"` with
 `recurse: true` — this prevents ArgoCD from picking up its own manifest.
 
@@ -168,6 +211,38 @@ prune on any Application that manages the `argocd` namespace.
 The correct field name is `valueFiles` (plural). `valuesFile` and `valuesFiles`
 are silently ignored — ArgoCD syncs green but your values aren't applied.
 
+**argocd-cm configmap missing:**
+If the UI shows `configmap "argocd-cm" not found`, recreate it:
+```bash
+kubectl create configmap argocd-cm \
+  --namespace argocd \
+  --from-literal=url=https://argocd.local.mk-labs.cloud
+kubectl -n argocd rollout restart deployment argocd-server
+```
+
+**1Password Connect base64 error:**
+`illegal base64 data at input byte 0` means the credentials secret was created
+with raw JSON instead of base64-encoded content. Delete and recreate:
+```bash
+kubectl delete secret op-credentials -n onepassword-connect
+kubectl create secret generic op-credentials \
+  --namespace onepassword-connect \
+  --from-literal=1password-credentials.json=$(base64 -w 0 /home/wed/1password-credentials.json)
+kubectl -n onepassword-connect rollout restart deployment onepassword-connect
+```
+
+**1Password Connect invalid Authorization header:**
+`invalid header field value for "Authorization"` means the token has a trailing
+newline. Recreate the token file and secret:
+```bash
+echo -n "your-token" > /home/wed/connect-token
+kubectl delete secret connect-token -n onepassword-connect
+kubectl create secret generic connect-token \
+  --namespace onepassword-connect \
+  --from-file=token=/home/wed/connect-token
+kubectl -n onepassword-connect rollout restart deployment onepassword-connect
+```
+
 ---
 
 ## Post-bootstrap

cluster/argocd/values.yaml

@@ -33,7 +33,7 @@ server:
     hostname: argocd.local.mk-labs.cloud
     tls: true
     annotations:
-      cert-manager.io/cluster-issuer: letsencrypt-production
+      cert-manager.io/cluster-issuer: letsencrypt-prod
       nginx.ingress.kubernetes.io/ssl-passthrough: "false"
       nginx.ingress.kubernetes.io/backend-protocol: "HTTP"
 
@@ -68,6 +68,17 @@ configs:
       hs.message = "IP pool ready"
       return hs
 
+    # Ignore ESO-injected default fields on ExternalSecret resources.
+    # ESO mutates these fields after sync causing false OutOfSync state.
+    resource.customizations.ignoreDifferences.external-secrets.io_ExternalSecret: |
+      jqPathExpressions:
+        - '.spec.data[]?.remoteRef?.conversionStrategy'
+        - '.spec.data[]?.remoteRef?.decodingStrategy'
+        - '.spec.data[]?.remoteRef?.metadataPolicy'
+        - '.spec.dataFrom[]?.extract?.conversionStrategy'
+        - '.spec.dataFrom[]?.extract?.decodingStrategy'
+        - '.spec.dataFrom[]?.extract?.metadataPolicy'
+  
   # RBAC — admin gets full access, SSO groups mapped after Authentik integration
   rbac:
     policy.default: role:readonly