From 6023ee25e1ee1e2587bcd3aff7447c3a84e953d4 Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Sat, 6 Jun 2026 17:43:22 -0500 Subject: [PATCH] feat(firecrawl): Day 3 - Complete Kubernetes manifests for Firecrawl deployment - Created PersistentVolumeClaim for PostgreSQL (10GB, nfs-emporium) - Created ConfigMaps for API and Playwright service configuration - Created 7 Deployment manifests: * firecrawl-api (2 CPU, 4-6GB RAM) * firecrawl-api-worker (1 CPU, 3-4GB RAM) * firecrawl-api-nuq-worker (1 CPU, 3-4GB RAM) * firecrawl-playwright (2 CPU, 4GB RAM, 1GB tmpfs) * nuq-postgres (1 CPU, 2GB RAM, 10GB PVC) * redis (0.5 CPU, 1GB RAM) * rabbitmq (0.5 CPU, 1GB RAM) - Created 5 ClusterIP Services for inter-service communication - Created HTTPRoute for external access via Gateway API * Primary hostname: spaceship-earth.local.mk-labs.cloud * Alias: firecrawl.local.mk-labs.cloud - All manifests validated with kubectl dry-run=client Next steps (Day 4): Configure ExternalSecrets for 1Password integration Next steps (Day 5): Deploy to cluster and verify functionality Total resources: 8 CPU, 22GB RAM, 10GB storage --- .../firecrawl/configmap-playwright.yaml | 13 +++ cluster/applications/firecrawl/configmap.yaml | 45 ++++++++++ .../firecrawl/deployment-api.yaml | 60 ++++++++++++++ .../firecrawl/deployment-nuq-worker.yaml | 62 ++++++++++++++ .../firecrawl/deployment-playwright.yaml | 63 ++++++++++++++ .../firecrawl/deployment-postgres.yaml | 83 +++++++++++++++++++ .../firecrawl/deployment-rabbitmq.yaml | 61 ++++++++++++++ .../firecrawl/deployment-redis.yaml | 56 +++++++++++++ .../firecrawl/deployment-worker.yaml | 62 ++++++++++++++ cluster/applications/firecrawl/httproute.yaml | 26 ++++++ cluster/applications/firecrawl/namespace.yaml | 13 +++ .../applications/firecrawl/postgres-pvc.yaml | 17 ++++ .../applications/firecrawl/service-api.yaml | 18 ++++ .../firecrawl/service-playwright.yaml | 18 ++++ .../firecrawl/service-postgres.yaml | 18 ++++ .../firecrawl/service-rabbitmq.yaml | 22 +++++ .../applications/firecrawl/service-redis.yaml | 18 ++++ 17 files changed, 655 insertions(+) create mode 100644 cluster/applications/firecrawl/configmap-playwright.yaml create mode 100644 cluster/applications/firecrawl/configmap.yaml create mode 100644 cluster/applications/firecrawl/deployment-api.yaml create mode 100644 cluster/applications/firecrawl/deployment-nuq-worker.yaml create mode 100644 cluster/applications/firecrawl/deployment-playwright.yaml create mode 100644 cluster/applications/firecrawl/deployment-postgres.yaml create mode 100644 cluster/applications/firecrawl/deployment-rabbitmq.yaml create mode 100644 cluster/applications/firecrawl/deployment-redis.yaml create mode 100644 cluster/applications/firecrawl/deployment-worker.yaml create mode 100644 cluster/applications/firecrawl/httproute.yaml create mode 100644 cluster/applications/firecrawl/namespace.yaml create mode 100644 cluster/applications/firecrawl/postgres-pvc.yaml create mode 100644 cluster/applications/firecrawl/service-api.yaml create mode 100644 cluster/applications/firecrawl/service-playwright.yaml create mode 100644 cluster/applications/firecrawl/service-postgres.yaml create mode 100644 cluster/applications/firecrawl/service-rabbitmq.yaml create mode 100644 cluster/applications/firecrawl/service-redis.yaml diff --git a/cluster/applications/firecrawl/configmap-playwright.yaml b/cluster/applications/firecrawl/configmap-playwright.yaml new file mode 100644 index 0000000..80203af --- /dev/null +++ b/cluster/applications/firecrawl/configmap-playwright.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: playwright-service-config + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: browser + app.kubernetes.io/part-of: firecrawl +data: + PORT: "3000" + ALLOW_LOCAL_WEBHOOKS: "false" + MAX_CONCURRENT_PAGES: "10" diff --git a/cluster/applications/firecrawl/configmap.yaml b/cluster/applications/firecrawl/configmap.yaml new file mode 100644 index 0000000..43253a8 --- /dev/null +++ b/cluster/applications/firecrawl/configmap.yaml @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: firecrawl-config + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: configuration + app.kubernetes.io/part-of: firecrawl +data: + # Service URLs (Kubernetes internal DNS) + REDIS_URL: "redis://redis:6379" + REDIS_RATE_LIMIT_URL: "redis://redis:6379" + POSTGRES_HOST: "nuq-postgres" + POSTGRES_PORT: "5432" + POSTGRES_USER: "postgres" + POSTGRES_DB: "postgres" + NUQ_RABBITMQ_URL: "amqp://rabbitmq:5672" + PLAYWRIGHT_MICROSERVICE_URL: "http://playwright-service:3000/scrape" + + # Server Configuration + HOST: "0.0.0.0" + PORT: "3002" + WORKER_PORT: "3005" + EXTRACT_WORKER_PORT: "3004" + USE_DB_AUTHENTICATION: "false" + ENV: "production" + + # Performance Tuning + NUM_WORKERS_PER_QUEUE: "8" + CRAWL_CONCURRENT_REQUESTS: "10" + MAX_CONCURRENT_JOBS: "5" + BROWSER_POOL_SIZE: "5" + MAX_CONCURRENT_PAGES: "10" + HARNESS_STARTUP_TIMEOUT_MS: "60000" + + # Resource Limits (Self-Protection) + MAX_CPU: "0.8" + MAX_RAM: "0.8" + + # Logging + LOGGING_LEVEL: "info" + + # Security + ALLOW_LOCAL_WEBHOOKS: "false" diff --git a/cluster/applications/firecrawl/deployment-api.yaml b/cluster/applications/firecrawl/deployment-api.yaml new file mode 100644 index 0000000..92c69f8 --- /dev/null +++ b/cluster/applications/firecrawl/deployment-api.yaml @@ -0,0 +1,60 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: firecrawl-api + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: api + app.kubernetes.io/part-of: firecrawl +spec: + replicas: 1 + selector: + matchLabels: + app: firecrawl-api + template: + metadata: + labels: + app: firecrawl-api + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: api + spec: + containers: + - name: api + image: the-seas.local.mk-labs.cloud/library/firecrawl-api:latest + imagePullPolicy: Always + ports: + - name: http + containerPort: 3002 + protocol: TCP + envFrom: + - configMapRef: + name: firecrawl-config + - secretRef: + name: firecrawl-secrets + env: + - name: FLY_PROCESS_GROUP + value: "app" + resources: + requests: + cpu: 1000m + memory: 4Gi + limits: + cpu: 2000m + memory: 6Gi + livenessProbe: + httpGet: + path: /v0/health/liveness + port: 3002 + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /v0/health/readiness + port: 3002 + initialDelaySeconds: 30 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 diff --git a/cluster/applications/firecrawl/deployment-nuq-worker.yaml b/cluster/applications/firecrawl/deployment-nuq-worker.yaml new file mode 100644 index 0000000..ab66529 --- /dev/null +++ b/cluster/applications/firecrawl/deployment-nuq-worker.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: firecrawl-api-nuq-worker + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: nuq-worker + app.kubernetes.io/part-of: firecrawl +spec: + replicas: 1 + selector: + matchLabels: + app: firecrawl-api-nuq-worker + template: + metadata: + labels: + app: firecrawl-api-nuq-worker + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: nuq-worker + spec: + containers: + - name: nuq-worker + image: the-seas.local.mk-labs.cloud/library/firecrawl-api:latest + imagePullPolicy: Always + command: ["node"] + args: ["dist/src/services/worker/nuq-worker.js"] + envFrom: + - configMapRef: + name: firecrawl-config + - secretRef: + name: firecrawl-secrets + env: + - name: FLY_PROCESS_GROUP + value: "nuq-worker" + resources: + requests: + cpu: 500m + memory: 3Gi + limits: + cpu: 1000m + memory: 4Gi + livenessProbe: + exec: + command: + - pgrep + - -f + - nuq-worker + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + exec: + command: + - pgrep + - -f + - nuq-worker + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 diff --git a/cluster/applications/firecrawl/deployment-playwright.yaml b/cluster/applications/firecrawl/deployment-playwright.yaml new file mode 100644 index 0000000..24212c5 --- /dev/null +++ b/cluster/applications/firecrawl/deployment-playwright.yaml @@ -0,0 +1,63 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: playwright-service + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: browser + app.kubernetes.io/part-of: firecrawl +spec: + replicas: 1 + selector: + matchLabels: + app: playwright-service + template: + metadata: + labels: + app: playwright-service + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: browser + spec: + containers: + - name: playwright + image: the-seas.local.mk-labs.cloud/library/firecrawl-playwright:latest + imagePullPolicy: Always + ports: + - name: http + containerPort: 3000 + protocol: TCP + envFrom: + - configMapRef: + name: playwright-service-config + resources: + requests: + cpu: 1000m + memory: 2Gi + limits: + cpu: 2000m + memory: 4Gi + volumeMounts: + - name: tmpfs + mountPath: /tmp + livenessProbe: + httpGet: + path: / + port: 3000 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: / + port: 3000 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + volumes: + - name: tmpfs + emptyDir: + medium: Memory + sizeLimit: 1Gi diff --git a/cluster/applications/firecrawl/deployment-postgres.yaml b/cluster/applications/firecrawl/deployment-postgres.yaml new file mode 100644 index 0000000..0e38b96 --- /dev/null +++ b/cluster/applications/firecrawl/deployment-postgres.yaml @@ -0,0 +1,83 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nuq-postgres + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: database + app.kubernetes.io/part-of: firecrawl +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: nuq-postgres + template: + metadata: + labels: + app: nuq-postgres + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: database + spec: + containers: + - name: postgres + image: the-seas.local.mk-labs.cloud/library/firecrawl-postgres:latest + imagePullPolicy: Always + ports: + - name: postgres + containerPort: 5432 + protocol: TCP + env: + - name: POSTGRES_USER + valueFrom: + configMapKeyRef: + name: firecrawl-config + key: POSTGRES_USER + - name: POSTGRES_DB + valueFrom: + configMapKeyRef: + name: firecrawl-config + key: POSTGRES_DB + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: firecrawl-secrets + key: POSTGRES_PASSWORD + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + volumeMounts: + - name: postgres-data + mountPath: /var/lib/postgresql/data + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi + livenessProbe: + exec: + command: + - pg_isready + - -U + - postgres + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + exec: + command: + - pg_isready + - -U + - postgres + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + volumes: + - name: postgres-data + persistentVolumeClaim: + claimName: postgres-data diff --git a/cluster/applications/firecrawl/deployment-rabbitmq.yaml b/cluster/applications/firecrawl/deployment-rabbitmq.yaml new file mode 100644 index 0000000..1f633b4 --- /dev/null +++ b/cluster/applications/firecrawl/deployment-rabbitmq.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rabbitmq + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: messaging + app.kubernetes.io/part-of: firecrawl +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: rabbitmq + template: + metadata: + labels: + app: rabbitmq + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: messaging + spec: + containers: + - name: rabbitmq + image: rabbitmq:3-management + imagePullPolicy: IfNotPresent + ports: + - name: amqp + containerPort: 5672 + protocol: TCP + - name: management + containerPort: 15672 + protocol: TCP + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 500m + memory: 1Gi + livenessProbe: + exec: + command: + - rabbitmq-diagnostics + - -q + - ping + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 3 + readinessProbe: + exec: + command: + - rabbitmq-diagnostics + - -q + - check_running + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 diff --git a/cluster/applications/firecrawl/deployment-redis.yaml b/cluster/applications/firecrawl/deployment-redis.yaml new file mode 100644 index 0000000..da7247b --- /dev/null +++ b/cluster/applications/firecrawl/deployment-redis.yaml @@ -0,0 +1,56 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: cache + app.kubernetes.io/part-of: firecrawl +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: cache + spec: + containers: + - name: redis + image: redis:alpine + imagePullPolicy: IfNotPresent + ports: + - name: redis + containerPort: 6379 + protocol: TCP + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 500m + memory: 1Gi + livenessProbe: + exec: + command: + - redis-cli + - ping + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + exec: + command: + - redis-cli + - ping + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 diff --git a/cluster/applications/firecrawl/deployment-worker.yaml b/cluster/applications/firecrawl/deployment-worker.yaml new file mode 100644 index 0000000..b409a74 --- /dev/null +++ b/cluster/applications/firecrawl/deployment-worker.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: firecrawl-api-worker + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: worker + app.kubernetes.io/part-of: firecrawl +spec: + replicas: 1 + selector: + matchLabels: + app: firecrawl-api-worker + template: + metadata: + labels: + app: firecrawl-api-worker + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: worker + spec: + containers: + - name: worker + image: the-seas.local.mk-labs.cloud/library/firecrawl-api:latest + imagePullPolicy: Always + command: ["node"] + args: ["dist/src/services/queue-worker.js"] + ports: + - name: http + containerPort: 3005 + protocol: TCP + envFrom: + - configMapRef: + name: firecrawl-config + - secretRef: + name: firecrawl-secrets + env: + - name: FLY_PROCESS_GROUP + value: "worker" + resources: + requests: + cpu: 500m + memory: 3Gi + limits: + cpu: 1000m + memory: 4Gi + livenessProbe: + httpGet: + path: / + port: 3005 + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: / + port: 3005 + initialDelaySeconds: 30 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 diff --git a/cluster/applications/firecrawl/httproute.yaml b/cluster/applications/firecrawl/httproute.yaml new file mode 100644 index 0000000..22c7cf0 --- /dev/null +++ b/cluster/applications/firecrawl/httproute.yaml @@ -0,0 +1,26 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: firecrawl + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: ingress + app.kubernetes.io/part-of: firecrawl + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod +spec: + parentRefs: + - name: gateway + namespace: default + hostnames: + - spaceship-earth.local.mk-labs.cloud + - firecrawl.local.mk-labs.cloud + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: firecrawl-api + port: 3002 diff --git a/cluster/applications/firecrawl/namespace.yaml b/cluster/applications/firecrawl/namespace.yaml new file mode 100644 index 0000000..e5fea4f --- /dev/null +++ b/cluster/applications/firecrawl/namespace.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: firecrawl + labels: + name: firecrawl + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: application + app.kubernetes.io/part-of: platform-buildout + epcot-theme: spaceship-earth + annotations: + description: "Firecrawl web scraping and crawling service - enables JARVIS web search capability" diff --git a/cluster/applications/firecrawl/postgres-pvc.yaml b/cluster/applications/firecrawl/postgres-pvc.yaml new file mode 100644 index 0000000..0230e1c --- /dev/null +++ b/cluster/applications/firecrawl/postgres-pvc.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgres-data + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: database + app.kubernetes.io/part-of: firecrawl +spec: + accessModes: + - ReadWriteOnce + storageClassName: nfs-emporium + resources: + requests: + storage: 10Gi diff --git a/cluster/applications/firecrawl/service-api.yaml b/cluster/applications/firecrawl/service-api.yaml new file mode 100644 index 0000000..c3d12bf --- /dev/null +++ b/cluster/applications/firecrawl/service-api.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: firecrawl-api + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: api + app.kubernetes.io/part-of: firecrawl +spec: + type: ClusterIP + selector: + app: firecrawl-api + ports: + - name: http + port: 3002 + targetPort: 3002 + protocol: TCP diff --git a/cluster/applications/firecrawl/service-playwright.yaml b/cluster/applications/firecrawl/service-playwright.yaml new file mode 100644 index 0000000..c539bb8 --- /dev/null +++ b/cluster/applications/firecrawl/service-playwright.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: playwright-service + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: browser + app.kubernetes.io/part-of: firecrawl +spec: + type: ClusterIP + selector: + app: playwright-service + ports: + - name: http + port: 3000 + targetPort: 3000 + protocol: TCP diff --git a/cluster/applications/firecrawl/service-postgres.yaml b/cluster/applications/firecrawl/service-postgres.yaml new file mode 100644 index 0000000..e21715c --- /dev/null +++ b/cluster/applications/firecrawl/service-postgres.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: nuq-postgres + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: database + app.kubernetes.io/part-of: firecrawl +spec: + type: ClusterIP + selector: + app: nuq-postgres + ports: + - name: postgres + port: 5432 + targetPort: 5432 + protocol: TCP diff --git a/cluster/applications/firecrawl/service-rabbitmq.yaml b/cluster/applications/firecrawl/service-rabbitmq.yaml new file mode 100644 index 0000000..1dd5e6c --- /dev/null +++ b/cluster/applications/firecrawl/service-rabbitmq.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + name: rabbitmq + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: messaging + app.kubernetes.io/part-of: firecrawl +spec: + type: ClusterIP + selector: + app: rabbitmq + ports: + - name: amqp + port: 5672 + targetPort: 5672 + protocol: TCP + - name: management + port: 15672 + targetPort: 15672 + protocol: TCP diff --git a/cluster/applications/firecrawl/service-redis.yaml b/cluster/applications/firecrawl/service-redis.yaml new file mode 100644 index 0000000..86d23b9 --- /dev/null +++ b/cluster/applications/firecrawl/service-redis.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: cache + app.kubernetes.io/part-of: firecrawl +spec: + type: ClusterIP + selector: + app: redis + ports: + - name: redis + port: 6379 + targetPort: 6379 + protocol: TCP