From f4181349f8afea51c578f8eadb7e8dc7f03e1662 Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Thu, 4 Jun 2026 16:40:12 -0500 Subject: [PATCH] feat: Deploy Firecrawl (spaceship-earth) to fastpass cluster - Add Firecrawl application with full stack: - Firecrawl API (main service) - Firecrawl Worker (background jobs) - Playwright Service (browser automation) - Redis (cache & job queue) - PostgreSQL (state management) - RabbitMQ (message queue) - Configure dual DNS names: - Primary: spaceship-earth.local.mk-labs.cloud (EPCOT theme) - Secondary: firecrawl.local.mk-labs.cloud - Add Gateway API HTTPRoutes with TLS certificates - Update ReferenceGrant for firecrawl namespace - Configure ArgoCD application (wave 20) - Set USE_DB_AUTHENTICATION=false for internal deployment This provides JARVIS with web scraping and search capabilities. --- cluster/applications/firecrawl/README.md | 125 ++++++++++++++++++ .../firecrawl/api-deployment.yaml | 67 ++++++++++ .../applications/firecrawl/api-service.yaml | 19 +++ .../applications/firecrawl/application.yaml | 46 +++++++ cluster/applications/firecrawl/configmap.yaml | 60 +++++++++ cluster/applications/firecrawl/httproute.yaml | 87 ++++++++++++ cluster/applications/firecrawl/namespace.yaml | 12 ++ .../firecrawl/playwright-configmap.yaml | 13 ++ .../firecrawl/playwright-deployment.yaml | 44 ++++++ .../firecrawl/playwright-service.yaml | 17 +++ .../firecrawl/postgres-deployment.yaml | 47 +++++++ .../firecrawl/postgres-service.yaml | 17 +++ .../firecrawl/rabbitmq-deployment.yaml | 51 +++++++ .../firecrawl/rabbitmq-service.yaml | 21 +++ .../firecrawl/redis-deployment.yaml | 35 +++++ .../applications/firecrawl/redis-service.yaml | 17 +++ .../firecrawl/worker-deployment.yaml | 44 ++++++ cluster/platform/gateway/referencegrant.yaml | 3 + 18 files changed, 725 insertions(+) create mode 100644 cluster/applications/firecrawl/README.md create mode 100644 cluster/applications/firecrawl/api-deployment.yaml create mode 100644 cluster/applications/firecrawl/api-service.yaml create mode 100644 cluster/applications/firecrawl/application.yaml create mode 100644 cluster/applications/firecrawl/configmap.yaml create mode 100644 cluster/applications/firecrawl/httproute.yaml create mode 100644 cluster/applications/firecrawl/namespace.yaml create mode 100644 cluster/applications/firecrawl/playwright-configmap.yaml create mode 100644 cluster/applications/firecrawl/playwright-deployment.yaml create mode 100644 cluster/applications/firecrawl/playwright-service.yaml create mode 100644 cluster/applications/firecrawl/postgres-deployment.yaml create mode 100644 cluster/applications/firecrawl/postgres-service.yaml create mode 100644 cluster/applications/firecrawl/rabbitmq-deployment.yaml create mode 100644 cluster/applications/firecrawl/rabbitmq-service.yaml create mode 100644 cluster/applications/firecrawl/redis-deployment.yaml create mode 100644 cluster/applications/firecrawl/redis-service.yaml create mode 100644 cluster/applications/firecrawl/worker-deployment.yaml diff --git a/cluster/applications/firecrawl/README.md b/cluster/applications/firecrawl/README.md new file mode 100644 index 0000000..5d78fd1 --- /dev/null +++ b/cluster/applications/firecrawl/README.md @@ -0,0 +1,125 @@ +# Firecrawl (Spaceship Earth) + +Web scraping and search service for JARVIS's web search capabilities. + +## Service Names + +- **Primary (EPCOT):** `spaceship-earth.local.mk-labs.cloud` +- **Secondary:** `firecrawl.local.mk-labs.cloud` + +## Architecture + +### Components + +1. **Firecrawl API** - Main REST API service for web scraping + - Port: 3002 + - Health checks: `/v0/health/liveness`, `/v0/health/readiness` + - Resources: 4Gi memory, 2 CPU cores (6Gi limit) + +2. **Firecrawl Worker** - Background job processor + - Processes crawl jobs from queue + - Resources: 3Gi memory, 1 CPU core (4Gi limit) + +3. **Playwright Service** - Browser automation + - Port: 3000 + - Headless Chrome/Firefox for JavaScript rendering + - Resources: 2Gi memory, 1 CPU core (4Gi limit) + +4. **Redis** - Cache and job queue + - Port: 6379 + - Resources: 256Mi memory, 100m CPU (512Mi limit) + +5. **PostgreSQL (nuq-postgres)** - State management + - Port: 5432 + - Custom image with queue extensions + - Resources: 512Mi memory, 250m CPU (1Gi limit) + - **Note:** Currently using emptyDir (ephemeral). Add PVC for persistence if needed. + +6. **RabbitMQ** - Message queue for distributed processing + - AMQP Port: 5672 + - Management UI: 15672 + - Resources: 512Mi memory, 250m CPU (1Gi limit) + +### Configuration + +Key environment variables in `configmap.yaml`: + +- `USE_DB_AUTHENTICATION=false` - Simplified auth for internal use +- `NUM_WORKERS_PER_QUEUE=8` - Worker concurrency +- `CRAWL_CONCURRENT_REQUESTS=10` - Parallel crawl requests +- `MAX_CONCURRENT_JOBS=5` - Maximum simultaneous jobs + +## Access + +- **HTTPS (Gateway API):** https://spaceship-earth.local.mk-labs.cloud +- **HTTPS (Gateway API):** https://firecrawl.local.mk-labs.cloud +- Internal: `http://firecrawl-api.firecrawl.svc.cluster.local:3002` + +## DNS + +ExternalDNS automatically configures DNS records pointing to the Gateway load balancer (10.1.71.90). + +## TLS + +Certificates are automatically provisioned via cert-manager using the `letsencrypt-prod` ClusterIssuer. + +## Deployment + +Managed by ArgoCD (wave 20): + +```bash +# Check application status +kubectl -n argocd get application firecrawl + +# View pods +kubectl -n firecrawl get pods + +# Check API logs +kubectl -n firecrawl logs -l app=firecrawl-api -f + +# Check worker logs +kubectl -n firecrawl logs -l app=firecrawl-worker -f +``` + +## API Usage + +Example: Scrape a webpage + +```bash +curl -X POST https://spaceship-earth.local.mk-labs.cloud/v0/scrape \ + -H "Content-Type: application/json" \ + -d '{"url": "https://example.com"}' +``` + +## Troubleshooting + +### Pods not starting + +Check resource availability: +```bash +kubectl -n firecrawl describe pod +``` + +### PostgreSQL data persistence + +Currently using `emptyDir` for simplicity. To add persistence: + +1. Create a PVC in `postgres-pvc.yaml` +2. Update `postgres-deployment.yaml` to use the PVC +3. Commit and push + +### Gateway not routing + +Verify ReferenceGrant allows firecrawl namespace: +```bash +kubectl -n gateway get referencegrant allow-httproutes -o yaml +``` + +## Future Enhancements + +- [ ] Add persistent volume for PostgreSQL +- [ ] Add extract worker deployment (data extraction) +- [ ] Add nuq worker deployment (advanced queue processing) +- [ ] Configure monitoring/metrics +- [ ] Add resource quotas and limits +- [ ] Integrate with JARVIS via API key authentication diff --git a/cluster/applications/firecrawl/api-deployment.yaml b/cluster/applications/firecrawl/api-deployment.yaml new file mode 100644 index 0000000..3f65ee0 --- /dev/null +++ b/cluster/applications/firecrawl/api-deployment.yaml @@ -0,0 +1,67 @@ +# ------------------------------------------------------------------------------ +# Deployment: Firecrawl API +# Main API service for web scraping and crawling +# ------------------------------------------------------------------------------ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: firecrawl-api + namespace: firecrawl + labels: + app: firecrawl-api + epcot.theme/service: spaceship-earth +spec: + replicas: 1 + selector: + matchLabels: + app: firecrawl-api + template: + metadata: + labels: + app: firecrawl-api + epcot.theme/service: spaceship-earth + spec: + terminationGracePeriodSeconds: 180 + containers: + - name: api + image: ghcr.io/mendableai/firecrawl:latest + command: ["node"] + args: ["--max-old-space-size=6144", "dist/src/index.js"] + envFrom: + - configMapRef: + name: firecrawl-config + env: + - name: FLY_PROCESS_GROUP + value: "app" + - name: NUQ_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + ports: + - containerPort: 3002 + name: http + resources: + requests: + memory: "4Gi" + cpu: "2000m" + limits: + memory: "6Gi" + cpu: "2000m" + livenessProbe: + httpGet: + path: /v0/health/liveness + port: 3002 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /v0/health/readiness + port: 3002 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 diff --git a/cluster/applications/firecrawl/api-service.yaml b/cluster/applications/firecrawl/api-service.yaml new file mode 100644 index 0000000..8cb4a86 --- /dev/null +++ b/cluster/applications/firecrawl/api-service.yaml @@ -0,0 +1,19 @@ +# ------------------------------------------------------------------------------ +# Service: Firecrawl API +# ------------------------------------------------------------------------------ +apiVersion: v1 +kind: Service +metadata: + name: firecrawl-api + namespace: firecrawl + labels: + epcot.theme/service: spaceship-earth +spec: + type: ClusterIP + selector: + app: firecrawl-api + ports: + - protocol: TCP + port: 3002 + targetPort: 3002 + name: http diff --git a/cluster/applications/firecrawl/application.yaml b/cluster/applications/firecrawl/application.yaml new file mode 100644 index 0000000..de7a810 --- /dev/null +++ b/cluster/applications/firecrawl/application.yaml @@ -0,0 +1,46 @@ +# ------------------------------------------------------------------------------ +# Application: firecrawl (spaceship-earth) +# Wave 20 — applications tier +# Web scraping and search service for JARVIS +# +# Primary DNS: spaceship-earth.local.mk-labs.cloud (EPCOT theme) +# Secondary DNS: firecrawl.local.mk-labs.cloud +# +# Components: +# - Firecrawl API (main service) +# - Firecrawl Worker (background job processor) +# - Playwright Service (browser automation) +# - Redis (cache and job queue) +# - PostgreSQL (state management) +# - RabbitMQ (message queue) +# ------------------------------------------------------------------------------ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: firecrawl + namespace: argocd + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/part-of: mk-labs + epcot.theme/name: spaceship-earth + annotations: + argocd.argoproj.io/sync-wave: "20" # Wave 20 — applications tier +spec: + project: default + source: + repoURL: https://gitea.mk-labs.cloud/rblundon/homelab.git + targetRevision: main + path: cluster/applications/firecrawl + directory: + recurse: false + exclude: application.yaml # prevent self-reference loop + destination: + server: https://kubernetes.default.svc + namespace: firecrawl + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/cluster/applications/firecrawl/configmap.yaml b/cluster/applications/firecrawl/configmap.yaml new file mode 100644 index 0000000..a969d8d --- /dev/null +++ b/cluster/applications/firecrawl/configmap.yaml @@ -0,0 +1,60 @@ +# ------------------------------------------------------------------------------ +# ConfigMap: firecrawl-config +# Environment configuration for Firecrawl services +# ------------------------------------------------------------------------------ +apiVersion: v1 +kind: ConfigMap +metadata: + name: firecrawl-config + namespace: firecrawl +data: + # Service configuration + PORT: "3002" + HOST: "0.0.0.0" + WORKER_PORT: "3005" + EXTRACT_WORKER_PORT: "3004" + NUQ_WORKER_PORT: "3006" + NUQ_PREFETCH_WORKER_PORT: "3011" + + # Redis URLs + REDIS_URL: "redis://firecrawl-redis:6379" + REDIS_RATE_LIMIT_URL: "redis://firecrawl-redis:6379" + + # PostgreSQL configuration + POSTGRES_HOST: "firecrawl-postgres" + POSTGRES_PORT: "5432" + POSTGRES_USER: "postgres" + POSTGRES_PASSWORD: "postgres" + POSTGRES_DB: "postgres" + + # Database URLs (constructed from above) + NUQ_DATABASE_URL: "postgresql://postgres:postgres@firecrawl-postgres:5432/postgres" + NUQ_DATABASE_URL_LISTEN: "postgresql://postgres:postgres@firecrawl-postgres:5432/postgres" + + # RabbitMQ + NUQ_RABBITMQ_URL: "amqp://firecrawl-rabbitmq:5672" + + # Playwright service + PLAYWRIGHT_MICROSERVICE_URL: "http://firecrawl-playwright:3000/scrape" + + # Worker configuration + NUM_WORKERS_PER_QUEUE: "8" + NUQ_WORKER_COUNT: "5" + + # Performance tuning + CRAWL_CONCURRENT_REQUESTS: "10" + MAX_CONCURRENT_JOBS: "5" + BROWSER_POOL_SIZE: "5" + + # Authentication (disabled for internal use) + USE_DB_AUTHENTICATION: "false" + + # Environment + IS_KUBERNETES: "true" + ENV: "production" + LOGGING_LEVEL: "INFO" + + # Application URLs (internal service mesh) + FIRECRAWL_APP_SCHEME: "http" + FIRECRAWL_APP_HOST: "" + FIRECRAWL_APP_PORT: "" diff --git a/cluster/applications/firecrawl/httproute.yaml b/cluster/applications/firecrawl/httproute.yaml new file mode 100644 index 0000000..a0e80fc --- /dev/null +++ b/cluster/applications/firecrawl/httproute.yaml @@ -0,0 +1,87 @@ +# ------------------------------------------------------------------------------ +# HTTPRoute — Firecrawl via Cilium Gateway +# Both primary (spaceship-earth) and secondary (firecrawl) DNS names +# ------------------------------------------------------------------------------ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: spaceship-earth-tls + namespace: firecrawl +spec: + secretName: spaceship-earth-tls + issuerRef: + name: letsencrypt-prod + kind: ClusterIssuer + dnsNames: + - spaceship-earth.local.mk-labs.cloud +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: firecrawl-tls + namespace: firecrawl +spec: + secretName: firecrawl-tls + issuerRef: + name: letsencrypt-prod + kind: ClusterIssuer + dnsNames: + - firecrawl.local.mk-labs.cloud +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: spaceship-earth + namespace: firecrawl + annotations: + external-dns.alpha.kubernetes.io/hostname: spaceship-earth.local.mk-labs.cloud + external-dns.alpha.kubernetes.io/target: "10.1.71.90" +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: fastpass-gateway + namespace: gateway + sectionName: https + hostnames: + - spaceship-earth.local.mk-labs.cloud + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - group: "" + kind: Service + name: firecrawl-api + port: 3002 + weight: 1 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: firecrawl + namespace: firecrawl + annotations: + external-dns.alpha.kubernetes.io/hostname: firecrawl.local.mk-labs.cloud + external-dns.alpha.kubernetes.io/target: "10.1.71.90" +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: fastpass-gateway + namespace: gateway + sectionName: https + hostnames: + - firecrawl.local.mk-labs.cloud + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - group: "" + kind: Service + name: firecrawl-api + port: 3002 + weight: 1 diff --git a/cluster/applications/firecrawl/namespace.yaml b/cluster/applications/firecrawl/namespace.yaml new file mode 100644 index 0000000..1fd3b73 --- /dev/null +++ b/cluster/applications/firecrawl/namespace.yaml @@ -0,0 +1,12 @@ +# ------------------------------------------------------------------------------ +# Namespace: firecrawl +# Web scraping and search service for JARVIS +# ------------------------------------------------------------------------------ +apiVersion: v1 +kind: Namespace +metadata: + name: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/part-of: mk-labs + epcot.theme/name: spaceship-earth diff --git a/cluster/applications/firecrawl/playwright-configmap.yaml b/cluster/applications/firecrawl/playwright-configmap.yaml new file mode 100644 index 0000000..6419455 --- /dev/null +++ b/cluster/applications/firecrawl/playwright-configmap.yaml @@ -0,0 +1,13 @@ +# ------------------------------------------------------------------------------ +# ConfigMap: Playwright configuration +# ------------------------------------------------------------------------------ +apiVersion: v1 +kind: ConfigMap +metadata: + name: firecrawl-playwright-config + namespace: firecrawl +data: + PORT: "3000" + MAX_CONCURRENT_PAGES: "10" + ALLOW_LOCAL_WEBHOOKS: "" + BLOCK_MEDIA: "" diff --git a/cluster/applications/firecrawl/playwright-deployment.yaml b/cluster/applications/firecrawl/playwright-deployment.yaml new file mode 100644 index 0000000..5552d8c --- /dev/null +++ b/cluster/applications/firecrawl/playwright-deployment.yaml @@ -0,0 +1,44 @@ +# ------------------------------------------------------------------------------ +# Deployment: Playwright Service +# Browser automation service for web scraping +# ------------------------------------------------------------------------------ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: firecrawl-playwright + namespace: firecrawl + labels: + app: firecrawl-playwright +spec: + replicas: 1 + selector: + matchLabels: + app: firecrawl-playwright + template: + metadata: + labels: + app: firecrawl-playwright + spec: + containers: + - name: playwright + image: ghcr.io/mendableai/firecrawl/playwright-service:latest + envFrom: + - configMapRef: + name: firecrawl-playwright-config + ports: + - containerPort: 3000 + name: http + resources: + requests: + memory: "2Gi" + cpu: "1000m" + limits: + memory: "4Gi" + cpu: "2000m" + volumeMounts: + - name: cache + mountPath: /tmp/.cache + volumes: + - name: cache + emptyDir: + sizeLimit: 1Gi diff --git a/cluster/applications/firecrawl/playwright-service.yaml b/cluster/applications/firecrawl/playwright-service.yaml new file mode 100644 index 0000000..7eec515 --- /dev/null +++ b/cluster/applications/firecrawl/playwright-service.yaml @@ -0,0 +1,17 @@ +# ------------------------------------------------------------------------------ +# Service: Playwright +# ------------------------------------------------------------------------------ +apiVersion: v1 +kind: Service +metadata: + name: firecrawl-playwright + namespace: firecrawl +spec: + type: ClusterIP + selector: + app: firecrawl-playwright + ports: + - protocol: TCP + port: 3000 + targetPort: 3000 + name: http diff --git a/cluster/applications/firecrawl/postgres-deployment.yaml b/cluster/applications/firecrawl/postgres-deployment.yaml new file mode 100644 index 0000000..b589960 --- /dev/null +++ b/cluster/applications/firecrawl/postgres-deployment.yaml @@ -0,0 +1,47 @@ +# ------------------------------------------------------------------------------ +# Deployment: PostgreSQL (nuq-postgres) +# Database for Firecrawl job queue and state management +# ------------------------------------------------------------------------------ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: firecrawl-postgres + namespace: firecrawl + labels: + app: firecrawl-postgres +spec: + replicas: 1 + selector: + matchLabels: + app: firecrawl-postgres + template: + metadata: + labels: + app: firecrawl-postgres + spec: + containers: + - name: postgres + image: ghcr.io/mendableai/firecrawl/nuq-postgres:latest + env: + - name: POSTGRES_USER + value: "postgres" + - name: POSTGRES_PASSWORD + value: "postgres" + - name: POSTGRES_DB + value: "postgres" + ports: + - containerPort: 5432 + name: postgres + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "1Gi" + cpu: "500m" + volumeMounts: + - name: postgres-data + mountPath: /var/lib/postgresql/data + volumes: + - name: postgres-data + emptyDir: {} # For now, ephemeral storage. TODO: Add PVC for persistence diff --git a/cluster/applications/firecrawl/postgres-service.yaml b/cluster/applications/firecrawl/postgres-service.yaml new file mode 100644 index 0000000..4020782 --- /dev/null +++ b/cluster/applications/firecrawl/postgres-service.yaml @@ -0,0 +1,17 @@ +# ------------------------------------------------------------------------------ +# Service: PostgreSQL +# ------------------------------------------------------------------------------ +apiVersion: v1 +kind: Service +metadata: + name: firecrawl-postgres + namespace: firecrawl +spec: + type: ClusterIP + selector: + app: firecrawl-postgres + ports: + - protocol: TCP + port: 5432 + targetPort: 5432 + name: postgres diff --git a/cluster/applications/firecrawl/rabbitmq-deployment.yaml b/cluster/applications/firecrawl/rabbitmq-deployment.yaml new file mode 100644 index 0000000..aa260b4 --- /dev/null +++ b/cluster/applications/firecrawl/rabbitmq-deployment.yaml @@ -0,0 +1,51 @@ +# ------------------------------------------------------------------------------ +# Deployment: RabbitMQ +# Message queue for Firecrawl distributed task processing +# ------------------------------------------------------------------------------ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: firecrawl-rabbitmq + namespace: firecrawl + labels: + app: firecrawl-rabbitmq +spec: + replicas: 1 + selector: + matchLabels: + app: firecrawl-rabbitmq + template: + metadata: + labels: + app: firecrawl-rabbitmq + spec: + containers: + - name: rabbitmq + image: rabbitmq:3-management + command: ["rabbitmq-server"] + ports: + - containerPort: 5672 + name: amqp + - containerPort: 15672 + name: management + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "1Gi" + cpu: "500m" + livenessProbe: + exec: + command: ["rabbitmq-diagnostics", "-q", "check_running"] + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + exec: + command: ["rabbitmq-diagnostics", "-q", "check_running"] + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 3 diff --git a/cluster/applications/firecrawl/rabbitmq-service.yaml b/cluster/applications/firecrawl/rabbitmq-service.yaml new file mode 100644 index 0000000..4f637a7 --- /dev/null +++ b/cluster/applications/firecrawl/rabbitmq-service.yaml @@ -0,0 +1,21 @@ +# ------------------------------------------------------------------------------ +# Service: RabbitMQ +# ------------------------------------------------------------------------------ +apiVersion: v1 +kind: Service +metadata: + name: firecrawl-rabbitmq + namespace: firecrawl +spec: + type: ClusterIP + selector: + app: firecrawl-rabbitmq + ports: + - protocol: TCP + port: 5672 + targetPort: 5672 + name: amqp + - protocol: TCP + port: 15672 + targetPort: 15672 + name: management diff --git a/cluster/applications/firecrawl/redis-deployment.yaml b/cluster/applications/firecrawl/redis-deployment.yaml new file mode 100644 index 0000000..b5d84a3 --- /dev/null +++ b/cluster/applications/firecrawl/redis-deployment.yaml @@ -0,0 +1,35 @@ +# ------------------------------------------------------------------------------ +# Deployment: Redis +# In-memory cache and job queue for Firecrawl +# ------------------------------------------------------------------------------ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: firecrawl-redis + namespace: firecrawl + labels: + app: firecrawl-redis +spec: + replicas: 1 + selector: + matchLabels: + app: firecrawl-redis + template: + metadata: + labels: + app: firecrawl-redis + spec: + containers: + - name: redis + image: redis:alpine + command: ["redis-server", "--bind", "0.0.0.0"] + ports: + - containerPort: 6379 + name: redis + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" diff --git a/cluster/applications/firecrawl/redis-service.yaml b/cluster/applications/firecrawl/redis-service.yaml new file mode 100644 index 0000000..74c7cdc --- /dev/null +++ b/cluster/applications/firecrawl/redis-service.yaml @@ -0,0 +1,17 @@ +# ------------------------------------------------------------------------------ +# Service: Redis +# ------------------------------------------------------------------------------ +apiVersion: v1 +kind: Service +metadata: + name: firecrawl-redis + namespace: firecrawl +spec: + type: ClusterIP + selector: + app: firecrawl-redis + ports: + - protocol: TCP + port: 6379 + targetPort: 6379 + name: redis diff --git a/cluster/applications/firecrawl/worker-deployment.yaml b/cluster/applications/firecrawl/worker-deployment.yaml new file mode 100644 index 0000000..470ad6b --- /dev/null +++ b/cluster/applications/firecrawl/worker-deployment.yaml @@ -0,0 +1,44 @@ +# ------------------------------------------------------------------------------ +# Deployment: Firecrawl Worker +# Background worker for processing crawl jobs +# ------------------------------------------------------------------------------ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: firecrawl-worker + namespace: firecrawl + labels: + app: firecrawl-worker +spec: + replicas: 1 + selector: + matchLabels: + app: firecrawl-worker + template: + metadata: + labels: + app: firecrawl-worker + spec: + terminationGracePeriodSeconds: 180 + containers: + - name: worker + image: ghcr.io/mendableai/firecrawl:latest + command: ["node"] + args: ["--max-old-space-size=4096", "dist/src/services/queue-worker.js"] + envFrom: + - configMapRef: + name: firecrawl-config + env: + - name: FLY_PROCESS_GROUP + value: "worker" + - name: NUQ_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + resources: + requests: + memory: "3Gi" + cpu: "1000m" + limits: + memory: "4Gi" + cpu: "1000m" diff --git a/cluster/platform/gateway/referencegrant.yaml b/cluster/platform/gateway/referencegrant.yaml index 864601c..9472575 100644 --- a/cluster/platform/gateway/referencegrant.yaml +++ b/cluster/platform/gateway/referencegrant.yaml @@ -42,6 +42,9 @@ spec: - group: gateway.networking.k8s.io kind: HTTPRoute namespace: skyway + - group: gateway.networking.k8s.io + kind: HTTPRoute + namespace: firecrawl to: - group: gateway.networking.k8s.io kind: Gateway