refactor(firecrawl): Convert to production-ready Helm chart with persistent storage
CRITICAL FIXES: ✅ Add PersistentVolumeClaims for all stateful services - PostgreSQL: 20Gi PVC on nfs-emporium (data persistence) - Redis: 10Gi PVC on nfs-emporium (cache and queue) - RabbitMQ: 5Gi PVC on nfs-emporium (message queue) ✅ Pin all image versions (no more 'latest' tags) - Firecrawl API/Worker: v1.0.0 - Playwright Service: v1.0.0 - PostgreSQL (nuq-postgres): v1.0.0 - Redis: 7.4.1-alpine - RabbitMQ: 3.13.7-management-alpine ✅ Convert raw manifests to proper Helm chart - Template-based configuration - Centralized values.yaml - Proper Helm helpers and labels - Easy configuration management WHAT CHANGED: - Created chart/ directory with full Helm chart structure - Moved old manifests to old-manifests/ for reference - Updated ArgoCD Application to use Helm chart source - Added comprehensive README and MIGRATION docs - All services now use nfs-emporium storage class - Redis configured with AOF persistence - Proper resource limits and health checks - Gateway/HTTPRoute configs integrated into chart DEPLOYMENT: ArgoCD will automatically sync and apply changes. Old ephemeral data will be lost (fresh start with persistence). Resolves data loss issues and brings deployment to production standards.
This commit is contained in:
Hermes Agent service account
@@ -0,0 +1,67 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# Deployment: Firecrawl API
|
||||
# Main API service for web scraping and crawling
|
||||
# ------------------------------------------------------------------------------
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: firecrawl-api
|
||||
namespace: firecrawl
|
||||
labels:
|
||||
app: firecrawl-api
|
||||
epcot.theme/service: spaceship-earth
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: firecrawl-api
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: firecrawl-api
|
||||
epcot.theme/service: spaceship-earth
|
||||
spec:
|
||||
terminationGracePeriodSeconds: 180
|
||||
containers:
|
||||
- name: api
|
||||
image: ghcr.io/mendableai/firecrawl:latest
|
||||
command: ["node"]
|
||||
args: ["--max-old-space-size=6144", "dist/src/index.js"]
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: firecrawl-config
|
||||
env:
|
||||
- name: FLY_PROCESS_GROUP
|
||||
value: "app"
|
||||
- name: NUQ_POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
ports:
|
||||
- containerPort: 3002
|
||||
name: http
|
||||
resources:
|
||||
requests:
|
||||
memory: "4Gi"
|
||||
cpu: "2000m"
|
||||
limits:
|
||||
memory: "6Gi"
|
||||
cpu: "2000m"
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /v0/health/liveness
|
||||
port: 3002
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 5
|
||||
successThreshold: 1
|
||||
failureThreshold: 3
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /v0/health/readiness
|
||||
port: 3002
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 5
|
||||
successThreshold: 1
|
||||
failureThreshold: 3
|
||||
@@ -0,0 +1,19 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# Service: Firecrawl API
|
||||
# ------------------------------------------------------------------------------
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: firecrawl-api
|
||||
namespace: firecrawl
|
||||
labels:
|
||||
epcot.theme/service: spaceship-earth
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: firecrawl-api
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 3002
|
||||
targetPort: 3002
|
||||
name: http
|
||||
60
cluster/applications/firecrawl/old-manifests/configmap.yaml
Normal file
60
cluster/applications/firecrawl/old-manifests/configmap.yaml
Normal file
@@ -0,0 +1,60 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# ConfigMap: firecrawl-config
|
||||
# Environment configuration for Firecrawl services
|
||||
# ------------------------------------------------------------------------------
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: firecrawl-config
|
||||
namespace: firecrawl
|
||||
data:
|
||||
# Service configuration
|
||||
PORT: "3002"
|
||||
HOST: "0.0.0.0"
|
||||
WORKER_PORT: "3005"
|
||||
EXTRACT_WORKER_PORT: "3004"
|
||||
NUQ_WORKER_PORT: "3006"
|
||||
NUQ_PREFETCH_WORKER_PORT: "3011"
|
||||
|
||||
# Redis URLs
|
||||
REDIS_URL: "redis://firecrawl-redis:6379"
|
||||
REDIS_RATE_LIMIT_URL: "redis://firecrawl-redis:6379"
|
||||
|
||||
# PostgreSQL configuration
|
||||
POSTGRES_HOST: "firecrawl-postgres"
|
||||
POSTGRES_PORT: "5432"
|
||||
POSTGRES_USER: "postgres"
|
||||
POSTGRES_PASSWORD: "postgres"
|
||||
POSTGRES_DB: "postgres"
|
||||
|
||||
# Database URLs (constructed from above)
|
||||
NUQ_DATABASE_URL: "postgresql://postgres:postgres@firecrawl-postgres:5432/postgres"
|
||||
NUQ_DATABASE_URL_LISTEN: "postgresql://postgres:postgres@firecrawl-postgres:5432/postgres"
|
||||
|
||||
# RabbitMQ
|
||||
NUQ_RABBITMQ_URL: "amqp://firecrawl-rabbitmq:5672"
|
||||
|
||||
# Playwright service
|
||||
PLAYWRIGHT_MICROSERVICE_URL: "http://firecrawl-playwright:3000/scrape"
|
||||
|
||||
# Worker configuration
|
||||
NUM_WORKERS_PER_QUEUE: "8"
|
||||
NUQ_WORKER_COUNT: "5"
|
||||
|
||||
# Performance tuning
|
||||
CRAWL_CONCURRENT_REQUESTS: "10"
|
||||
MAX_CONCURRENT_JOBS: "5"
|
||||
BROWSER_POOL_SIZE: "5"
|
||||
|
||||
# Authentication (disabled for internal use)
|
||||
USE_DB_AUTHENTICATION: "false"
|
||||
|
||||
# Environment
|
||||
IS_KUBERNETES: "true"
|
||||
ENV: "production"
|
||||
LOGGING_LEVEL: "INFO"
|
||||
|
||||
# Application URLs (internal service mesh)
|
||||
FIRECRAWL_APP_SCHEME: "http"
|
||||
FIRECRAWL_APP_HOST: ""
|
||||
FIRECRAWL_APP_PORT: ""
|
||||
87
cluster/applications/firecrawl/old-manifests/httproute.yaml
Normal file
87
cluster/applications/firecrawl/old-manifests/httproute.yaml
Normal file
@@ -0,0 +1,87 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# HTTPRoute — Firecrawl via Cilium Gateway
|
||||
# Both primary (spaceship-earth) and secondary (firecrawl) DNS names
|
||||
# ------------------------------------------------------------------------------
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: spaceship-earth-tls
|
||||
namespace: firecrawl
|
||||
spec:
|
||||
secretName: spaceship-earth-tls
|
||||
issuerRef:
|
||||
name: letsencrypt-prod
|
||||
kind: ClusterIssuer
|
||||
dnsNames:
|
||||
- spaceship-earth.local.mk-labs.cloud
|
||||
---
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: firecrawl-tls
|
||||
namespace: firecrawl
|
||||
spec:
|
||||
secretName: firecrawl-tls
|
||||
issuerRef:
|
||||
name: letsencrypt-prod
|
||||
kind: ClusterIssuer
|
||||
dnsNames:
|
||||
- firecrawl.local.mk-labs.cloud
|
||||
---
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: HTTPRoute
|
||||
metadata:
|
||||
name: spaceship-earth
|
||||
namespace: firecrawl
|
||||
annotations:
|
||||
external-dns.alpha.kubernetes.io/hostname: spaceship-earth.local.mk-labs.cloud
|
||||
external-dns.alpha.kubernetes.io/target: "10.1.71.90"
|
||||
spec:
|
||||
parentRefs:
|
||||
- group: gateway.networking.k8s.io
|
||||
kind: Gateway
|
||||
name: fastpass-gateway
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- spaceship-earth.local.mk-labs.cloud
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
type: PathPrefix
|
||||
value: /
|
||||
backendRefs:
|
||||
- group: ""
|
||||
kind: Service
|
||||
name: firecrawl-api
|
||||
port: 3002
|
||||
weight: 1
|
||||
---
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: HTTPRoute
|
||||
metadata:
|
||||
name: firecrawl
|
||||
namespace: firecrawl
|
||||
annotations:
|
||||
external-dns.alpha.kubernetes.io/hostname: firecrawl.local.mk-labs.cloud
|
||||
external-dns.alpha.kubernetes.io/target: "10.1.71.90"
|
||||
spec:
|
||||
parentRefs:
|
||||
- group: gateway.networking.k8s.io
|
||||
kind: Gateway
|
||||
name: fastpass-gateway
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- firecrawl.local.mk-labs.cloud
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
type: PathPrefix
|
||||
value: /
|
||||
backendRefs:
|
||||
- group: ""
|
||||
kind: Service
|
||||
name: firecrawl-api
|
||||
port: 3002
|
||||
weight: 1
|
||||
12
cluster/applications/firecrawl/old-manifests/namespace.yaml
Normal file
12
cluster/applications/firecrawl/old-manifests/namespace.yaml
Normal file
@@ -0,0 +1,12 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# Namespace: firecrawl
|
||||
# Web scraping and search service for JARVIS
|
||||
# ------------------------------------------------------------------------------
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: firecrawl
|
||||
labels:
|
||||
app.kubernetes.io/name: firecrawl
|
||||
app.kubernetes.io/part-of: mk-labs
|
||||
epcot.theme/name: spaceship-earth
|
||||
@@ -0,0 +1,13 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# ConfigMap: Playwright configuration
|
||||
# ------------------------------------------------------------------------------
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: firecrawl-playwright-config
|
||||
namespace: firecrawl
|
||||
data:
|
||||
PORT: "3000"
|
||||
MAX_CONCURRENT_PAGES: "10"
|
||||
ALLOW_LOCAL_WEBHOOKS: ""
|
||||
BLOCK_MEDIA: ""
|
||||
@@ -0,0 +1,44 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# Deployment: Playwright Service
|
||||
# Browser automation service for web scraping
|
||||
# ------------------------------------------------------------------------------
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: firecrawl-playwright
|
||||
namespace: firecrawl
|
||||
labels:
|
||||
app: firecrawl-playwright
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: firecrawl-playwright
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: firecrawl-playwright
|
||||
spec:
|
||||
containers:
|
||||
- name: playwright
|
||||
image: ghcr.io/mendableai/firecrawl/playwright-service:latest
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: firecrawl-playwright-config
|
||||
ports:
|
||||
- containerPort: 3000
|
||||
name: http
|
||||
resources:
|
||||
requests:
|
||||
memory: "2Gi"
|
||||
cpu: "1000m"
|
||||
limits:
|
||||
memory: "4Gi"
|
||||
cpu: "2000m"
|
||||
volumeMounts:
|
||||
- name: cache
|
||||
mountPath: /tmp/.cache
|
||||
volumes:
|
||||
- name: cache
|
||||
emptyDir:
|
||||
sizeLimit: 1Gi
|
||||
@@ -0,0 +1,17 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# Service: Playwright
|
||||
# ------------------------------------------------------------------------------
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: firecrawl-playwright
|
||||
namespace: firecrawl
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: firecrawl-playwright
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 3000
|
||||
targetPort: 3000
|
||||
name: http
|
||||
@@ -0,0 +1,47 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# Deployment: PostgreSQL (nuq-postgres)
|
||||
# Database for Firecrawl job queue and state management
|
||||
# ------------------------------------------------------------------------------
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: firecrawl-postgres
|
||||
namespace: firecrawl
|
||||
labels:
|
||||
app: firecrawl-postgres
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: firecrawl-postgres
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: firecrawl-postgres
|
||||
spec:
|
||||
containers:
|
||||
- name: postgres
|
||||
image: ghcr.io/mendableai/firecrawl/nuq-postgres:latest
|
||||
env:
|
||||
- name: POSTGRES_USER
|
||||
value: "postgres"
|
||||
- name: POSTGRES_PASSWORD
|
||||
value: "postgres"
|
||||
- name: POSTGRES_DB
|
||||
value: "postgres"
|
||||
ports:
|
||||
- containerPort: 5432
|
||||
name: postgres
|
||||
resources:
|
||||
requests:
|
||||
memory: "512Mi"
|
||||
cpu: "250m"
|
||||
limits:
|
||||
memory: "1Gi"
|
||||
cpu: "500m"
|
||||
volumeMounts:
|
||||
- name: postgres-data
|
||||
mountPath: /var/lib/postgresql/data
|
||||
volumes:
|
||||
- name: postgres-data
|
||||
emptyDir: {} # For now, ephemeral storage. TODO: Add PVC for persistence
|
||||
@@ -0,0 +1,17 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# Service: PostgreSQL
|
||||
# ------------------------------------------------------------------------------
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: firecrawl-postgres
|
||||
namespace: firecrawl
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: firecrawl-postgres
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 5432
|
||||
targetPort: 5432
|
||||
name: postgres
|
||||
@@ -0,0 +1,51 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# Deployment: RabbitMQ
|
||||
# Message queue for Firecrawl distributed task processing
|
||||
# ------------------------------------------------------------------------------
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: firecrawl-rabbitmq
|
||||
namespace: firecrawl
|
||||
labels:
|
||||
app: firecrawl-rabbitmq
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: firecrawl-rabbitmq
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: firecrawl-rabbitmq
|
||||
spec:
|
||||
containers:
|
||||
- name: rabbitmq
|
||||
image: rabbitmq:3-management
|
||||
command: ["rabbitmq-server"]
|
||||
ports:
|
||||
- containerPort: 5672
|
||||
name: amqp
|
||||
- containerPort: 15672
|
||||
name: management
|
||||
resources:
|
||||
requests:
|
||||
memory: "512Mi"
|
||||
cpu: "250m"
|
||||
limits:
|
||||
memory: "1Gi"
|
||||
cpu: "500m"
|
||||
livenessProbe:
|
||||
exec:
|
||||
command: ["rabbitmq-diagnostics", "-q", "check_running"]
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 3
|
||||
readinessProbe:
|
||||
exec:
|
||||
command: ["rabbitmq-diagnostics", "-q", "check_running"]
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 3
|
||||
@@ -0,0 +1,21 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# Service: RabbitMQ
|
||||
# ------------------------------------------------------------------------------
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: firecrawl-rabbitmq
|
||||
namespace: firecrawl
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: firecrawl-rabbitmq
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 5672
|
||||
targetPort: 5672
|
||||
name: amqp
|
||||
- protocol: TCP
|
||||
port: 15672
|
||||
targetPort: 15672
|
||||
name: management
|
||||
@@ -0,0 +1,35 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# Deployment: Redis
|
||||
# In-memory cache and job queue for Firecrawl
|
||||
# ------------------------------------------------------------------------------
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: firecrawl-redis
|
||||
namespace: firecrawl
|
||||
labels:
|
||||
app: firecrawl-redis
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: firecrawl-redis
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: firecrawl-redis
|
||||
spec:
|
||||
containers:
|
||||
- name: redis
|
||||
image: redis:alpine
|
||||
command: ["redis-server", "--bind", "0.0.0.0"]
|
||||
ports:
|
||||
- containerPort: 6379
|
||||
name: redis
|
||||
resources:
|
||||
requests:
|
||||
memory: "256Mi"
|
||||
cpu: "100m"
|
||||
limits:
|
||||
memory: "512Mi"
|
||||
cpu: "500m"
|
||||
@@ -0,0 +1,17 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# Service: Redis
|
||||
# ------------------------------------------------------------------------------
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: firecrawl-redis
|
||||
namespace: firecrawl
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: firecrawl-redis
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 6379
|
||||
targetPort: 6379
|
||||
name: redis
|
||||
@@ -0,0 +1,44 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# Deployment: Firecrawl Worker
|
||||
# Background worker for processing crawl jobs
|
||||
# ------------------------------------------------------------------------------
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: firecrawl-worker
|
||||
namespace: firecrawl
|
||||
labels:
|
||||
app: firecrawl-worker
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: firecrawl-worker
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: firecrawl-worker
|
||||
spec:
|
||||
terminationGracePeriodSeconds: 180
|
||||
containers:
|
||||
- name: worker
|
||||
image: ghcr.io/mendableai/firecrawl:latest
|
||||
command: ["node"]
|
||||
args: ["--max-old-space-size=4096", "dist/src/services/queue-worker.js"]
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: firecrawl-config
|
||||
env:
|
||||
- name: FLY_PROCESS_GROUP
|
||||
value: "worker"
|
||||
- name: NUQ_POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
resources:
|
||||
requests:
|
||||
memory: "3Gi"
|
||||
cpu: "1000m"
|
||||
limits:
|
||||
memory: "4Gi"
|
||||
cpu: "1000m"
|
||||
Reference in New Issue
Block a user