refactor(firecrawl): Convert to production-ready Helm chart with persistent storage
CRITICAL FIXES: ✅ Add PersistentVolumeClaims for all stateful services - PostgreSQL: 20Gi PVC on nfs-emporium (data persistence) - Redis: 10Gi PVC on nfs-emporium (cache and queue) - RabbitMQ: 5Gi PVC on nfs-emporium (message queue) ✅ Pin all image versions (no more 'latest' tags) - Firecrawl API/Worker: v1.0.0 - Playwright Service: v1.0.0 - PostgreSQL (nuq-postgres): v1.0.0 - Redis: 7.4.1-alpine - RabbitMQ: 3.13.7-management-alpine ✅ Convert raw manifests to proper Helm chart - Template-based configuration - Centralized values.yaml - Proper Helm helpers and labels - Easy configuration management WHAT CHANGED: - Created chart/ directory with full Helm chart structure - Moved old manifests to old-manifests/ for reference - Updated ArgoCD Application to use Helm chart source - Added comprehensive README and MIGRATION docs - All services now use nfs-emporium storage class - Redis configured with AOF persistence - Proper resource limits and health checks - Gateway/HTTPRoute configs integrated into chart DEPLOYMENT: ArgoCD will automatically sync and apply changes. Old ephemeral data will be lost (fresh start with persistence). Resolves data loss issues and brings deployment to production standards.
This commit is contained in:
Hermes Agent service account
240
cluster/applications/firecrawl/chart/values.yaml
Normal file
240
cluster/applications/firecrawl/chart/values.yaml
Normal file
@@ -0,0 +1,240 @@
|
||||
# ==============================================================================
|
||||
# Firecrawl Helm Chart Values
|
||||
# Production-ready configuration for mk-labs homelab
|
||||
# ==============================================================================
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Image Configuration
|
||||
# ------------------------------------------------------------------------------
|
||||
image:
|
||||
# Main Firecrawl API and Worker image
|
||||
repository: ghcr.io/mendableai/firecrawl
|
||||
tag: "v1.0.0" # Pinned version - update explicitly
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
playwright:
|
||||
repository: ghcr.io/mendableai/firecrawl/playwright-service
|
||||
tag: "v1.0.0" # Pinned version - update explicitly
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# API Service Configuration
|
||||
# ------------------------------------------------------------------------------
|
||||
api:
|
||||
replicaCount: 1
|
||||
|
||||
resources:
|
||||
requests:
|
||||
memory: "4Gi"
|
||||
cpu: "2000m"
|
||||
limits:
|
||||
memory: "6Gi"
|
||||
cpu: "2000m"
|
||||
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 3002
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Worker Service Configuration
|
||||
# ------------------------------------------------------------------------------
|
||||
worker:
|
||||
replicaCount: 1
|
||||
|
||||
resources:
|
||||
requests:
|
||||
memory: "3Gi"
|
||||
cpu: "1000m"
|
||||
limits:
|
||||
memory: "4Gi"
|
||||
cpu: "1000m"
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Playwright Service Configuration
|
||||
# ------------------------------------------------------------------------------
|
||||
playwrightService:
|
||||
replicaCount: 1
|
||||
|
||||
resources:
|
||||
requests:
|
||||
memory: "2Gi"
|
||||
cpu: "1000m"
|
||||
limits:
|
||||
memory: "4Gi"
|
||||
cpu: "2000m"
|
||||
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 3000
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# PostgreSQL Configuration (nuq-postgres)
|
||||
# ------------------------------------------------------------------------------
|
||||
postgres:
|
||||
image:
|
||||
repository: ghcr.io/mendableai/firecrawl/nuq-postgres
|
||||
tag: "v1.0.0" # Pinned version
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
replicaCount: 1
|
||||
|
||||
auth:
|
||||
username: postgres
|
||||
password: postgres
|
||||
database: postgres
|
||||
|
||||
resources:
|
||||
requests:
|
||||
memory: "512Mi"
|
||||
cpu: "250m"
|
||||
limits:
|
||||
memory: "1Gi"
|
||||
cpu: "500m"
|
||||
|
||||
# Persistent storage configuration
|
||||
persistence:
|
||||
enabled: true
|
||||
storageClass: "nfs-emporium"
|
||||
accessMode: ReadWriteOnce
|
||||
size: 20Gi
|
||||
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 5432
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Redis Configuration
|
||||
# ------------------------------------------------------------------------------
|
||||
redis:
|
||||
image:
|
||||
repository: redis
|
||||
tag: "7.4.1-alpine" # Pinned stable version
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
replicaCount: 1
|
||||
|
||||
resources:
|
||||
requests:
|
||||
memory: "256Mi"
|
||||
cpu: "100m"
|
||||
limits:
|
||||
memory: "512Mi"
|
||||
cpu: "500m"
|
||||
|
||||
# Persistent storage configuration
|
||||
persistence:
|
||||
enabled: true
|
||||
storageClass: "nfs-emporium"
|
||||
accessMode: ReadWriteOnce
|
||||
size: 10Gi
|
||||
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 6379
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# RabbitMQ Configuration
|
||||
# ------------------------------------------------------------------------------
|
||||
rabbitmq:
|
||||
enabled: true
|
||||
|
||||
image:
|
||||
repository: rabbitmq
|
||||
tag: "3.13.7-management-alpine" # Pinned stable version
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
replicaCount: 1
|
||||
|
||||
resources:
|
||||
requests:
|
||||
memory: "512Mi"
|
||||
cpu: "250m"
|
||||
limits:
|
||||
memory: "1Gi"
|
||||
cpu: "500m"
|
||||
|
||||
# Persistent storage configuration
|
||||
persistence:
|
||||
enabled: true
|
||||
storageClass: "nfs-emporium"
|
||||
accessMode: ReadWriteOnce
|
||||
size: 5Gi
|
||||
|
||||
service:
|
||||
type: ClusterIP
|
||||
amqpPort: 5672
|
||||
managementPort: 15672
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Application Configuration
|
||||
# ------------------------------------------------------------------------------
|
||||
config:
|
||||
# Environment
|
||||
ENV: "production"
|
||||
IS_KUBERNETES: "true"
|
||||
LOGGING_LEVEL: "INFO"
|
||||
|
||||
# Service configuration
|
||||
PORT: "3002"
|
||||
HOST: "0.0.0.0"
|
||||
WORKER_PORT: "3005"
|
||||
|
||||
# Performance tuning
|
||||
NUM_WORKERS_PER_QUEUE: "8"
|
||||
CRAWL_CONCURRENT_REQUESTS: "10"
|
||||
MAX_CONCURRENT_JOBS: "5"
|
||||
|
||||
# Authentication
|
||||
USE_DB_AUTHENTICATION: "false"
|
||||
|
||||
# URLs (auto-generated if empty)
|
||||
REDIS_URL: ""
|
||||
REDIS_RATE_LIMIT_URL: ""
|
||||
PLAYWRIGHT_MICROSERVICE_URL: ""
|
||||
|
||||
# Optional external services
|
||||
OPENAI_BASE_URL: ""
|
||||
MODEL_NAME: ""
|
||||
MODEL_EMBEDDING_NAME: ""
|
||||
OLLAMA_BASE_URL: ""
|
||||
PROXY_SERVER: ""
|
||||
PROXY_USERNAME: ""
|
||||
SEARXNG_ENDPOINT: ""
|
||||
SEARXNG_ENGINES: ""
|
||||
SEARXNG_CATEGORIES: ""
|
||||
SELF_HOSTED_WEBHOOK_URL: ""
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Playwright Service Configuration
|
||||
# ------------------------------------------------------------------------------
|
||||
playwrightConfig:
|
||||
PORT: "3000"
|
||||
ALLOW_LOCAL_WEBHOOKS: ""
|
||||
BLOCK_MEDIA: ""
|
||||
MAX_CONCURRENT_PAGES: "10"
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Secrets (Optional - use ExternalSecrets in production)
|
||||
# ------------------------------------------------------------------------------
|
||||
secrets:
|
||||
OPENAI_API_KEY: ""
|
||||
SLACK_WEBHOOK_URL: ""
|
||||
LLAMAPARSE_API_KEY: ""
|
||||
BULL_AUTH_KEY: ""
|
||||
TEST_API_KEY: ""
|
||||
FIRE_ENGINE_BETA_URL: ""
|
||||
SUPABASE_ANON_TOKEN: ""
|
||||
SUPABASE_URL: ""
|
||||
SUPABASE_SERVICE_TOKEN: ""
|
||||
PROXY_PASSWORD: ""
|
||||
SELF_HOSTED_WEBHOOK_HMAC_SECRET: ""
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Namespace Configuration
|
||||
# ------------------------------------------------------------------------------
|
||||
namespace:
|
||||
name: firecrawl
|
||||
labels:
|
||||
app.kubernetes.io/name: firecrawl
|
||||
app.kubernetes.io/part-of: mk-labs
|
||||
epcot.theme/name: spaceship-earth
|
||||
Reference in New Issue
Block a user