feat: Deploy Firecrawl (spaceship-earth) to fastpass cluster
- Add Firecrawl application with full stack: - Firecrawl API (main service) - Firecrawl Worker (background jobs) - Playwright Service (browser automation) - Redis (cache & job queue) - PostgreSQL (state management) - RabbitMQ (message queue) - Configure dual DNS names: - Primary: spaceship-earth.local.mk-labs.cloud (EPCOT theme) - Secondary: firecrawl.local.mk-labs.cloud - Add Gateway API HTTPRoutes with TLS certificates - Update ReferenceGrant for firecrawl namespace - Configure ArgoCD application (wave 20) - Set USE_DB_AUTHENTICATION=false for internal deployment This provides JARVIS with web scraping and search capabilities.
This commit is contained in:
60
cluster/applications/firecrawl/configmap.yaml
Normal file
60
cluster/applications/firecrawl/configmap.yaml
Normal file
@@ -0,0 +1,60 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# ConfigMap: firecrawl-config
|
||||
# Environment configuration for Firecrawl services
|
||||
# ------------------------------------------------------------------------------
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: firecrawl-config
|
||||
namespace: firecrawl
|
||||
data:
|
||||
# Service configuration
|
||||
PORT: "3002"
|
||||
HOST: "0.0.0.0"
|
||||
WORKER_PORT: "3005"
|
||||
EXTRACT_WORKER_PORT: "3004"
|
||||
NUQ_WORKER_PORT: "3006"
|
||||
NUQ_PREFETCH_WORKER_PORT: "3011"
|
||||
|
||||
# Redis URLs
|
||||
REDIS_URL: "redis://firecrawl-redis:6379"
|
||||
REDIS_RATE_LIMIT_URL: "redis://firecrawl-redis:6379"
|
||||
|
||||
# PostgreSQL configuration
|
||||
POSTGRES_HOST: "firecrawl-postgres"
|
||||
POSTGRES_PORT: "5432"
|
||||
POSTGRES_USER: "postgres"
|
||||
POSTGRES_PASSWORD: "postgres"
|
||||
POSTGRES_DB: "postgres"
|
||||
|
||||
# Database URLs (constructed from above)
|
||||
NUQ_DATABASE_URL: "postgresql://postgres:postgres@firecrawl-postgres:5432/postgres"
|
||||
NUQ_DATABASE_URL_LISTEN: "postgresql://postgres:postgres@firecrawl-postgres:5432/postgres"
|
||||
|
||||
# RabbitMQ
|
||||
NUQ_RABBITMQ_URL: "amqp://firecrawl-rabbitmq:5672"
|
||||
|
||||
# Playwright service
|
||||
PLAYWRIGHT_MICROSERVICE_URL: "http://firecrawl-playwright:3000/scrape"
|
||||
|
||||
# Worker configuration
|
||||
NUM_WORKERS_PER_QUEUE: "8"
|
||||
NUQ_WORKER_COUNT: "5"
|
||||
|
||||
# Performance tuning
|
||||
CRAWL_CONCURRENT_REQUESTS: "10"
|
||||
MAX_CONCURRENT_JOBS: "5"
|
||||
BROWSER_POOL_SIZE: "5"
|
||||
|
||||
# Authentication (disabled for internal use)
|
||||
USE_DB_AUTHENTICATION: "false"
|
||||
|
||||
# Environment
|
||||
IS_KUBERNETES: "true"
|
||||
ENV: "production"
|
||||
LOGGING_LEVEL: "INFO"
|
||||
|
||||
# Application URLs (internal service mesh)
|
||||
FIRECRAWL_APP_SCHEME: "http"
|
||||
FIRECRAWL_APP_HOST: ""
|
||||
FIRECRAWL_APP_PORT: ""
|
||||
Reference in New Issue
Block a user