f4181349f8
- Add Firecrawl application with full stack: - Firecrawl API (main service) - Firecrawl Worker (background jobs) - Playwright Service (browser automation) - Redis (cache & job queue) - PostgreSQL (state management) - RabbitMQ (message queue) - Configure dual DNS names: - Primary: spaceship-earth.local.mk-labs.cloud (EPCOT theme) - Secondary: firecrawl.local.mk-labs.cloud - Add Gateway API HTTPRoutes with TLS certificates - Update ReferenceGrant for firecrawl namespace - Configure ArgoCD application (wave 20) - Set USE_DB_AUTHENTICATION=false for internal deployment This provides JARVIS with web scraping and search capabilities.
61 lines
1.7 KiB
YAML
61 lines
1.7 KiB
YAML
# ------------------------------------------------------------------------------
|
|
# ConfigMap: firecrawl-config
|
|
# Environment configuration for Firecrawl services
|
|
# ------------------------------------------------------------------------------
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: firecrawl-config
|
|
namespace: firecrawl
|
|
data:
|
|
# Service configuration
|
|
PORT: "3002"
|
|
HOST: "0.0.0.0"
|
|
WORKER_PORT: "3005"
|
|
EXTRACT_WORKER_PORT: "3004"
|
|
NUQ_WORKER_PORT: "3006"
|
|
NUQ_PREFETCH_WORKER_PORT: "3011"
|
|
|
|
# Redis URLs
|
|
REDIS_URL: "redis://firecrawl-redis:6379"
|
|
REDIS_RATE_LIMIT_URL: "redis://firecrawl-redis:6379"
|
|
|
|
# PostgreSQL configuration
|
|
POSTGRES_HOST: "firecrawl-postgres"
|
|
POSTGRES_PORT: "5432"
|
|
POSTGRES_USER: "postgres"
|
|
POSTGRES_PASSWORD: "postgres"
|
|
POSTGRES_DB: "postgres"
|
|
|
|
# Database URLs (constructed from above)
|
|
NUQ_DATABASE_URL: "postgresql://postgres:postgres@firecrawl-postgres:5432/postgres"
|
|
NUQ_DATABASE_URL_LISTEN: "postgresql://postgres:postgres@firecrawl-postgres:5432/postgres"
|
|
|
|
# RabbitMQ
|
|
NUQ_RABBITMQ_URL: "amqp://firecrawl-rabbitmq:5672"
|
|
|
|
# Playwright service
|
|
PLAYWRIGHT_MICROSERVICE_URL: "http://firecrawl-playwright:3000/scrape"
|
|
|
|
# Worker configuration
|
|
NUM_WORKERS_PER_QUEUE: "8"
|
|
NUQ_WORKER_COUNT: "5"
|
|
|
|
# Performance tuning
|
|
CRAWL_CONCURRENT_REQUESTS: "10"
|
|
MAX_CONCURRENT_JOBS: "5"
|
|
BROWSER_POOL_SIZE: "5"
|
|
|
|
# Authentication (disabled for internal use)
|
|
USE_DB_AUTHENTICATION: "false"
|
|
|
|
# Environment
|
|
IS_KUBERNETES: "true"
|
|
ENV: "production"
|
|
LOGGING_LEVEL: "INFO"
|
|
|
|
# Application URLs (internal service mesh)
|
|
FIRECRAWL_APP_SCHEME: "http"
|
|
FIRECRAWL_APP_HOST: ""
|
|
FIRECRAWL_APP_PORT: ""
|