From 6bcb6fa93fc227afbe991fc911293eaf762baa75 Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Thu, 4 Jun 2026 16:56:29 -0500 Subject: [PATCH] refactor(firecrawl): Convert to production-ready Helm chart with persistent storage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CRITICAL FIXES: ✅ Add PersistentVolumeClaims for all stateful services - PostgreSQL: 20Gi PVC on nfs-emporium (data persistence) - Redis: 10Gi PVC on nfs-emporium (cache and queue) - RabbitMQ: 5Gi PVC on nfs-emporium (message queue) ✅ Pin all image versions (no more 'latest' tags) - Firecrawl API/Worker: v1.0.0 - Playwright Service: v1.0.0 - PostgreSQL (nuq-postgres): v1.0.0 - Redis: 7.4.1-alpine - RabbitMQ: 3.13.7-management-alpine ✅ Convert raw manifests to proper Helm chart - Template-based configuration - Centralized values.yaml - Proper Helm helpers and labels - Easy configuration management WHAT CHANGED: - Created chart/ directory with full Helm chart structure - Moved old manifests to old-manifests/ for reference - Updated ArgoCD Application to use Helm chart source - Added comprehensive README and MIGRATION docs - All services now use nfs-emporium storage class - Redis configured with AOF persistence - Proper resource limits and health checks - Gateway/HTTPRoute configs integrated into chart DEPLOYMENT: ArgoCD will automatically sync and apply changes. Old ephemeral data will be lost (fresh start with persistence). Resolves data loss issues and brings deployment to production standards. --- cluster/applications/firecrawl/MIGRATION.md | 97 ++++++ .../applications/firecrawl/application.yaml | 23 +- .../applications/firecrawl/chart/Chart.yaml | 8 + .../applications/firecrawl/chart/README.md | 309 ++++++++++++++++++ .../firecrawl/chart/templates/_helpers.tpl | 49 +++ .../firecrawl/chart/templates/api.yaml | 81 +++++ .../firecrawl/chart/templates/configmap.yaml | 89 +++++ .../firecrawl/chart/templates/httproute.yaml | 87 +++++ .../firecrawl/chart/templates/namespace.yaml | 6 + .../firecrawl/chart/templates/playwright.yaml | 58 ++++ .../firecrawl/chart/templates/postgres.yaml | 67 ++++ .../firecrawl/chart/templates/pvc.yaml | 53 +++ .../firecrawl/chart/templates/rabbitmq.yaml | 82 +++++ .../firecrawl/chart/templates/redis.yaml | 73 +++++ .../firecrawl/chart/templates/worker.yaml | 39 +++ .../applications/firecrawl/chart/values.yaml | 240 ++++++++++++++ .../{ => old-manifests}/api-deployment.yaml | 0 .../{ => old-manifests}/api-service.yaml | 0 .../{ => old-manifests}/configmap.yaml | 0 .../{ => old-manifests}/httproute.yaml | 0 .../{ => old-manifests}/namespace.yaml | 0 .../playwright-configmap.yaml | 0 .../playwright-deployment.yaml | 0 .../playwright-service.yaml | 0 .../postgres-deployment.yaml | 0 .../{ => old-manifests}/postgres-service.yaml | 0 .../rabbitmq-deployment.yaml | 0 .../{ => old-manifests}/rabbitmq-service.yaml | 0 .../{ => old-manifests}/redis-deployment.yaml | 0 .../{ => old-manifests}/redis-service.yaml | 0 .../worker-deployment.yaml | 0 31 files changed, 1354 insertions(+), 7 deletions(-) create mode 100644 cluster/applications/firecrawl/MIGRATION.md create mode 100644 cluster/applications/firecrawl/chart/Chart.yaml create mode 100644 cluster/applications/firecrawl/chart/README.md create mode 100644 cluster/applications/firecrawl/chart/templates/_helpers.tpl create mode 100644 cluster/applications/firecrawl/chart/templates/api.yaml create mode 100644 cluster/applications/firecrawl/chart/templates/configmap.yaml create mode 100644 cluster/applications/firecrawl/chart/templates/httproute.yaml create mode 100644 cluster/applications/firecrawl/chart/templates/namespace.yaml create mode 100644 cluster/applications/firecrawl/chart/templates/playwright.yaml create mode 100644 cluster/applications/firecrawl/chart/templates/postgres.yaml create mode 100644 cluster/applications/firecrawl/chart/templates/pvc.yaml create mode 100644 cluster/applications/firecrawl/chart/templates/rabbitmq.yaml create mode 100644 cluster/applications/firecrawl/chart/templates/redis.yaml create mode 100644 cluster/applications/firecrawl/chart/templates/worker.yaml create mode 100644 cluster/applications/firecrawl/chart/values.yaml rename cluster/applications/firecrawl/{ => old-manifests}/api-deployment.yaml (100%) rename cluster/applications/firecrawl/{ => old-manifests}/api-service.yaml (100%) rename cluster/applications/firecrawl/{ => old-manifests}/configmap.yaml (100%) rename cluster/applications/firecrawl/{ => old-manifests}/httproute.yaml (100%) rename cluster/applications/firecrawl/{ => old-manifests}/namespace.yaml (100%) rename cluster/applications/firecrawl/{ => old-manifests}/playwright-configmap.yaml (100%) rename cluster/applications/firecrawl/{ => old-manifests}/playwright-deployment.yaml (100%) rename cluster/applications/firecrawl/{ => old-manifests}/playwright-service.yaml (100%) rename cluster/applications/firecrawl/{ => old-manifests}/postgres-deployment.yaml (100%) rename cluster/applications/firecrawl/{ => old-manifests}/postgres-service.yaml (100%) rename cluster/applications/firecrawl/{ => old-manifests}/rabbitmq-deployment.yaml (100%) rename cluster/applications/firecrawl/{ => old-manifests}/rabbitmq-service.yaml (100%) rename cluster/applications/firecrawl/{ => old-manifests}/redis-deployment.yaml (100%) rename cluster/applications/firecrawl/{ => old-manifests}/redis-service.yaml (100%) rename cluster/applications/firecrawl/{ => old-manifests}/worker-deployment.yaml (100%) diff --git a/cluster/applications/firecrawl/MIGRATION.md b/cluster/applications/firecrawl/MIGRATION.md new file mode 100644 index 0000000..65a1b39 --- /dev/null +++ b/cluster/applications/firecrawl/MIGRATION.md @@ -0,0 +1,97 @@ +# Migration to Helm Chart + +**Date**: 2026-06-04 +**Reason**: Production-ready refactoring + +## Critical Issues Fixed + +### 1. ❌ No Persistent Storage → ✅ PersistentVolumeClaims +**Old**: PostgreSQL and Redis used `emptyDir` volumes +- All data lost on pod restart +- Not production-ready + +**New**: All stateful services have PVCs +- PostgreSQL: 20Gi PVC on nfs-emporium +- Redis: 10Gi PVC on nfs-emporium +- RabbitMQ: 5Gi PVC on nfs-emporium + +### 2. ❌ Using `latest` Tags → ✅ Pinned Versions +**Old**: Images used `latest` or unspecified tags +- Unpredictable updates +- Hard to rollback +- No version control + +**New**: All images pinned to specific versions +- Firecrawl: `v1.0.0` +- Redis: `7.4.1-alpine` +- RabbitMQ: `3.13.7-management-alpine` +- PostgreSQL: `v1.0.0` +- Playwright: `v1.0.0` + +### 3. ❌ Raw Manifests → ✅ Helm Chart +**Old**: Individual YAML files +- Hard to maintain +- No templating +- Difficult to customize + +**New**: Proper Helm chart structure +- Template-based configuration +- Easy value overrides +- Industry standard deployment + +## Migration Steps + +1. **Backup** (if needed): Old deployment had ephemeral storage, no data to preserve +2. **ArgoCD Auto-Sync**: Push changes and ArgoCD will: + - Delete old resources + - Create new Helm release + - Provision PVCs + - Deploy with pinned versions + +## Old Files Location + +All previous manifests are archived in `old-manifests/` directory for reference. + +## Rollback Procedure + +If needed to rollback: +```bash +# Revert git commit +git revert HEAD + +# ArgoCD will automatically sync back to old state +# Or manually: +argocd app sync firecrawl +``` + +## Verification + +After deployment: +```bash +# Check all PVCs are bound +kubectl -n firecrawl get pvc + +# Verify pods are running +kubectl -n firecrawl get pods + +# Check image versions +kubectl -n firecrawl get pods -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.containers[*].image}{"\n"}{end}' + +# Test API +curl https://spaceship-earth.local.mk-labs.cloud/v0/health/liveness +``` + +## Benefits + +✅ **Data Persistence**: No more data loss on restarts +✅ **Version Control**: Explicit version management +✅ **Configuration Management**: Clean Helm values +✅ **Maintainability**: Standard Helm patterns +✅ **Production Ready**: Follows Kubernetes best practices + +## Notes + +- Storage class `nfs-emporium` provides persistent NFS-backed storage +- All resources properly labeled with Helm chart metadata +- Gateway/HTTPRoute configurations preserved +- EPCOT theming (spaceship-earth) maintained diff --git a/cluster/applications/firecrawl/application.yaml b/cluster/applications/firecrawl/application.yaml index de7a810..c8f4f4d 100644 --- a/cluster/applications/firecrawl/application.yaml +++ b/cluster/applications/firecrawl/application.yaml @@ -10,9 +10,15 @@ # - Firecrawl API (main service) # - Firecrawl Worker (background job processor) # - Playwright Service (browser automation) -# - Redis (cache and job queue) -# - PostgreSQL (state management) -# - RabbitMQ (message queue) +# - Redis (cache and job queue) - WITH PERSISTENT STORAGE +# - PostgreSQL (state management) - WITH PERSISTENT STORAGE +# - RabbitMQ (message queue) - WITH PERSISTENT STORAGE +# +# PRODUCTION-READY FEATURES: +# - All stateful services use PersistentVolumeClaims +# - All images pinned to specific versions +# - Helm chart for configuration management +# - Resource limits and requests properly configured # ------------------------------------------------------------------------------ apiVersion: argoproj.io/v1alpha1 kind: Application @@ -30,10 +36,13 @@ spec: source: repoURL: https://gitea.mk-labs.cloud/rblundon/homelab.git targetRevision: main - path: cluster/applications/firecrawl - directory: - recurse: false - exclude: application.yaml # prevent self-reference loop + path: cluster/applications/firecrawl/chart + helm: + releaseName: firecrawl + values: | + # Production configuration + # Storage classes use nfs-emporium (Synology NFS) + # All images pinned to specific versions for stability destination: server: https://kubernetes.default.svc namespace: firecrawl diff --git a/cluster/applications/firecrawl/chart/Chart.yaml b/cluster/applications/firecrawl/chart/Chart.yaml new file mode 100644 index 0000000..93fd0dd --- /dev/null +++ b/cluster/applications/firecrawl/chart/Chart.yaml @@ -0,0 +1,8 @@ +apiVersion: v2 +name: firecrawl +description: A Helm chart for deploying Firecrawl web scraping service +type: application +version: 1.0.0 +appVersion: "1.0.0" +maintainers: + - name: mk-labs diff --git a/cluster/applications/firecrawl/chart/README.md b/cluster/applications/firecrawl/chart/README.md new file mode 100644 index 0000000..740dd0d --- /dev/null +++ b/cluster/applications/firecrawl/chart/README.md @@ -0,0 +1,309 @@ +# Firecrawl Helm Chart + +Production-ready Helm chart for deploying Firecrawl web scraping service. + +## Overview + +Firecrawl is a web scraping and search service that provides JARVIS with web context capabilities. This deployment includes all necessary components with proper persistent storage and pinned image versions. + +## Components + +### Application Services +- **Firecrawl API** - Main REST API service + - Port: 3002 + - Health checks: `/v0/health/liveness`, `/v0/health/readiness` + - Resources: 4Gi memory, 2 CPU cores + +- **Firecrawl Worker** - Background job processor + - Processes crawl jobs from queue + - Resources: 3Gi memory, 1 CPU core + +- **Playwright Service** - Browser automation + - Port: 3000 + - Headless Chrome/Firefox for JavaScript rendering + - Resources: 2Gi memory, 1 CPU core + +### Infrastructure Services +- **PostgreSQL** (nuq-postgres) - State management + - Port: 5432 + - Custom image with queue extensions + - **Persistent Storage**: 20Gi PVC on nfs-emporium + +- **Redis** - Cache and job queue + - Port: 6379 + - AOF persistence enabled + - **Persistent Storage**: 10Gi PVC on nfs-emporium + +- **RabbitMQ** - Message queue + - AMQP Port: 5672 + - Management UI: 15672 + - **Persistent Storage**: 5Gi PVC on nfs-emporium + +## Production Features + +### ✅ Persistent Storage +All stateful services use PersistentVolumeClaims backed by NFS storage: +- PostgreSQL: 20Gi (database state) +- Redis: 10Gi (cache and job queue) +- RabbitMQ: 5Gi (message queue data) + +### ✅ Pinned Image Versions +All container images use specific version tags (no `latest`): +- Firecrawl API/Worker: `ghcr.io/mendableai/firecrawl:v1.0.0` +- Playwright Service: `ghcr.io/mendableai/firecrawl/playwright-service:v1.0.0` +- PostgreSQL: `ghcr.io/mendableai/firecrawl/nuq-postgres:v1.0.0` +- Redis: `redis:7.4.1-alpine` +- RabbitMQ: `rabbitmq:3.13.7-management-alpine` + +### ✅ Helm Configuration Management +Proper Helm chart structure with: +- Configurable values via `values.yaml` +- Template helpers for consistent naming +- Resource requests and limits +- Health checks and probes +- Service discovery + +## Configuration + +### Key Values + +```yaml +# Image versions (pinned) +image: + repository: ghcr.io/mendableai/firecrawl + tag: "v1.0.0" + +# Persistent storage +postgres.persistence: + enabled: true + storageClass: "nfs-emporium" + size: 20Gi + +redis.persistence: + enabled: true + storageClass: "nfs-emporium" + size: 10Gi + +rabbitmq.persistence: + enabled: true + storageClass: "nfs-emporium" + size: 5Gi + +# Performance tuning +config: + NUM_WORKERS_PER_QUEUE: "8" + CRAWL_CONCURRENT_REQUESTS: "10" + MAX_CONCURRENT_JOBS: "5" +``` + +See `values.yaml` for all configuration options. + +## Deployment + +### ArgoCD (Recommended) +This application is managed by ArgoCD in wave 20 (applications tier): + +```bash +# Check application status +kubectl -n argocd get application firecrawl + +# Sync manually if needed +argocd app sync firecrawl + +# View application details +argocd app get firecrawl +``` + +### Manual Helm Deployment +```bash +# From repository root +cd cluster/applications/firecrawl + +# Install +helm install firecrawl ./chart -n firecrawl --create-namespace + +# Upgrade +helm upgrade firecrawl ./chart -n firecrawl + +# Uninstall +helm uninstall firecrawl -n firecrawl +``` + +## Access + +### External Access (via Gateway API) +- **Primary**: https://spaceship-earth.local.mk-labs.cloud +- **Secondary**: https://firecrawl.local.mk-labs.cloud + +DNS automatically configured via ExternalDNS pointing to Gateway LB (10.1.71.90). + +### Internal Access +``` +http://firecrawl-api.firecrawl.svc.cluster.local:3002 +``` + +### API Usage Example +```bash +curl -X POST https://spaceship-earth.local.mk-labs.cloud/v0/scrape \ + -H "Content-Type: application/json" \ + -d '{"url": "https://example.com"}' +``` + +## Monitoring + +### View Pods +```bash +kubectl -n firecrawl get pods +``` + +### Check Logs +```bash +# API logs +kubectl -n firecrawl logs -l app.kubernetes.io/component=api -f + +# Worker logs +kubectl -n firecrawl logs -l app.kubernetes.io/component=worker -f + +# PostgreSQL logs +kubectl -n firecrawl logs -l app.kubernetes.io/component=postgres -f + +# Redis logs +kubectl -n firecrawl logs -l app.kubernetes.io/component=redis -f +``` + +### Check Storage +```bash +# View PVCs +kubectl -n firecrawl get pvc + +# Check PVC details +kubectl -n firecrawl describe pvc firecrawl-postgres-pvc +kubectl -n firecrawl describe pvc firecrawl-redis-pvc +kubectl -n firecrawl describe pvc firecrawl-rabbitmq-pvc +``` + +## Troubleshooting + +### Pods Not Starting +```bash +kubectl -n firecrawl describe pod +kubectl -n firecrawl get events --sort-by='.lastTimestamp' +``` + +### Storage Issues +```bash +# Check PVC binding +kubectl -n firecrawl get pvc + +# Verify NFS storage class +kubectl get storageclass nfs-emporium -o yaml + +# Check NFS CSI driver +kubectl -n kube-system get pods -l app=csi-nfs-controller +``` + +### Database Connection Issues +```bash +# Test PostgreSQL connectivity from API pod +kubectl -n firecrawl exec -it deployment/firecrawl-api -- \ + sh -c 'apt-get update && apt-get install -y postgresql-client && \ + psql -h firecrawl-postgres -U postgres -d postgres -c "SELECT 1"' + +# Test Redis connectivity +kubectl -n firecrawl exec -it deployment/firecrawl-api -- \ + sh -c 'apt-get update && apt-get install -y redis-tools && \ + redis-cli -h firecrawl-redis PING' +``` + +### Gateway Routing Issues +```bash +# Verify ReferenceGrant +kubectl -n gateway get referencegrant allow-httproutes -o yaml + +# Check HTTPRoutes +kubectl -n firecrawl get httproute + +# View Gateway status +kubectl -n gateway get gateway fastpass-gateway -o yaml +``` + +## Upgrading + +### Update Image Versions +1. Edit `values.yaml` and update image tags +2. Commit and push changes +3. ArgoCD will automatically sync +4. Or manually: `helm upgrade firecrawl ./chart -n firecrawl` + +### Migrate from Old Deployment +The old raw manifest deployment will be automatically replaced by ArgoCD: +1. Commit this Helm chart +2. Push to repository +3. ArgoCD detects the change and applies Helm chart +4. Old resources are pruned +5. New resources with PVCs are created + +**Note**: Old PostgreSQL/Redis data was ephemeral and will be lost. Fresh start with persistent storage. + +## Storage Architecture + +### NFS Provisioner +- **Storage Class**: `nfs-emporium` +- **Provisioner**: `nfs.csi.k8s.io` +- **NFS Server**: emporium (Synology DS1621+) - 10.1.71.9 +- **NFS Export**: `/volume1/fastpass` +- **Reclaim Policy**: Delete +- **Binding Mode**: Immediate + +### PVC Structure +Each PVC gets a unique subdirectory: +``` +/volume1/fastpass/firecrawl/firecrawl-postgres-pvc/ +/volume1/fastpass/firecrawl/firecrawl-redis-pvc/ +/volume1/fastpass/firecrawl/firecrawl-rabbitmq-pvc/ +``` + +## Theme + +Part of the EPCOT-themed namespace with primary hostname **spaceship-earth** representing Firecrawl's role as the information and knowledge hub for JARVIS. + +## Maintenance + +### Backup Recommendations +```bash +# PostgreSQL backup (manual) +kubectl -n firecrawl exec deployment/firecrawl-postgres -- \ + pg_dump -U postgres postgres > firecrawl-backup-$(date +%Y%m%d).sql + +# NFS snapshots (via Synology) +# Use Synology Snapshot Replication for point-in-time backups +``` + +### Scaling +```bash +# Scale API replicas +helm upgrade firecrawl ./chart -n firecrawl \ + --set api.replicaCount=2 + +# Scale workers +helm upgrade firecrawl ./chart -n firecrawl \ + --set worker.replicaCount=3 +``` + +## Future Enhancements +- [ ] Add Prometheus metrics and ServiceMonitor +- [ ] Implement resource quotas and LimitRanges +- [ ] Add PodDisruptionBudgets for HA +- [ ] Configure NetworkPolicies for security +- [ ] Add external secrets management (HashiCorp Vault or Sealed Secrets) +- [ ] Implement horizontal pod autoscaling +- [ ] Add extract worker deployment +- [ ] Add nuq worker deployment + +## Support + +For issues or questions: +- Check ArgoCD application status +- Review pod logs +- Verify storage bindings +- Consult Firecrawl documentation: https://docs.firecrawl.dev diff --git a/cluster/applications/firecrawl/chart/templates/_helpers.tpl b/cluster/applications/firecrawl/chart/templates/_helpers.tpl new file mode 100644 index 0000000..eafdc33 --- /dev/null +++ b/cluster/applications/firecrawl/chart/templates/_helpers.tpl @@ -0,0 +1,49 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "firecrawl.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "firecrawl.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "firecrawl.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "firecrawl.labels" -}} +helm.sh/chart: {{ include "firecrawl.chart" . }} +{{ include "firecrawl.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "firecrawl.selectorLabels" -}} +app.kubernetes.io/name: {{ include "firecrawl.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/cluster/applications/firecrawl/chart/templates/api.yaml b/cluster/applications/firecrawl/chart/templates/api.yaml new file mode 100644 index 0000000..238d3cb --- /dev/null +++ b/cluster/applications/firecrawl/chart/templates/api.yaml @@ -0,0 +1,81 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "firecrawl.fullname" . }}-api + namespace: {{ .Values.namespace.name }} + labels: + {{- include "firecrawl.labels" . | nindent 4 }} + app.kubernetes.io/component: api + epcot.theme/service: spaceship-earth +spec: + replicas: {{ .Values.api.replicaCount }} + selector: + matchLabels: + {{- include "firecrawl.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: api + template: + metadata: + labels: + {{- include "firecrawl.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: api + epcot.theme/service: spaceship-earth + spec: + terminationGracePeriodSeconds: 180 + containers: + - name: api + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["node"] + args: ["--max-old-space-size=6144", "dist/src/index.js"] + envFrom: + - configMapRef: + name: {{ include "firecrawl.fullname" . }}-config + env: + - name: FLY_PROCESS_GROUP + value: "app" + - name: NUQ_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + ports: + - containerPort: 3002 + name: http + resources: + {{- toYaml .Values.api.resources | nindent 12 }} + livenessProbe: + httpGet: + path: /v0/health/liveness + port: 3002 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /v0/health/readiness + port: 3002 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "firecrawl.fullname" . }}-api + namespace: {{ .Values.namespace.name }} + labels: + {{- include "firecrawl.labels" . | nindent 4 }} + app.kubernetes.io/component: api +spec: + type: {{ .Values.api.service.type }} + ports: + - port: {{ .Values.api.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "firecrawl.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: api diff --git a/cluster/applications/firecrawl/chart/templates/configmap.yaml b/cluster/applications/firecrawl/chart/templates/configmap.yaml new file mode 100644 index 0000000..5ce3818 --- /dev/null +++ b/cluster/applications/firecrawl/chart/templates/configmap.yaml @@ -0,0 +1,89 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "firecrawl.fullname" . }}-config + namespace: {{ .Values.namespace.name }} + labels: + {{- include "firecrawl.labels" . | nindent 4 }} +data: + # Environment + ENV: {{ .Values.config.ENV | quote }} + IS_KUBERNETES: {{ .Values.config.IS_KUBERNETES | quote }} + LOGGING_LEVEL: {{ .Values.config.LOGGING_LEVEL | quote }} + + # Service configuration + PORT: {{ .Values.config.PORT | quote }} + HOST: {{ .Values.config.HOST | quote }} + WORKER_PORT: {{ .Values.config.WORKER_PORT | quote }} + + # Performance tuning + NUM_WORKERS_PER_QUEUE: {{ .Values.config.NUM_WORKERS_PER_QUEUE | quote }} + CRAWL_CONCURRENT_REQUESTS: {{ .Values.config.CRAWL_CONCURRENT_REQUESTS | quote }} + MAX_CONCURRENT_JOBS: {{ .Values.config.MAX_CONCURRENT_JOBS | quote }} + + # Authentication + USE_DB_AUTHENTICATION: {{ .Values.config.USE_DB_AUTHENTICATION | quote }} + + # Service URLs + REDIS_URL: {{ .Values.config.REDIS_URL | default (printf "redis://%s-redis:%d" (include "firecrawl.fullname" .) (.Values.redis.service.port | int)) | quote }} + REDIS_RATE_LIMIT_URL: {{ .Values.config.REDIS_RATE_LIMIT_URL | default (printf "redis://%s-redis:%d" (include "firecrawl.fullname" .) (.Values.redis.service.port | int)) | quote }} + PLAYWRIGHT_MICROSERVICE_URL: {{ .Values.config.PLAYWRIGHT_MICROSERVICE_URL | default (printf "http://%s-playwright:%d/scrape" (include "firecrawl.fullname" .) (.Values.playwrightService.service.port | int)) | quote }} + + # Database configuration + POSTGRES_HOST: {{ printf "%s-postgres" (include "firecrawl.fullname" .) | quote }} + POSTGRES_PORT: {{ .Values.postgres.service.port | quote }} + POSTGRES_USER: {{ .Values.postgres.auth.username | quote }} + POSTGRES_PASSWORD: {{ .Values.postgres.auth.password | quote }} + POSTGRES_DB: {{ .Values.postgres.auth.database | quote }} + + # RabbitMQ configuration + NUQ_RABBITMQ_URL: {{ printf "amqp://%s-rabbitmq:%d" (include "firecrawl.fullname" .) (.Values.rabbitmq.service.amqpPort | int) | quote }} + + # Optional external services + {{- if .Values.config.OPENAI_BASE_URL }} + OPENAI_BASE_URL: {{ .Values.config.OPENAI_BASE_URL | quote }} + {{- end }} + {{- if .Values.config.MODEL_NAME }} + MODEL_NAME: {{ .Values.config.MODEL_NAME | quote }} + {{- end }} + {{- if .Values.config.MODEL_EMBEDDING_NAME }} + MODEL_EMBEDDING_NAME: {{ .Values.config.MODEL_EMBEDDING_NAME | quote }} + {{- end }} + {{- if .Values.config.OLLAMA_BASE_URL }} + OLLAMA_BASE_URL: {{ .Values.config.OLLAMA_BASE_URL | quote }} + {{- end }} + {{- if .Values.config.PROXY_SERVER }} + PROXY_SERVER: {{ .Values.config.PROXY_SERVER | quote }} + {{- end }} + {{- if .Values.config.PROXY_USERNAME }} + PROXY_USERNAME: {{ .Values.config.PROXY_USERNAME | quote }} + {{- end }} + {{- if .Values.config.SEARXNG_ENDPOINT }} + SEARXNG_ENDPOINT: {{ .Values.config.SEARXNG_ENDPOINT | quote }} + {{- end }} + {{- if .Values.config.SEARXNG_ENGINES }} + SEARXNG_ENGINES: {{ .Values.config.SEARXNG_ENGINES | quote }} + {{- end }} + {{- if .Values.config.SEARXNG_CATEGORIES }} + SEARXNG_CATEGORIES: {{ .Values.config.SEARXNG_CATEGORIES | quote }} + {{- end }} + {{- if .Values.config.SELF_HOSTED_WEBHOOK_URL }} + SELF_HOSTED_WEBHOOK_URL: {{ .Values.config.SELF_HOSTED_WEBHOOK_URL | quote }} + {{- end }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "firecrawl.fullname" . }}-playwright-config + namespace: {{ .Values.namespace.name }} + labels: + {{- include "firecrawl.labels" . | nindent 4 }} +data: + PORT: {{ .Values.playwrightConfig.PORT | quote }} + MAX_CONCURRENT_PAGES: {{ .Values.playwrightConfig.MAX_CONCURRENT_PAGES | quote }} + {{- if .Values.playwrightConfig.ALLOW_LOCAL_WEBHOOKS }} + ALLOW_LOCAL_WEBHOOKS: {{ .Values.playwrightConfig.ALLOW_LOCAL_WEBHOOKS | quote }} + {{- end }} + {{- if .Values.playwrightConfig.BLOCK_MEDIA }} + BLOCK_MEDIA: {{ .Values.playwrightConfig.BLOCK_MEDIA | quote }} + {{- end }} diff --git a/cluster/applications/firecrawl/chart/templates/httproute.yaml b/cluster/applications/firecrawl/chart/templates/httproute.yaml new file mode 100644 index 0000000..84e708a --- /dev/null +++ b/cluster/applications/firecrawl/chart/templates/httproute.yaml @@ -0,0 +1,87 @@ +# ------------------------------------------------------------------------------ +# HTTPRoute — Firecrawl via Cilium Gateway +# Both primary (spaceship-earth) and secondary (firecrawl) DNS names +# ------------------------------------------------------------------------------ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: spaceship-earth-tls + namespace: {{ .Values.namespace.name }} +spec: + secretName: spaceship-earth-tls + issuerRef: + name: letsencrypt-prod + kind: ClusterIssuer + dnsNames: + - spaceship-earth.local.mk-labs.cloud +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: firecrawl-tls + namespace: {{ .Values.namespace.name }} +spec: + secretName: firecrawl-tls + issuerRef: + name: letsencrypt-prod + kind: ClusterIssuer + dnsNames: + - firecrawl.local.mk-labs.cloud +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: spaceship-earth + namespace: {{ .Values.namespace.name }} + annotations: + external-dns.alpha.kubernetes.io/hostname: spaceship-earth.local.mk-labs.cloud + external-dns.alpha.kubernetes.io/target: "10.1.71.90" +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: fastpass-gateway + namespace: gateway + sectionName: https + hostnames: + - spaceship-earth.local.mk-labs.cloud + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - group: "" + kind: Service + name: {{ include "firecrawl.fullname" . }}-api + port: {{ .Values.api.service.port }} + weight: 1 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: firecrawl + namespace: {{ .Values.namespace.name }} + annotations: + external-dns.alpha.kubernetes.io/hostname: firecrawl.local.mk-labs.cloud + external-dns.alpha.kubernetes.io/target: "10.1.71.90" +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: fastpass-gateway + namespace: gateway + sectionName: https + hostnames: + - firecrawl.local.mk-labs.cloud + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - group: "" + kind: Service + name: {{ include "firecrawl.fullname" . }}-api + port: {{ .Values.api.service.port }} + weight: 1 diff --git a/cluster/applications/firecrawl/chart/templates/namespace.yaml b/cluster/applications/firecrawl/chart/templates/namespace.yaml new file mode 100644 index 0000000..fb54d7b --- /dev/null +++ b/cluster/applications/firecrawl/chart/templates/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.namespace.name }} + labels: + {{- toYaml .Values.namespace.labels | nindent 4 }} diff --git a/cluster/applications/firecrawl/chart/templates/playwright.yaml b/cluster/applications/firecrawl/chart/templates/playwright.yaml new file mode 100644 index 0000000..078bb00 --- /dev/null +++ b/cluster/applications/firecrawl/chart/templates/playwright.yaml @@ -0,0 +1,58 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "firecrawl.fullname" . }}-playwright + namespace: {{ .Values.namespace.name }} + labels: + {{- include "firecrawl.labels" . | nindent 4 }} + app.kubernetes.io/component: playwright +spec: + replicas: {{ .Values.playwrightService.replicaCount }} + selector: + matchLabels: + {{- include "firecrawl.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: playwright + template: + metadata: + labels: + {{- include "firecrawl.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: playwright + spec: + containers: + - name: playwright + image: "{{ .Values.playwright.repository }}:{{ .Values.playwright.tag }}" + imagePullPolicy: {{ .Values.playwright.pullPolicy }} + envFrom: + - configMapRef: + name: {{ include "firecrawl.fullname" . }}-playwright-config + ports: + - containerPort: 3000 + name: http + resources: + {{- toYaml .Values.playwrightService.resources | nindent 12 }} + volumeMounts: + - name: cache + mountPath: /tmp/.cache + volumes: + - name: cache + emptyDir: + sizeLimit: 1Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "firecrawl.fullname" . }}-playwright + namespace: {{ .Values.namespace.name }} + labels: + {{- include "firecrawl.labels" . | nindent 4 }} + app.kubernetes.io/component: playwright +spec: + type: {{ .Values.playwrightService.service.type }} + ports: + - port: {{ .Values.playwrightService.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "firecrawl.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: playwright diff --git a/cluster/applications/firecrawl/chart/templates/postgres.yaml b/cluster/applications/firecrawl/chart/templates/postgres.yaml new file mode 100644 index 0000000..2c9c63e --- /dev/null +++ b/cluster/applications/firecrawl/chart/templates/postgres.yaml @@ -0,0 +1,67 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "firecrawl.fullname" . }}-postgres + namespace: {{ .Values.namespace.name }} + labels: + {{- include "firecrawl.labels" . | nindent 4 }} + app.kubernetes.io/component: postgres +spec: + replicas: {{ .Values.postgres.replicaCount }} + selector: + matchLabels: + {{- include "firecrawl.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: postgres + template: + metadata: + labels: + {{- include "firecrawl.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: postgres + spec: + containers: + - name: postgres + image: "{{ .Values.postgres.image.repository }}:{{ .Values.postgres.image.tag }}" + imagePullPolicy: {{ .Values.postgres.image.pullPolicy }} + env: + - name: POSTGRES_USER + value: {{ .Values.postgres.auth.username | quote }} + - name: POSTGRES_PASSWORD + value: {{ .Values.postgres.auth.password | quote }} + - name: POSTGRES_DB + value: {{ .Values.postgres.auth.database | quote }} + ports: + - containerPort: 5432 + name: postgres + resources: + {{- toYaml .Values.postgres.resources | nindent 12 }} + volumeMounts: + - name: postgres-data + mountPath: /var/lib/postgresql/data + subPath: postgres + volumes: + - name: postgres-data + {{- if .Values.postgres.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ include "firecrawl.fullname" . }}-postgres-pvc + {{- else }} + emptyDir: {} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "firecrawl.fullname" . }}-postgres + namespace: {{ .Values.namespace.name }} + labels: + {{- include "firecrawl.labels" . | nindent 4 }} + app.kubernetes.io/component: postgres +spec: + type: {{ .Values.postgres.service.type }} + ports: + - port: {{ .Values.postgres.service.port }} + targetPort: postgres + protocol: TCP + name: postgres + selector: + {{- include "firecrawl.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: postgres diff --git a/cluster/applications/firecrawl/chart/templates/pvc.yaml b/cluster/applications/firecrawl/chart/templates/pvc.yaml new file mode 100644 index 0000000..e0cc52a --- /dev/null +++ b/cluster/applications/firecrawl/chart/templates/pvc.yaml @@ -0,0 +1,53 @@ +{{- if .Values.postgres.persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "firecrawl.fullname" . }}-postgres-pvc + namespace: {{ .Values.namespace.name }} + labels: + {{- include "firecrawl.labels" . | nindent 4 }} + app.kubernetes.io/component: postgres +spec: + accessModes: + - {{ .Values.postgres.persistence.accessMode }} + storageClassName: {{ .Values.postgres.persistence.storageClass | quote }} + resources: + requests: + storage: {{ .Values.postgres.persistence.size | quote }} +{{- end }} +--- +{{- if .Values.redis.persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "firecrawl.fullname" . }}-redis-pvc + namespace: {{ .Values.namespace.name }} + labels: + {{- include "firecrawl.labels" . | nindent 4 }} + app.kubernetes.io/component: redis +spec: + accessModes: + - {{ .Values.redis.persistence.accessMode }} + storageClassName: {{ .Values.redis.persistence.storageClass | quote }} + resources: + requests: + storage: {{ .Values.redis.persistence.size | quote }} +{{- end }} +--- +{{- if and .Values.rabbitmq.enabled .Values.rabbitmq.persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "firecrawl.fullname" . }}-rabbitmq-pvc + namespace: {{ .Values.namespace.name }} + labels: + {{- include "firecrawl.labels" . | nindent 4 }} + app.kubernetes.io/component: rabbitmq +spec: + accessModes: + - {{ .Values.rabbitmq.persistence.accessMode }} + storageClassName: {{ .Values.rabbitmq.persistence.storageClass | quote }} + resources: + requests: + storage: {{ .Values.rabbitmq.persistence.size | quote }} +{{- end }} diff --git a/cluster/applications/firecrawl/chart/templates/rabbitmq.yaml b/cluster/applications/firecrawl/chart/templates/rabbitmq.yaml new file mode 100644 index 0000000..5f2d8fe --- /dev/null +++ b/cluster/applications/firecrawl/chart/templates/rabbitmq.yaml @@ -0,0 +1,82 @@ +{{- if .Values.rabbitmq.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "firecrawl.fullname" . }}-rabbitmq + namespace: {{ .Values.namespace.name }} + labels: + {{- include "firecrawl.labels" . | nindent 4 }} + app.kubernetes.io/component: rabbitmq +spec: + replicas: {{ .Values.rabbitmq.replicaCount }} + selector: + matchLabels: + {{- include "firecrawl.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: rabbitmq + template: + metadata: + labels: + {{- include "firecrawl.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: rabbitmq + spec: + containers: + - name: rabbitmq + image: "{{ .Values.rabbitmq.image.repository }}:{{ .Values.rabbitmq.image.tag }}" + imagePullPolicy: {{ .Values.rabbitmq.image.pullPolicy }} + command: ["rabbitmq-server"] + ports: + - containerPort: 5672 + name: amqp + - containerPort: 15672 + name: management + resources: + {{- toYaml .Values.rabbitmq.resources | nindent 12 }} + livenessProbe: + exec: + command: ["rabbitmq-diagnostics", "-q", "check_running"] + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + exec: + command: ["rabbitmq-diagnostics", "-q", "check_running"] + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 3 + volumeMounts: + {{- if .Values.rabbitmq.persistence.enabled }} + - name: rabbitmq-data + mountPath: /var/lib/rabbitmq + {{- end }} + volumes: + {{- if .Values.rabbitmq.persistence.enabled }} + - name: rabbitmq-data + persistentVolumeClaim: + claimName: {{ include "firecrawl.fullname" . }}-rabbitmq-pvc + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "firecrawl.fullname" . }}-rabbitmq + namespace: {{ .Values.namespace.name }} + labels: + {{- include "firecrawl.labels" . | nindent 4 }} + app.kubernetes.io/component: rabbitmq +spec: + type: {{ .Values.rabbitmq.service.type }} + ports: + - port: {{ .Values.rabbitmq.service.amqpPort }} + targetPort: amqp + protocol: TCP + name: amqp + - port: {{ .Values.rabbitmq.service.managementPort }} + targetPort: management + protocol: TCP + name: management + selector: + {{- include "firecrawl.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: rabbitmq +{{- end }} diff --git a/cluster/applications/firecrawl/chart/templates/redis.yaml b/cluster/applications/firecrawl/chart/templates/redis.yaml new file mode 100644 index 0000000..f6c4066 --- /dev/null +++ b/cluster/applications/firecrawl/chart/templates/redis.yaml @@ -0,0 +1,73 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "firecrawl.fullname" . }}-redis + namespace: {{ .Values.namespace.name }} + labels: + {{- include "firecrawl.labels" . | nindent 4 }} + app.kubernetes.io/component: redis +spec: + replicas: {{ .Values.redis.replicaCount }} + selector: + matchLabels: + {{- include "firecrawl.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: redis + template: + metadata: + labels: + {{- include "firecrawl.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: redis + spec: + containers: + - name: redis + image: "{{ .Values.redis.image.repository }}:{{ .Values.redis.image.tag }}" + imagePullPolicy: {{ .Values.redis.image.pullPolicy }} + command: ["redis-server"] + args: + - "--bind" + - "0.0.0.0" + {{- if .Values.redis.persistence.enabled }} + - "--appendonly" + - "yes" + - "--save" + - "900 1" + - "--save" + - "300 10" + - "--save" + - "60 10000" + {{- end }} + ports: + - containerPort: 6379 + name: redis + resources: + {{- toYaml .Values.redis.resources | nindent 12 }} + volumeMounts: + {{- if .Values.redis.persistence.enabled }} + - name: redis-data + mountPath: /data + {{- end }} + volumes: + {{- if .Values.redis.persistence.enabled }} + - name: redis-data + persistentVolumeClaim: + claimName: {{ include "firecrawl.fullname" . }}-redis-pvc + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "firecrawl.fullname" . }}-redis + namespace: {{ .Values.namespace.name }} + labels: + {{- include "firecrawl.labels" . | nindent 4 }} + app.kubernetes.io/component: redis +spec: + type: {{ .Values.redis.service.type }} + ports: + - port: {{ .Values.redis.service.port }} + targetPort: redis + protocol: TCP + name: redis + selector: + {{- include "firecrawl.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: redis diff --git a/cluster/applications/firecrawl/chart/templates/worker.yaml b/cluster/applications/firecrawl/chart/templates/worker.yaml new file mode 100644 index 0000000..6adc5a8 --- /dev/null +++ b/cluster/applications/firecrawl/chart/templates/worker.yaml @@ -0,0 +1,39 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "firecrawl.fullname" . }}-worker + namespace: {{ .Values.namespace.name }} + labels: + {{- include "firecrawl.labels" . | nindent 4 }} + app.kubernetes.io/component: worker +spec: + replicas: {{ .Values.worker.replicaCount }} + selector: + matchLabels: + {{- include "firecrawl.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: worker + template: + metadata: + labels: + {{- include "firecrawl.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: worker + spec: + terminationGracePeriodSeconds: 180 + containers: + - name: worker + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["node"] + args: ["--max-old-space-size=4096", "dist/src/services/queue-worker.js"] + envFrom: + - configMapRef: + name: {{ include "firecrawl.fullname" . }}-config + env: + - name: FLY_PROCESS_GROUP + value: "worker" + - name: NUQ_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + resources: + {{- toYaml .Values.worker.resources | nindent 12 }} diff --git a/cluster/applications/firecrawl/chart/values.yaml b/cluster/applications/firecrawl/chart/values.yaml new file mode 100644 index 0000000..a8db8fb --- /dev/null +++ b/cluster/applications/firecrawl/chart/values.yaml @@ -0,0 +1,240 @@ +# ============================================================================== +# Firecrawl Helm Chart Values +# Production-ready configuration for mk-labs homelab +# ============================================================================== + +# ------------------------------------------------------------------------------ +# Image Configuration +# ------------------------------------------------------------------------------ +image: + # Main Firecrawl API and Worker image + repository: ghcr.io/mendableai/firecrawl + tag: "v1.0.0" # Pinned version - update explicitly + pullPolicy: IfNotPresent + +playwright: + repository: ghcr.io/mendableai/firecrawl/playwright-service + tag: "v1.0.0" # Pinned version - update explicitly + pullPolicy: IfNotPresent + +# ------------------------------------------------------------------------------ +# API Service Configuration +# ------------------------------------------------------------------------------ +api: + replicaCount: 1 + + resources: + requests: + memory: "4Gi" + cpu: "2000m" + limits: + memory: "6Gi" + cpu: "2000m" + + service: + type: ClusterIP + port: 3002 + +# ------------------------------------------------------------------------------ +# Worker Service Configuration +# ------------------------------------------------------------------------------ +worker: + replicaCount: 1 + + resources: + requests: + memory: "3Gi" + cpu: "1000m" + limits: + memory: "4Gi" + cpu: "1000m" + +# ------------------------------------------------------------------------------ +# Playwright Service Configuration +# ------------------------------------------------------------------------------ +playwrightService: + replicaCount: 1 + + resources: + requests: + memory: "2Gi" + cpu: "1000m" + limits: + memory: "4Gi" + cpu: "2000m" + + service: + type: ClusterIP + port: 3000 + +# ------------------------------------------------------------------------------ +# PostgreSQL Configuration (nuq-postgres) +# ------------------------------------------------------------------------------ +postgres: + image: + repository: ghcr.io/mendableai/firecrawl/nuq-postgres + tag: "v1.0.0" # Pinned version + pullPolicy: IfNotPresent + + replicaCount: 1 + + auth: + username: postgres + password: postgres + database: postgres + + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "1Gi" + cpu: "500m" + + # Persistent storage configuration + persistence: + enabled: true + storageClass: "nfs-emporium" + accessMode: ReadWriteOnce + size: 20Gi + + service: + type: ClusterIP + port: 5432 + +# ------------------------------------------------------------------------------ +# Redis Configuration +# ------------------------------------------------------------------------------ +redis: + image: + repository: redis + tag: "7.4.1-alpine" # Pinned stable version + pullPolicy: IfNotPresent + + replicaCount: 1 + + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + + # Persistent storage configuration + persistence: + enabled: true + storageClass: "nfs-emporium" + accessMode: ReadWriteOnce + size: 10Gi + + service: + type: ClusterIP + port: 6379 + +# ------------------------------------------------------------------------------ +# RabbitMQ Configuration +# ------------------------------------------------------------------------------ +rabbitmq: + enabled: true + + image: + repository: rabbitmq + tag: "3.13.7-management-alpine" # Pinned stable version + pullPolicy: IfNotPresent + + replicaCount: 1 + + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "1Gi" + cpu: "500m" + + # Persistent storage configuration + persistence: + enabled: true + storageClass: "nfs-emporium" + accessMode: ReadWriteOnce + size: 5Gi + + service: + type: ClusterIP + amqpPort: 5672 + managementPort: 15672 + +# ------------------------------------------------------------------------------ +# Application Configuration +# ------------------------------------------------------------------------------ +config: + # Environment + ENV: "production" + IS_KUBERNETES: "true" + LOGGING_LEVEL: "INFO" + + # Service configuration + PORT: "3002" + HOST: "0.0.0.0" + WORKER_PORT: "3005" + + # Performance tuning + NUM_WORKERS_PER_QUEUE: "8" + CRAWL_CONCURRENT_REQUESTS: "10" + MAX_CONCURRENT_JOBS: "5" + + # Authentication + USE_DB_AUTHENTICATION: "false" + + # URLs (auto-generated if empty) + REDIS_URL: "" + REDIS_RATE_LIMIT_URL: "" + PLAYWRIGHT_MICROSERVICE_URL: "" + + # Optional external services + OPENAI_BASE_URL: "" + MODEL_NAME: "" + MODEL_EMBEDDING_NAME: "" + OLLAMA_BASE_URL: "" + PROXY_SERVER: "" + PROXY_USERNAME: "" + SEARXNG_ENDPOINT: "" + SEARXNG_ENGINES: "" + SEARXNG_CATEGORIES: "" + SELF_HOSTED_WEBHOOK_URL: "" + +# ------------------------------------------------------------------------------ +# Playwright Service Configuration +# ------------------------------------------------------------------------------ +playwrightConfig: + PORT: "3000" + ALLOW_LOCAL_WEBHOOKS: "" + BLOCK_MEDIA: "" + MAX_CONCURRENT_PAGES: "10" + +# ------------------------------------------------------------------------------ +# Secrets (Optional - use ExternalSecrets in production) +# ------------------------------------------------------------------------------ +secrets: + OPENAI_API_KEY: "" + SLACK_WEBHOOK_URL: "" + LLAMAPARSE_API_KEY: "" + BULL_AUTH_KEY: "" + TEST_API_KEY: "" + FIRE_ENGINE_BETA_URL: "" + SUPABASE_ANON_TOKEN: "" + SUPABASE_URL: "" + SUPABASE_SERVICE_TOKEN: "" + PROXY_PASSWORD: "" + SELF_HOSTED_WEBHOOK_HMAC_SECRET: "" + +# ------------------------------------------------------------------------------ +# Namespace Configuration +# ------------------------------------------------------------------------------ +namespace: + name: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/part-of: mk-labs + epcot.theme/name: spaceship-earth diff --git a/cluster/applications/firecrawl/api-deployment.yaml b/cluster/applications/firecrawl/old-manifests/api-deployment.yaml similarity index 100% rename from cluster/applications/firecrawl/api-deployment.yaml rename to cluster/applications/firecrawl/old-manifests/api-deployment.yaml diff --git a/cluster/applications/firecrawl/api-service.yaml b/cluster/applications/firecrawl/old-manifests/api-service.yaml similarity index 100% rename from cluster/applications/firecrawl/api-service.yaml rename to cluster/applications/firecrawl/old-manifests/api-service.yaml diff --git a/cluster/applications/firecrawl/configmap.yaml b/cluster/applications/firecrawl/old-manifests/configmap.yaml similarity index 100% rename from cluster/applications/firecrawl/configmap.yaml rename to cluster/applications/firecrawl/old-manifests/configmap.yaml diff --git a/cluster/applications/firecrawl/httproute.yaml b/cluster/applications/firecrawl/old-manifests/httproute.yaml similarity index 100% rename from cluster/applications/firecrawl/httproute.yaml rename to cluster/applications/firecrawl/old-manifests/httproute.yaml diff --git a/cluster/applications/firecrawl/namespace.yaml b/cluster/applications/firecrawl/old-manifests/namespace.yaml similarity index 100% rename from cluster/applications/firecrawl/namespace.yaml rename to cluster/applications/firecrawl/old-manifests/namespace.yaml diff --git a/cluster/applications/firecrawl/playwright-configmap.yaml b/cluster/applications/firecrawl/old-manifests/playwright-configmap.yaml similarity index 100% rename from cluster/applications/firecrawl/playwright-configmap.yaml rename to cluster/applications/firecrawl/old-manifests/playwright-configmap.yaml diff --git a/cluster/applications/firecrawl/playwright-deployment.yaml b/cluster/applications/firecrawl/old-manifests/playwright-deployment.yaml similarity index 100% rename from cluster/applications/firecrawl/playwright-deployment.yaml rename to cluster/applications/firecrawl/old-manifests/playwright-deployment.yaml diff --git a/cluster/applications/firecrawl/playwright-service.yaml b/cluster/applications/firecrawl/old-manifests/playwright-service.yaml similarity index 100% rename from cluster/applications/firecrawl/playwright-service.yaml rename to cluster/applications/firecrawl/old-manifests/playwright-service.yaml diff --git a/cluster/applications/firecrawl/postgres-deployment.yaml b/cluster/applications/firecrawl/old-manifests/postgres-deployment.yaml similarity index 100% rename from cluster/applications/firecrawl/postgres-deployment.yaml rename to cluster/applications/firecrawl/old-manifests/postgres-deployment.yaml diff --git a/cluster/applications/firecrawl/postgres-service.yaml b/cluster/applications/firecrawl/old-manifests/postgres-service.yaml similarity index 100% rename from cluster/applications/firecrawl/postgres-service.yaml rename to cluster/applications/firecrawl/old-manifests/postgres-service.yaml diff --git a/cluster/applications/firecrawl/rabbitmq-deployment.yaml b/cluster/applications/firecrawl/old-manifests/rabbitmq-deployment.yaml similarity index 100% rename from cluster/applications/firecrawl/rabbitmq-deployment.yaml rename to cluster/applications/firecrawl/old-manifests/rabbitmq-deployment.yaml diff --git a/cluster/applications/firecrawl/rabbitmq-service.yaml b/cluster/applications/firecrawl/old-manifests/rabbitmq-service.yaml similarity index 100% rename from cluster/applications/firecrawl/rabbitmq-service.yaml rename to cluster/applications/firecrawl/old-manifests/rabbitmq-service.yaml diff --git a/cluster/applications/firecrawl/redis-deployment.yaml b/cluster/applications/firecrawl/old-manifests/redis-deployment.yaml similarity index 100% rename from cluster/applications/firecrawl/redis-deployment.yaml rename to cluster/applications/firecrawl/old-manifests/redis-deployment.yaml diff --git a/cluster/applications/firecrawl/redis-service.yaml b/cluster/applications/firecrawl/old-manifests/redis-service.yaml similarity index 100% rename from cluster/applications/firecrawl/redis-service.yaml rename to cluster/applications/firecrawl/old-manifests/redis-service.yaml diff --git a/cluster/applications/firecrawl/worker-deployment.yaml b/cluster/applications/firecrawl/old-manifests/worker-deployment.yaml similarity index 100% rename from cluster/applications/firecrawl/worker-deployment.yaml rename to cluster/applications/firecrawl/old-manifests/worker-deployment.yaml