From 52e97f3a7c82cde61c15fd4bfefbe37b5ce1ef05 Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Thu, 4 Jun 2026 19:10:47 -0500 Subject: [PATCH] chore: Remove Firecrawl deployment - pausing until platform infra is ready Removing Firecrawl ArgoCD application and all manifests. The deployment was failing due to missing container images that need to be built from source. This requires platform infrastructure we don't have yet. Will return to Firecrawl deployment after Harbor registry and Tekton pipelines are deployed and configured. Note: ArgoCD also needs a thematic EPCOT name at some point. --- cluster/applications/firecrawl/MIGRATION.md | 97 ------ cluster/applications/firecrawl/README.md | 125 ------- .../firecrawl/REFACTOR-SUMMARY.md | 267 --------------- .../applications/firecrawl/application.yaml | 55 ---- .../applications/firecrawl/chart/Chart.yaml | 8 - .../applications/firecrawl/chart/README.md | 309 ------------------ .../firecrawl/chart/templates/_helpers.tpl | 49 --- .../firecrawl/chart/templates/api.yaml | 81 ----- .../firecrawl/chart/templates/configmap.yaml | 89 ----- .../firecrawl/chart/templates/httproute.yaml | 87 ----- .../firecrawl/chart/templates/namespace.yaml | 6 - .../firecrawl/chart/templates/playwright.yaml | 58 ---- .../firecrawl/chart/templates/postgres.yaml | 67 ---- .../firecrawl/chart/templates/pvc.yaml | 53 --- .../firecrawl/chart/templates/rabbitmq.yaml | 82 ----- .../firecrawl/chart/templates/redis.yaml | 73 ----- .../firecrawl/chart/templates/worker.yaml | 39 --- .../applications/firecrawl/chart/values.yaml | 240 -------------- .../old-manifests/api-deployment.yaml | 67 ---- .../firecrawl/old-manifests/api-service.yaml | 19 -- .../firecrawl/old-manifests/configmap.yaml | 60 ---- .../firecrawl/old-manifests/httproute.yaml | 87 ----- .../firecrawl/old-manifests/namespace.yaml | 12 - .../old-manifests/playwright-configmap.yaml | 13 - .../old-manifests/playwright-deployment.yaml | 44 --- .../old-manifests/playwright-service.yaml | 17 - .../old-manifests/postgres-deployment.yaml | 47 --- .../old-manifests/postgres-service.yaml | 17 - .../old-manifests/rabbitmq-deployment.yaml | 51 --- .../old-manifests/rabbitmq-service.yaml | 21 -- .../old-manifests/redis-deployment.yaml | 35 -- .../old-manifests/redis-service.yaml | 17 - .../old-manifests/worker-deployment.yaml | 44 --- 33 files changed, 2336 deletions(-) delete mode 100644 cluster/applications/firecrawl/MIGRATION.md delete mode 100644 cluster/applications/firecrawl/README.md delete mode 100644 cluster/applications/firecrawl/REFACTOR-SUMMARY.md delete mode 100644 cluster/applications/firecrawl/application.yaml delete mode 100644 cluster/applications/firecrawl/chart/Chart.yaml delete mode 100644 cluster/applications/firecrawl/chart/README.md delete mode 100644 cluster/applications/firecrawl/chart/templates/_helpers.tpl delete mode 100644 cluster/applications/firecrawl/chart/templates/api.yaml delete mode 100644 cluster/applications/firecrawl/chart/templates/configmap.yaml delete mode 100644 cluster/applications/firecrawl/chart/templates/httproute.yaml delete mode 100644 cluster/applications/firecrawl/chart/templates/namespace.yaml delete mode 100644 cluster/applications/firecrawl/chart/templates/playwright.yaml delete mode 100644 cluster/applications/firecrawl/chart/templates/postgres.yaml delete mode 100644 cluster/applications/firecrawl/chart/templates/pvc.yaml delete mode 100644 cluster/applications/firecrawl/chart/templates/rabbitmq.yaml delete mode 100644 cluster/applications/firecrawl/chart/templates/redis.yaml delete mode 100644 cluster/applications/firecrawl/chart/templates/worker.yaml delete mode 100644 cluster/applications/firecrawl/chart/values.yaml delete mode 100644 cluster/applications/firecrawl/old-manifests/api-deployment.yaml delete mode 100644 cluster/applications/firecrawl/old-manifests/api-service.yaml delete mode 100644 cluster/applications/firecrawl/old-manifests/configmap.yaml delete mode 100644 cluster/applications/firecrawl/old-manifests/httproute.yaml delete mode 100644 cluster/applications/firecrawl/old-manifests/namespace.yaml delete mode 100644 cluster/applications/firecrawl/old-manifests/playwright-configmap.yaml delete mode 100644 cluster/applications/firecrawl/old-manifests/playwright-deployment.yaml delete mode 100644 cluster/applications/firecrawl/old-manifests/playwright-service.yaml delete mode 100644 cluster/applications/firecrawl/old-manifests/postgres-deployment.yaml delete mode 100644 cluster/applications/firecrawl/old-manifests/postgres-service.yaml delete mode 100644 cluster/applications/firecrawl/old-manifests/rabbitmq-deployment.yaml delete mode 100644 cluster/applications/firecrawl/old-manifests/rabbitmq-service.yaml delete mode 100644 cluster/applications/firecrawl/old-manifests/redis-deployment.yaml delete mode 100644 cluster/applications/firecrawl/old-manifests/redis-service.yaml delete mode 100644 cluster/applications/firecrawl/old-manifests/worker-deployment.yaml diff --git a/cluster/applications/firecrawl/MIGRATION.md b/cluster/applications/firecrawl/MIGRATION.md deleted file mode 100644 index 65a1b39..0000000 --- a/cluster/applications/firecrawl/MIGRATION.md +++ /dev/null @@ -1,97 +0,0 @@ -# Migration to Helm Chart - -**Date**: 2026-06-04 -**Reason**: Production-ready refactoring - -## Critical Issues Fixed - -### 1. ❌ No Persistent Storage → ✅ PersistentVolumeClaims -**Old**: PostgreSQL and Redis used `emptyDir` volumes -- All data lost on pod restart -- Not production-ready - -**New**: All stateful services have PVCs -- PostgreSQL: 20Gi PVC on nfs-emporium -- Redis: 10Gi PVC on nfs-emporium -- RabbitMQ: 5Gi PVC on nfs-emporium - -### 2. ❌ Using `latest` Tags → ✅ Pinned Versions -**Old**: Images used `latest` or unspecified tags -- Unpredictable updates -- Hard to rollback -- No version control - -**New**: All images pinned to specific versions -- Firecrawl: `v1.0.0` -- Redis: `7.4.1-alpine` -- RabbitMQ: `3.13.7-management-alpine` -- PostgreSQL: `v1.0.0` -- Playwright: `v1.0.0` - -### 3. ❌ Raw Manifests → ✅ Helm Chart -**Old**: Individual YAML files -- Hard to maintain -- No templating -- Difficult to customize - -**New**: Proper Helm chart structure -- Template-based configuration -- Easy value overrides -- Industry standard deployment - -## Migration Steps - -1. **Backup** (if needed): Old deployment had ephemeral storage, no data to preserve -2. **ArgoCD Auto-Sync**: Push changes and ArgoCD will: - - Delete old resources - - Create new Helm release - - Provision PVCs - - Deploy with pinned versions - -## Old Files Location - -All previous manifests are archived in `old-manifests/` directory for reference. - -## Rollback Procedure - -If needed to rollback: -```bash -# Revert git commit -git revert HEAD - -# ArgoCD will automatically sync back to old state -# Or manually: -argocd app sync firecrawl -``` - -## Verification - -After deployment: -```bash -# Check all PVCs are bound -kubectl -n firecrawl get pvc - -# Verify pods are running -kubectl -n firecrawl get pods - -# Check image versions -kubectl -n firecrawl get pods -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.containers[*].image}{"\n"}{end}' - -# Test API -curl https://spaceship-earth.local.mk-labs.cloud/v0/health/liveness -``` - -## Benefits - -✅ **Data Persistence**: No more data loss on restarts -✅ **Version Control**: Explicit version management -✅ **Configuration Management**: Clean Helm values -✅ **Maintainability**: Standard Helm patterns -✅ **Production Ready**: Follows Kubernetes best practices - -## Notes - -- Storage class `nfs-emporium` provides persistent NFS-backed storage -- All resources properly labeled with Helm chart metadata -- Gateway/HTTPRoute configurations preserved -- EPCOT theming (spaceship-earth) maintained diff --git a/cluster/applications/firecrawl/README.md b/cluster/applications/firecrawl/README.md deleted file mode 100644 index 5d78fd1..0000000 --- a/cluster/applications/firecrawl/README.md +++ /dev/null @@ -1,125 +0,0 @@ -# Firecrawl (Spaceship Earth) - -Web scraping and search service for JARVIS's web search capabilities. - -## Service Names - -- **Primary (EPCOT):** `spaceship-earth.local.mk-labs.cloud` -- **Secondary:** `firecrawl.local.mk-labs.cloud` - -## Architecture - -### Components - -1. **Firecrawl API** - Main REST API service for web scraping - - Port: 3002 - - Health checks: `/v0/health/liveness`, `/v0/health/readiness` - - Resources: 4Gi memory, 2 CPU cores (6Gi limit) - -2. **Firecrawl Worker** - Background job processor - - Processes crawl jobs from queue - - Resources: 3Gi memory, 1 CPU core (4Gi limit) - -3. **Playwright Service** - Browser automation - - Port: 3000 - - Headless Chrome/Firefox for JavaScript rendering - - Resources: 2Gi memory, 1 CPU core (4Gi limit) - -4. **Redis** - Cache and job queue - - Port: 6379 - - Resources: 256Mi memory, 100m CPU (512Mi limit) - -5. **PostgreSQL (nuq-postgres)** - State management - - Port: 5432 - - Custom image with queue extensions - - Resources: 512Mi memory, 250m CPU (1Gi limit) - - **Note:** Currently using emptyDir (ephemeral). Add PVC for persistence if needed. - -6. **RabbitMQ** - Message queue for distributed processing - - AMQP Port: 5672 - - Management UI: 15672 - - Resources: 512Mi memory, 250m CPU (1Gi limit) - -### Configuration - -Key environment variables in `configmap.yaml`: - -- `USE_DB_AUTHENTICATION=false` - Simplified auth for internal use -- `NUM_WORKERS_PER_QUEUE=8` - Worker concurrency -- `CRAWL_CONCURRENT_REQUESTS=10` - Parallel crawl requests -- `MAX_CONCURRENT_JOBS=5` - Maximum simultaneous jobs - -## Access - -- **HTTPS (Gateway API):** https://spaceship-earth.local.mk-labs.cloud -- **HTTPS (Gateway API):** https://firecrawl.local.mk-labs.cloud -- Internal: `http://firecrawl-api.firecrawl.svc.cluster.local:3002` - -## DNS - -ExternalDNS automatically configures DNS records pointing to the Gateway load balancer (10.1.71.90). - -## TLS - -Certificates are automatically provisioned via cert-manager using the `letsencrypt-prod` ClusterIssuer. - -## Deployment - -Managed by ArgoCD (wave 20): - -```bash -# Check application status -kubectl -n argocd get application firecrawl - -# View pods -kubectl -n firecrawl get pods - -# Check API logs -kubectl -n firecrawl logs -l app=firecrawl-api -f - -# Check worker logs -kubectl -n firecrawl logs -l app=firecrawl-worker -f -``` - -## API Usage - -Example: Scrape a webpage - -```bash -curl -X POST https://spaceship-earth.local.mk-labs.cloud/v0/scrape \ - -H "Content-Type: application/json" \ - -d '{"url": "https://example.com"}' -``` - -## Troubleshooting - -### Pods not starting - -Check resource availability: -```bash -kubectl -n firecrawl describe pod -``` - -### PostgreSQL data persistence - -Currently using `emptyDir` for simplicity. To add persistence: - -1. Create a PVC in `postgres-pvc.yaml` -2. Update `postgres-deployment.yaml` to use the PVC -3. Commit and push - -### Gateway not routing - -Verify ReferenceGrant allows firecrawl namespace: -```bash -kubectl -n gateway get referencegrant allow-httproutes -o yaml -``` - -## Future Enhancements - -- [ ] Add persistent volume for PostgreSQL -- [ ] Add extract worker deployment (data extraction) -- [ ] Add nuq worker deployment (advanced queue processing) -- [ ] Configure monitoring/metrics -- [ ] Add resource quotas and limits -- [ ] Integrate with JARVIS via API key authentication diff --git a/cluster/applications/firecrawl/REFACTOR-SUMMARY.md b/cluster/applications/firecrawl/REFACTOR-SUMMARY.md deleted file mode 100644 index 3416294..0000000 --- a/cluster/applications/firecrawl/REFACTOR-SUMMARY.md +++ /dev/null @@ -1,267 +0,0 @@ -# Firecrawl Production Refactoring - Summary - -## Mission Status: ✅ COMPLETE - -All critical issues identified by Ryan have been resolved. The Firecrawl deployment is now production-ready. - ---- - -## Critical Fixes Implemented - -### 1. ✅ Persistent Storage Added -**Problem**: Redis and PostgreSQL used `emptyDir` volumes - all data lost on pod restart. - -**Solution**: PersistentVolumeClaims for all stateful services -- **PostgreSQL**: 20Gi PVC on `nfs-emporium` storage class - - Location: `/volume1/fastpass/firecrawl/firecrawl-postgres-pvc/` - - Stores: Database state and queue management - -- **Redis**: 10Gi PVC on `nfs-emporium` storage class - - Location: `/volume1/fastpass/firecrawl/firecrawl-redis-pvc/` - - Stores: Cache and job queue data - - Config: AOF persistence enabled with snapshots - -- **RabbitMQ**: 5Gi PVC on `nfs-emporium` storage class - - Location: `/volume1/fastpass/firecrawl/firecrawl-rabbitmq-pvc/` - - Stores: Message queue state - -### 2. ✅ Image Versions Pinned -**Problem**: All images used `latest` tags - unpredictable updates, hard to rollback. - -**Solution**: Explicit version tags for all images -| Component | Repository | Tag | -|-----------|-----------|-----| -| Firecrawl API | `ghcr.io/mendableai/firecrawl` | `v1.0.0` | -| Firecrawl Worker | `ghcr.io/mendableai/firecrawl` | `v1.0.0` | -| Playwright Service | `ghcr.io/mendableai/firecrawl/playwright-service` | `v1.0.0` | -| PostgreSQL | `ghcr.io/mendableai/firecrawl/nuq-postgres` | `v1.0.0` | -| Redis | `redis` | `7.4.1-alpine` | -| RabbitMQ | `rabbitmq` | `3.13.7-management-alpine` | - -### 3. ✅ Converted to Helm Chart -**Problem**: Raw YAML manifests - hard to maintain, no templating. - -**Solution**: Production-ready Helm chart structure -``` -chart/ -├── Chart.yaml # Chart metadata -├── values.yaml # Configuration values (200+ lines) -├── README.md # Complete documentation -└── templates/ - ├── _helpers.tpl # Template helpers - ├── namespace.yaml # Namespace definition - ├── configmap.yaml # Configuration - ├── pvc.yaml # PersistentVolumeClaims - ├── postgres.yaml # PostgreSQL deployment + service - ├── redis.yaml # Redis deployment + service - ├── rabbitmq.yaml # RabbitMQ deployment + service - ├── playwright.yaml # Playwright deployment + service - ├── api.yaml # API deployment + service - ├── worker.yaml # Worker deployment - └── httproute.yaml # Gateway/HTTPRoute configs -``` - ---- - -## Research Findings - -### Firecrawl Official Sources -- **GitHub**: https://github.com/mendableai/firecrawl -- **Current Version**: 1.0.0 (from package.json) -- **Official Helm Chart**: Found at `/examples/kubernetes/firecrawl-helm/` - - Used as reference for structure - - Adapted for mk-labs infrastructure - -### Storage Configuration -- **Storage Class**: `nfs-emporium` (existing in cluster) -- **Provisioner**: `nfs.csi.k8s.io` -- **Backend**: Synology DS1621+ (emporium) - 10.1.71.9 -- **NFS Export**: `/volume1/fastpass` -- **Reclaim Policy**: Delete -- **Binding Mode**: Immediate - -### Image Version Research -- Checked Firecrawl repository for version tags -- Used official docker-compose.yaml as reference -- Selected stable Alpine-based images for Redis/RabbitMQ -- Pinned to Firecrawl v1.0.0 based on package.json - -### Patterns from Other Applications -Reviewed monitoring stack for PVC patterns: -- Prometheus uses volumeClaimTemplate in StatefulSet -- Alertmanager uses similar pattern -- Applied same storage class (`nfs-emporium`) -- Used comparable sizing for workload types - ---- - -## Files Created - -### Helm Chart Structure -1. **chart/Chart.yaml** - Chart metadata (v1.0.0) -2. **chart/values.yaml** - Complete configuration (6KB) -3. **chart/README.md** - Comprehensive documentation (8KB) -4. **chart/templates/_helpers.tpl** - Template functions -5. **chart/templates/namespace.yaml** - Namespace with labels -6. **chart/templates/configmap.yaml** - App and Playwright config -7. **chart/templates/pvc.yaml** - All three PVCs -8. **chart/templates/postgres.yaml** - PostgreSQL deployment + service -9. **chart/templates/redis.yaml** - Redis deployment + service -10. **chart/templates/rabbitmq.yaml** - RabbitMQ deployment + service -11. **chart/templates/playwright.yaml** - Playwright deployment + service -12. **chart/templates/api.yaml** - API deployment + service -13. **chart/templates/worker.yaml** - Worker deployment -14. **chart/templates/httproute.yaml** - Gateway routing + TLS certs - -### Documentation -15. **MIGRATION.md** - Migration guide with rollback procedures -16. **README.md** - Updated with Helm chart references - -### Modified -17. **application.yaml** - Updated ArgoCD app to use Helm chart source - -### Archived -18. **old-manifests/** - All 15 original YAML files preserved - ---- - -## Deployment Architecture - -### Persistent Storage Breakdown -``` -Total Storage Allocated: 35Gi - -PostgreSQL: 20Gi (57%) ━━━━━━━━━━━━━━━━━━━━━ -Redis: 10Gi (29%) ━━━━━━━━━━━ -RabbitMQ: 5Gi (14%) ━━━━━━ -``` - -### Resource Allocation -``` -Component CPU Request Memory Request ------------------------------------------------- -API 2000m 4Gi -Worker 1000m 3Gi -Playwright 1000m 2Gi -PostgreSQL 250m 512Mi -Redis 100m 256Mi -RabbitMQ 250m 512Mi ------------------------------------------------- -TOTAL 4600m 10.25Gi -``` - ---- - -## Verification Commands - -```bash -# Check ArgoCD sync status -kubectl -n argocd get application firecrawl - -# Verify all PVCs are bound -kubectl -n firecrawl get pvc - -# Check all pods are running -kubectl -n firecrawl get pods - -# Verify image versions (no 'latest' tags) -kubectl -n firecrawl get pods -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.containers[*].image}{"\n"}{end}' - -# Test API health -curl https://spaceship-earth.local.mk-labs.cloud/v0/health/liveness - -# Check storage usage -kubectl -n firecrawl exec deployment/firecrawl-postgres -- df -h /var/lib/postgresql/data -kubectl -n firecrawl exec deployment/firecrawl-redis -- df -h /data -``` - ---- - -## Git Commit Details - -**Commit**: 6bcb6fa -**Branch**: main -**Status**: ✅ Pushed to origin - -**Changes**: -- 31 files changed -- 1354 insertions (+) -- 7 deletions (-) -- 15 files moved to old-manifests/ -- 16 new files created - ---- - -## ArgoCD Sync Behavior - -When ArgoCD detects these changes: - -1. **Prune Phase**: Old resources deleted - - Old deployments (without PVCs) - - Old services - - Old configmaps - -2. **Create Phase**: New resources created - - Namespace (with updated labels) - - PVCs provisioned via NFS CSI - - New deployments (with PVC mounts) - - New services (with Helm naming) - -3. **Sync Phase**: HTTPRoutes updated - - TLS certificates renewed if needed - - Gateway routes point to new service names - -**Data Impact**: ⚠️ Old PostgreSQL/Redis data will be lost (was ephemeral anyway). Fresh start with persistent storage. - ---- - -## Production Readiness Checklist - -✅ **Persistent Storage**: All stateful services have PVCs -✅ **Version Pinning**: No `latest` tags anywhere -✅ **Configuration Management**: Helm chart with proper values -✅ **Resource Limits**: All pods have requests and limits -✅ **Health Checks**: Liveness and readiness probes configured -✅ **Documentation**: Complete README and migration guide -✅ **Backup Strategy**: NFS backend supports Synology snapshots -✅ **Monitoring Ready**: Labels for Prometheus ServiceMonitor -✅ **Network Policy Ready**: Proper component labels -✅ **GitOps**: Fully managed by ArgoCD - ---- - -## Next Steps (Optional Enhancements) - -1. **Monitoring**: Add Prometheus ServiceMonitor and dashboards -2. **Alerting**: Configure alerts for storage, memory, and API errors -3. **Secrets Management**: Integrate with Vault or Sealed Secrets -4. **High Availability**: Scale API replicas, add PodDisruptionBudgets -5. **Network Policies**: Restrict inter-pod communication -6. **Resource Quotas**: Add namespace quotas and limit ranges -7. **Autoscaling**: Implement HPA for API and Worker -8. **Additional Workers**: Deploy extract and nuq workers - ---- - -## References - -- **Firecrawl Docs**: https://docs.firecrawl.dev -- **Firecrawl GitHub**: https://github.com/mendableai/firecrawl -- **Official Helm Chart**: /examples/kubernetes/firecrawl-helm/ -- **Monitoring Stack**: cluster/applications/monitoring/ (for PVC patterns) -- **Storage Class**: cluster/platform/nfs-csi/storageclass.yaml -- **Gateway Config**: cluster/platform/gateway/ - ---- - -## Contact - -For issues or questions about this deployment: -- Check ArgoCD application status -- Review pod logs: `kubectl -n firecrawl logs -l app.kubernetes.io/name=firecrawl` -- Verify storage: `kubectl -n firecrawl get pvc` -- Consult chart README: `cluster/applications/firecrawl/chart/README.md` - ---- - -**Mission Accomplished**: Firecrawl is now production-ready with persistent storage, pinned versions, and proper Helm chart management. Ryan's concerns have been fully addressed. 🚀 diff --git a/cluster/applications/firecrawl/application.yaml b/cluster/applications/firecrawl/application.yaml deleted file mode 100644 index c8f4f4d..0000000 --- a/cluster/applications/firecrawl/application.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# ------------------------------------------------------------------------------ -# Application: firecrawl (spaceship-earth) -# Wave 20 — applications tier -# Web scraping and search service for JARVIS -# -# Primary DNS: spaceship-earth.local.mk-labs.cloud (EPCOT theme) -# Secondary DNS: firecrawl.local.mk-labs.cloud -# -# Components: -# - Firecrawl API (main service) -# - Firecrawl Worker (background job processor) -# - Playwright Service (browser automation) -# - Redis (cache and job queue) - WITH PERSISTENT STORAGE -# - PostgreSQL (state management) - WITH PERSISTENT STORAGE -# - RabbitMQ (message queue) - WITH PERSISTENT STORAGE -# -# PRODUCTION-READY FEATURES: -# - All stateful services use PersistentVolumeClaims -# - All images pinned to specific versions -# - Helm chart for configuration management -# - Resource limits and requests properly configured -# ------------------------------------------------------------------------------ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: firecrawl - namespace: argocd - labels: - app.kubernetes.io/name: firecrawl - app.kubernetes.io/part-of: mk-labs - epcot.theme/name: spaceship-earth - annotations: - argocd.argoproj.io/sync-wave: "20" # Wave 20 — applications tier -spec: - project: default - source: - repoURL: https://gitea.mk-labs.cloud/rblundon/homelab.git - targetRevision: main - path: cluster/applications/firecrawl/chart - helm: - releaseName: firecrawl - values: | - # Production configuration - # Storage classes use nfs-emporium (Synology NFS) - # All images pinned to specific versions for stability - destination: - server: https://kubernetes.default.svc - namespace: firecrawl - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true - - ServerSideApply=true diff --git a/cluster/applications/firecrawl/chart/Chart.yaml b/cluster/applications/firecrawl/chart/Chart.yaml deleted file mode 100644 index 93fd0dd..0000000 --- a/cluster/applications/firecrawl/chart/Chart.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v2 -name: firecrawl -description: A Helm chart for deploying Firecrawl web scraping service -type: application -version: 1.0.0 -appVersion: "1.0.0" -maintainers: - - name: mk-labs diff --git a/cluster/applications/firecrawl/chart/README.md b/cluster/applications/firecrawl/chart/README.md deleted file mode 100644 index 740dd0d..0000000 --- a/cluster/applications/firecrawl/chart/README.md +++ /dev/null @@ -1,309 +0,0 @@ -# Firecrawl Helm Chart - -Production-ready Helm chart for deploying Firecrawl web scraping service. - -## Overview - -Firecrawl is a web scraping and search service that provides JARVIS with web context capabilities. This deployment includes all necessary components with proper persistent storage and pinned image versions. - -## Components - -### Application Services -- **Firecrawl API** - Main REST API service - - Port: 3002 - - Health checks: `/v0/health/liveness`, `/v0/health/readiness` - - Resources: 4Gi memory, 2 CPU cores - -- **Firecrawl Worker** - Background job processor - - Processes crawl jobs from queue - - Resources: 3Gi memory, 1 CPU core - -- **Playwright Service** - Browser automation - - Port: 3000 - - Headless Chrome/Firefox for JavaScript rendering - - Resources: 2Gi memory, 1 CPU core - -### Infrastructure Services -- **PostgreSQL** (nuq-postgres) - State management - - Port: 5432 - - Custom image with queue extensions - - **Persistent Storage**: 20Gi PVC on nfs-emporium - -- **Redis** - Cache and job queue - - Port: 6379 - - AOF persistence enabled - - **Persistent Storage**: 10Gi PVC on nfs-emporium - -- **RabbitMQ** - Message queue - - AMQP Port: 5672 - - Management UI: 15672 - - **Persistent Storage**: 5Gi PVC on nfs-emporium - -## Production Features - -### ✅ Persistent Storage -All stateful services use PersistentVolumeClaims backed by NFS storage: -- PostgreSQL: 20Gi (database state) -- Redis: 10Gi (cache and job queue) -- RabbitMQ: 5Gi (message queue data) - -### ✅ Pinned Image Versions -All container images use specific version tags (no `latest`): -- Firecrawl API/Worker: `ghcr.io/mendableai/firecrawl:v1.0.0` -- Playwright Service: `ghcr.io/mendableai/firecrawl/playwright-service:v1.0.0` -- PostgreSQL: `ghcr.io/mendableai/firecrawl/nuq-postgres:v1.0.0` -- Redis: `redis:7.4.1-alpine` -- RabbitMQ: `rabbitmq:3.13.7-management-alpine` - -### ✅ Helm Configuration Management -Proper Helm chart structure with: -- Configurable values via `values.yaml` -- Template helpers for consistent naming -- Resource requests and limits -- Health checks and probes -- Service discovery - -## Configuration - -### Key Values - -```yaml -# Image versions (pinned) -image: - repository: ghcr.io/mendableai/firecrawl - tag: "v1.0.0" - -# Persistent storage -postgres.persistence: - enabled: true - storageClass: "nfs-emporium" - size: 20Gi - -redis.persistence: - enabled: true - storageClass: "nfs-emporium" - size: 10Gi - -rabbitmq.persistence: - enabled: true - storageClass: "nfs-emporium" - size: 5Gi - -# Performance tuning -config: - NUM_WORKERS_PER_QUEUE: "8" - CRAWL_CONCURRENT_REQUESTS: "10" - MAX_CONCURRENT_JOBS: "5" -``` - -See `values.yaml` for all configuration options. - -## Deployment - -### ArgoCD (Recommended) -This application is managed by ArgoCD in wave 20 (applications tier): - -```bash -# Check application status -kubectl -n argocd get application firecrawl - -# Sync manually if needed -argocd app sync firecrawl - -# View application details -argocd app get firecrawl -``` - -### Manual Helm Deployment -```bash -# From repository root -cd cluster/applications/firecrawl - -# Install -helm install firecrawl ./chart -n firecrawl --create-namespace - -# Upgrade -helm upgrade firecrawl ./chart -n firecrawl - -# Uninstall -helm uninstall firecrawl -n firecrawl -``` - -## Access - -### External Access (via Gateway API) -- **Primary**: https://spaceship-earth.local.mk-labs.cloud -- **Secondary**: https://firecrawl.local.mk-labs.cloud - -DNS automatically configured via ExternalDNS pointing to Gateway LB (10.1.71.90). - -### Internal Access -``` -http://firecrawl-api.firecrawl.svc.cluster.local:3002 -``` - -### API Usage Example -```bash -curl -X POST https://spaceship-earth.local.mk-labs.cloud/v0/scrape \ - -H "Content-Type: application/json" \ - -d '{"url": "https://example.com"}' -``` - -## Monitoring - -### View Pods -```bash -kubectl -n firecrawl get pods -``` - -### Check Logs -```bash -# API logs -kubectl -n firecrawl logs -l app.kubernetes.io/component=api -f - -# Worker logs -kubectl -n firecrawl logs -l app.kubernetes.io/component=worker -f - -# PostgreSQL logs -kubectl -n firecrawl logs -l app.kubernetes.io/component=postgres -f - -# Redis logs -kubectl -n firecrawl logs -l app.kubernetes.io/component=redis -f -``` - -### Check Storage -```bash -# View PVCs -kubectl -n firecrawl get pvc - -# Check PVC details -kubectl -n firecrawl describe pvc firecrawl-postgres-pvc -kubectl -n firecrawl describe pvc firecrawl-redis-pvc -kubectl -n firecrawl describe pvc firecrawl-rabbitmq-pvc -``` - -## Troubleshooting - -### Pods Not Starting -```bash -kubectl -n firecrawl describe pod -kubectl -n firecrawl get events --sort-by='.lastTimestamp' -``` - -### Storage Issues -```bash -# Check PVC binding -kubectl -n firecrawl get pvc - -# Verify NFS storage class -kubectl get storageclass nfs-emporium -o yaml - -# Check NFS CSI driver -kubectl -n kube-system get pods -l app=csi-nfs-controller -``` - -### Database Connection Issues -```bash -# Test PostgreSQL connectivity from API pod -kubectl -n firecrawl exec -it deployment/firecrawl-api -- \ - sh -c 'apt-get update && apt-get install -y postgresql-client && \ - psql -h firecrawl-postgres -U postgres -d postgres -c "SELECT 1"' - -# Test Redis connectivity -kubectl -n firecrawl exec -it deployment/firecrawl-api -- \ - sh -c 'apt-get update && apt-get install -y redis-tools && \ - redis-cli -h firecrawl-redis PING' -``` - -### Gateway Routing Issues -```bash -# Verify ReferenceGrant -kubectl -n gateway get referencegrant allow-httproutes -o yaml - -# Check HTTPRoutes -kubectl -n firecrawl get httproute - -# View Gateway status -kubectl -n gateway get gateway fastpass-gateway -o yaml -``` - -## Upgrading - -### Update Image Versions -1. Edit `values.yaml` and update image tags -2. Commit and push changes -3. ArgoCD will automatically sync -4. Or manually: `helm upgrade firecrawl ./chart -n firecrawl` - -### Migrate from Old Deployment -The old raw manifest deployment will be automatically replaced by ArgoCD: -1. Commit this Helm chart -2. Push to repository -3. ArgoCD detects the change and applies Helm chart -4. Old resources are pruned -5. New resources with PVCs are created - -**Note**: Old PostgreSQL/Redis data was ephemeral and will be lost. Fresh start with persistent storage. - -## Storage Architecture - -### NFS Provisioner -- **Storage Class**: `nfs-emporium` -- **Provisioner**: `nfs.csi.k8s.io` -- **NFS Server**: emporium (Synology DS1621+) - 10.1.71.9 -- **NFS Export**: `/volume1/fastpass` -- **Reclaim Policy**: Delete -- **Binding Mode**: Immediate - -### PVC Structure -Each PVC gets a unique subdirectory: -``` -/volume1/fastpass/firecrawl/firecrawl-postgres-pvc/ -/volume1/fastpass/firecrawl/firecrawl-redis-pvc/ -/volume1/fastpass/firecrawl/firecrawl-rabbitmq-pvc/ -``` - -## Theme - -Part of the EPCOT-themed namespace with primary hostname **spaceship-earth** representing Firecrawl's role as the information and knowledge hub for JARVIS. - -## Maintenance - -### Backup Recommendations -```bash -# PostgreSQL backup (manual) -kubectl -n firecrawl exec deployment/firecrawl-postgres -- \ - pg_dump -U postgres postgres > firecrawl-backup-$(date +%Y%m%d).sql - -# NFS snapshots (via Synology) -# Use Synology Snapshot Replication for point-in-time backups -``` - -### Scaling -```bash -# Scale API replicas -helm upgrade firecrawl ./chart -n firecrawl \ - --set api.replicaCount=2 - -# Scale workers -helm upgrade firecrawl ./chart -n firecrawl \ - --set worker.replicaCount=3 -``` - -## Future Enhancements -- [ ] Add Prometheus metrics and ServiceMonitor -- [ ] Implement resource quotas and LimitRanges -- [ ] Add PodDisruptionBudgets for HA -- [ ] Configure NetworkPolicies for security -- [ ] Add external secrets management (HashiCorp Vault or Sealed Secrets) -- [ ] Implement horizontal pod autoscaling -- [ ] Add extract worker deployment -- [ ] Add nuq worker deployment - -## Support - -For issues or questions: -- Check ArgoCD application status -- Review pod logs -- Verify storage bindings -- Consult Firecrawl documentation: https://docs.firecrawl.dev diff --git a/cluster/applications/firecrawl/chart/templates/_helpers.tpl b/cluster/applications/firecrawl/chart/templates/_helpers.tpl deleted file mode 100644 index eafdc33..0000000 --- a/cluster/applications/firecrawl/chart/templates/_helpers.tpl +++ /dev/null @@ -1,49 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "firecrawl.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -*/}} -{{- define "firecrawl.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "firecrawl.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "firecrawl.labels" -}} -helm.sh/chart: {{ include "firecrawl.chart" . }} -{{ include "firecrawl.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "firecrawl.selectorLabels" -}} -app.kubernetes.io/name: {{ include "firecrawl.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} diff --git a/cluster/applications/firecrawl/chart/templates/api.yaml b/cluster/applications/firecrawl/chart/templates/api.yaml deleted file mode 100644 index 238d3cb..0000000 --- a/cluster/applications/firecrawl/chart/templates/api.yaml +++ /dev/null @@ -1,81 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "firecrawl.fullname" . }}-api - namespace: {{ .Values.namespace.name }} - labels: - {{- include "firecrawl.labels" . | nindent 4 }} - app.kubernetes.io/component: api - epcot.theme/service: spaceship-earth -spec: - replicas: {{ .Values.api.replicaCount }} - selector: - matchLabels: - {{- include "firecrawl.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: api - template: - metadata: - labels: - {{- include "firecrawl.selectorLabels" . | nindent 8 }} - app.kubernetes.io/component: api - epcot.theme/service: spaceship-earth - spec: - terminationGracePeriodSeconds: 180 - containers: - - name: api - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - command: ["node"] - args: ["--max-old-space-size=6144", "dist/src/index.js"] - envFrom: - - configMapRef: - name: {{ include "firecrawl.fullname" . }}-config - env: - - name: FLY_PROCESS_GROUP - value: "app" - - name: NUQ_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - ports: - - containerPort: 3002 - name: http - resources: - {{- toYaml .Values.api.resources | nindent 12 }} - livenessProbe: - httpGet: - path: /v0/health/liveness - port: 3002 - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /v0/health/readiness - port: 3002 - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 3 ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "firecrawl.fullname" . }}-api - namespace: {{ .Values.namespace.name }} - labels: - {{- include "firecrawl.labels" . | nindent 4 }} - app.kubernetes.io/component: api -spec: - type: {{ .Values.api.service.type }} - ports: - - port: {{ .Values.api.service.port }} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "firecrawl.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: api diff --git a/cluster/applications/firecrawl/chart/templates/configmap.yaml b/cluster/applications/firecrawl/chart/templates/configmap.yaml deleted file mode 100644 index 5ce3818..0000000 --- a/cluster/applications/firecrawl/chart/templates/configmap.yaml +++ /dev/null @@ -1,89 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "firecrawl.fullname" . }}-config - namespace: {{ .Values.namespace.name }} - labels: - {{- include "firecrawl.labels" . | nindent 4 }} -data: - # Environment - ENV: {{ .Values.config.ENV | quote }} - IS_KUBERNETES: {{ .Values.config.IS_KUBERNETES | quote }} - LOGGING_LEVEL: {{ .Values.config.LOGGING_LEVEL | quote }} - - # Service configuration - PORT: {{ .Values.config.PORT | quote }} - HOST: {{ .Values.config.HOST | quote }} - WORKER_PORT: {{ .Values.config.WORKER_PORT | quote }} - - # Performance tuning - NUM_WORKERS_PER_QUEUE: {{ .Values.config.NUM_WORKERS_PER_QUEUE | quote }} - CRAWL_CONCURRENT_REQUESTS: {{ .Values.config.CRAWL_CONCURRENT_REQUESTS | quote }} - MAX_CONCURRENT_JOBS: {{ .Values.config.MAX_CONCURRENT_JOBS | quote }} - - # Authentication - USE_DB_AUTHENTICATION: {{ .Values.config.USE_DB_AUTHENTICATION | quote }} - - # Service URLs - REDIS_URL: {{ .Values.config.REDIS_URL | default (printf "redis://%s-redis:%d" (include "firecrawl.fullname" .) (.Values.redis.service.port | int)) | quote }} - REDIS_RATE_LIMIT_URL: {{ .Values.config.REDIS_RATE_LIMIT_URL | default (printf "redis://%s-redis:%d" (include "firecrawl.fullname" .) (.Values.redis.service.port | int)) | quote }} - PLAYWRIGHT_MICROSERVICE_URL: {{ .Values.config.PLAYWRIGHT_MICROSERVICE_URL | default (printf "http://%s-playwright:%d/scrape" (include "firecrawl.fullname" .) (.Values.playwrightService.service.port | int)) | quote }} - - # Database configuration - POSTGRES_HOST: {{ printf "%s-postgres" (include "firecrawl.fullname" .) | quote }} - POSTGRES_PORT: {{ .Values.postgres.service.port | quote }} - POSTGRES_USER: {{ .Values.postgres.auth.username | quote }} - POSTGRES_PASSWORD: {{ .Values.postgres.auth.password | quote }} - POSTGRES_DB: {{ .Values.postgres.auth.database | quote }} - - # RabbitMQ configuration - NUQ_RABBITMQ_URL: {{ printf "amqp://%s-rabbitmq:%d" (include "firecrawl.fullname" .) (.Values.rabbitmq.service.amqpPort | int) | quote }} - - # Optional external services - {{- if .Values.config.OPENAI_BASE_URL }} - OPENAI_BASE_URL: {{ .Values.config.OPENAI_BASE_URL | quote }} - {{- end }} - {{- if .Values.config.MODEL_NAME }} - MODEL_NAME: {{ .Values.config.MODEL_NAME | quote }} - {{- end }} - {{- if .Values.config.MODEL_EMBEDDING_NAME }} - MODEL_EMBEDDING_NAME: {{ .Values.config.MODEL_EMBEDDING_NAME | quote }} - {{- end }} - {{- if .Values.config.OLLAMA_BASE_URL }} - OLLAMA_BASE_URL: {{ .Values.config.OLLAMA_BASE_URL | quote }} - {{- end }} - {{- if .Values.config.PROXY_SERVER }} - PROXY_SERVER: {{ .Values.config.PROXY_SERVER | quote }} - {{- end }} - {{- if .Values.config.PROXY_USERNAME }} - PROXY_USERNAME: {{ .Values.config.PROXY_USERNAME | quote }} - {{- end }} - {{- if .Values.config.SEARXNG_ENDPOINT }} - SEARXNG_ENDPOINT: {{ .Values.config.SEARXNG_ENDPOINT | quote }} - {{- end }} - {{- if .Values.config.SEARXNG_ENGINES }} - SEARXNG_ENGINES: {{ .Values.config.SEARXNG_ENGINES | quote }} - {{- end }} - {{- if .Values.config.SEARXNG_CATEGORIES }} - SEARXNG_CATEGORIES: {{ .Values.config.SEARXNG_CATEGORIES | quote }} - {{- end }} - {{- if .Values.config.SELF_HOSTED_WEBHOOK_URL }} - SELF_HOSTED_WEBHOOK_URL: {{ .Values.config.SELF_HOSTED_WEBHOOK_URL | quote }} - {{- end }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "firecrawl.fullname" . }}-playwright-config - namespace: {{ .Values.namespace.name }} - labels: - {{- include "firecrawl.labels" . | nindent 4 }} -data: - PORT: {{ .Values.playwrightConfig.PORT | quote }} - MAX_CONCURRENT_PAGES: {{ .Values.playwrightConfig.MAX_CONCURRENT_PAGES | quote }} - {{- if .Values.playwrightConfig.ALLOW_LOCAL_WEBHOOKS }} - ALLOW_LOCAL_WEBHOOKS: {{ .Values.playwrightConfig.ALLOW_LOCAL_WEBHOOKS | quote }} - {{- end }} - {{- if .Values.playwrightConfig.BLOCK_MEDIA }} - BLOCK_MEDIA: {{ .Values.playwrightConfig.BLOCK_MEDIA | quote }} - {{- end }} diff --git a/cluster/applications/firecrawl/chart/templates/httproute.yaml b/cluster/applications/firecrawl/chart/templates/httproute.yaml deleted file mode 100644 index 84e708a..0000000 --- a/cluster/applications/firecrawl/chart/templates/httproute.yaml +++ /dev/null @@ -1,87 +0,0 @@ -# ------------------------------------------------------------------------------ -# HTTPRoute — Firecrawl via Cilium Gateway -# Both primary (spaceship-earth) and secondary (firecrawl) DNS names -# ------------------------------------------------------------------------------ -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: spaceship-earth-tls - namespace: {{ .Values.namespace.name }} -spec: - secretName: spaceship-earth-tls - issuerRef: - name: letsencrypt-prod - kind: ClusterIssuer - dnsNames: - - spaceship-earth.local.mk-labs.cloud ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: firecrawl-tls - namespace: {{ .Values.namespace.name }} -spec: - secretName: firecrawl-tls - issuerRef: - name: letsencrypt-prod - kind: ClusterIssuer - dnsNames: - - firecrawl.local.mk-labs.cloud ---- -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: spaceship-earth - namespace: {{ .Values.namespace.name }} - annotations: - external-dns.alpha.kubernetes.io/hostname: spaceship-earth.local.mk-labs.cloud - external-dns.alpha.kubernetes.io/target: "10.1.71.90" -spec: - parentRefs: - - group: gateway.networking.k8s.io - kind: Gateway - name: fastpass-gateway - namespace: gateway - sectionName: https - hostnames: - - spaceship-earth.local.mk-labs.cloud - rules: - - matches: - - path: - type: PathPrefix - value: / - backendRefs: - - group: "" - kind: Service - name: {{ include "firecrawl.fullname" . }}-api - port: {{ .Values.api.service.port }} - weight: 1 ---- -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: firecrawl - namespace: {{ .Values.namespace.name }} - annotations: - external-dns.alpha.kubernetes.io/hostname: firecrawl.local.mk-labs.cloud - external-dns.alpha.kubernetes.io/target: "10.1.71.90" -spec: - parentRefs: - - group: gateway.networking.k8s.io - kind: Gateway - name: fastpass-gateway - namespace: gateway - sectionName: https - hostnames: - - firecrawl.local.mk-labs.cloud - rules: - - matches: - - path: - type: PathPrefix - value: / - backendRefs: - - group: "" - kind: Service - name: {{ include "firecrawl.fullname" . }}-api - port: {{ .Values.api.service.port }} - weight: 1 diff --git a/cluster/applications/firecrawl/chart/templates/namespace.yaml b/cluster/applications/firecrawl/chart/templates/namespace.yaml deleted file mode 100644 index fb54d7b..0000000 --- a/cluster/applications/firecrawl/chart/templates/namespace.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: {{ .Values.namespace.name }} - labels: - {{- toYaml .Values.namespace.labels | nindent 4 }} diff --git a/cluster/applications/firecrawl/chart/templates/playwright.yaml b/cluster/applications/firecrawl/chart/templates/playwright.yaml deleted file mode 100644 index 078bb00..0000000 --- a/cluster/applications/firecrawl/chart/templates/playwright.yaml +++ /dev/null @@ -1,58 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "firecrawl.fullname" . }}-playwright - namespace: {{ .Values.namespace.name }} - labels: - {{- include "firecrawl.labels" . | nindent 4 }} - app.kubernetes.io/component: playwright -spec: - replicas: {{ .Values.playwrightService.replicaCount }} - selector: - matchLabels: - {{- include "firecrawl.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: playwright - template: - metadata: - labels: - {{- include "firecrawl.selectorLabels" . | nindent 8 }} - app.kubernetes.io/component: playwright - spec: - containers: - - name: playwright - image: "{{ .Values.playwright.repository }}:{{ .Values.playwright.tag }}" - imagePullPolicy: {{ .Values.playwright.pullPolicy }} - envFrom: - - configMapRef: - name: {{ include "firecrawl.fullname" . }}-playwright-config - ports: - - containerPort: 3000 - name: http - resources: - {{- toYaml .Values.playwrightService.resources | nindent 12 }} - volumeMounts: - - name: cache - mountPath: /tmp/.cache - volumes: - - name: cache - emptyDir: - sizeLimit: 1Gi ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "firecrawl.fullname" . }}-playwright - namespace: {{ .Values.namespace.name }} - labels: - {{- include "firecrawl.labels" . | nindent 4 }} - app.kubernetes.io/component: playwright -spec: - type: {{ .Values.playwrightService.service.type }} - ports: - - port: {{ .Values.playwrightService.service.port }} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "firecrawl.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: playwright diff --git a/cluster/applications/firecrawl/chart/templates/postgres.yaml b/cluster/applications/firecrawl/chart/templates/postgres.yaml deleted file mode 100644 index 2c9c63e..0000000 --- a/cluster/applications/firecrawl/chart/templates/postgres.yaml +++ /dev/null @@ -1,67 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "firecrawl.fullname" . }}-postgres - namespace: {{ .Values.namespace.name }} - labels: - {{- include "firecrawl.labels" . | nindent 4 }} - app.kubernetes.io/component: postgres -spec: - replicas: {{ .Values.postgres.replicaCount }} - selector: - matchLabels: - {{- include "firecrawl.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: postgres - template: - metadata: - labels: - {{- include "firecrawl.selectorLabels" . | nindent 8 }} - app.kubernetes.io/component: postgres - spec: - containers: - - name: postgres - image: "{{ .Values.postgres.image.repository }}:{{ .Values.postgres.image.tag }}" - imagePullPolicy: {{ .Values.postgres.image.pullPolicy }} - env: - - name: POSTGRES_USER - value: {{ .Values.postgres.auth.username | quote }} - - name: POSTGRES_PASSWORD - value: {{ .Values.postgres.auth.password | quote }} - - name: POSTGRES_DB - value: {{ .Values.postgres.auth.database | quote }} - ports: - - containerPort: 5432 - name: postgres - resources: - {{- toYaml .Values.postgres.resources | nindent 12 }} - volumeMounts: - - name: postgres-data - mountPath: /var/lib/postgresql/data - subPath: postgres - volumes: - - name: postgres-data - {{- if .Values.postgres.persistence.enabled }} - persistentVolumeClaim: - claimName: {{ include "firecrawl.fullname" . }}-postgres-pvc - {{- else }} - emptyDir: {} - {{- end }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "firecrawl.fullname" . }}-postgres - namespace: {{ .Values.namespace.name }} - labels: - {{- include "firecrawl.labels" . | nindent 4 }} - app.kubernetes.io/component: postgres -spec: - type: {{ .Values.postgres.service.type }} - ports: - - port: {{ .Values.postgres.service.port }} - targetPort: postgres - protocol: TCP - name: postgres - selector: - {{- include "firecrawl.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: postgres diff --git a/cluster/applications/firecrawl/chart/templates/pvc.yaml b/cluster/applications/firecrawl/chart/templates/pvc.yaml deleted file mode 100644 index e0cc52a..0000000 --- a/cluster/applications/firecrawl/chart/templates/pvc.yaml +++ /dev/null @@ -1,53 +0,0 @@ -{{- if .Values.postgres.persistence.enabled }} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ include "firecrawl.fullname" . }}-postgres-pvc - namespace: {{ .Values.namespace.name }} - labels: - {{- include "firecrawl.labels" . | nindent 4 }} - app.kubernetes.io/component: postgres -spec: - accessModes: - - {{ .Values.postgres.persistence.accessMode }} - storageClassName: {{ .Values.postgres.persistence.storageClass | quote }} - resources: - requests: - storage: {{ .Values.postgres.persistence.size | quote }} -{{- end }} ---- -{{- if .Values.redis.persistence.enabled }} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ include "firecrawl.fullname" . }}-redis-pvc - namespace: {{ .Values.namespace.name }} - labels: - {{- include "firecrawl.labels" . | nindent 4 }} - app.kubernetes.io/component: redis -spec: - accessModes: - - {{ .Values.redis.persistence.accessMode }} - storageClassName: {{ .Values.redis.persistence.storageClass | quote }} - resources: - requests: - storage: {{ .Values.redis.persistence.size | quote }} -{{- end }} ---- -{{- if and .Values.rabbitmq.enabled .Values.rabbitmq.persistence.enabled }} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ include "firecrawl.fullname" . }}-rabbitmq-pvc - namespace: {{ .Values.namespace.name }} - labels: - {{- include "firecrawl.labels" . | nindent 4 }} - app.kubernetes.io/component: rabbitmq -spec: - accessModes: - - {{ .Values.rabbitmq.persistence.accessMode }} - storageClassName: {{ .Values.rabbitmq.persistence.storageClass | quote }} - resources: - requests: - storage: {{ .Values.rabbitmq.persistence.size | quote }} -{{- end }} diff --git a/cluster/applications/firecrawl/chart/templates/rabbitmq.yaml b/cluster/applications/firecrawl/chart/templates/rabbitmq.yaml deleted file mode 100644 index 5f2d8fe..0000000 --- a/cluster/applications/firecrawl/chart/templates/rabbitmq.yaml +++ /dev/null @@ -1,82 +0,0 @@ -{{- if .Values.rabbitmq.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "firecrawl.fullname" . }}-rabbitmq - namespace: {{ .Values.namespace.name }} - labels: - {{- include "firecrawl.labels" . | nindent 4 }} - app.kubernetes.io/component: rabbitmq -spec: - replicas: {{ .Values.rabbitmq.replicaCount }} - selector: - matchLabels: - {{- include "firecrawl.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: rabbitmq - template: - metadata: - labels: - {{- include "firecrawl.selectorLabels" . | nindent 8 }} - app.kubernetes.io/component: rabbitmq - spec: - containers: - - name: rabbitmq - image: "{{ .Values.rabbitmq.image.repository }}:{{ .Values.rabbitmq.image.tag }}" - imagePullPolicy: {{ .Values.rabbitmq.image.pullPolicy }} - command: ["rabbitmq-server"] - ports: - - containerPort: 5672 - name: amqp - - containerPort: 15672 - name: management - resources: - {{- toYaml .Values.rabbitmq.resources | nindent 12 }} - livenessProbe: - exec: - command: ["rabbitmq-diagnostics", "-q", "check_running"] - initialDelaySeconds: 5 - periodSeconds: 5 - timeoutSeconds: 5 - failureThreshold: 3 - readinessProbe: - exec: - command: ["rabbitmq-diagnostics", "-q", "check_running"] - initialDelaySeconds: 5 - periodSeconds: 5 - timeoutSeconds: 5 - failureThreshold: 3 - volumeMounts: - {{- if .Values.rabbitmq.persistence.enabled }} - - name: rabbitmq-data - mountPath: /var/lib/rabbitmq - {{- end }} - volumes: - {{- if .Values.rabbitmq.persistence.enabled }} - - name: rabbitmq-data - persistentVolumeClaim: - claimName: {{ include "firecrawl.fullname" . }}-rabbitmq-pvc - {{- end }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "firecrawl.fullname" . }}-rabbitmq - namespace: {{ .Values.namespace.name }} - labels: - {{- include "firecrawl.labels" . | nindent 4 }} - app.kubernetes.io/component: rabbitmq -spec: - type: {{ .Values.rabbitmq.service.type }} - ports: - - port: {{ .Values.rabbitmq.service.amqpPort }} - targetPort: amqp - protocol: TCP - name: amqp - - port: {{ .Values.rabbitmq.service.managementPort }} - targetPort: management - protocol: TCP - name: management - selector: - {{- include "firecrawl.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: rabbitmq -{{- end }} diff --git a/cluster/applications/firecrawl/chart/templates/redis.yaml b/cluster/applications/firecrawl/chart/templates/redis.yaml deleted file mode 100644 index f6c4066..0000000 --- a/cluster/applications/firecrawl/chart/templates/redis.yaml +++ /dev/null @@ -1,73 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "firecrawl.fullname" . }}-redis - namespace: {{ .Values.namespace.name }} - labels: - {{- include "firecrawl.labels" . | nindent 4 }} - app.kubernetes.io/component: redis -spec: - replicas: {{ .Values.redis.replicaCount }} - selector: - matchLabels: - {{- include "firecrawl.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: redis - template: - metadata: - labels: - {{- include "firecrawl.selectorLabels" . | nindent 8 }} - app.kubernetes.io/component: redis - spec: - containers: - - name: redis - image: "{{ .Values.redis.image.repository }}:{{ .Values.redis.image.tag }}" - imagePullPolicy: {{ .Values.redis.image.pullPolicy }} - command: ["redis-server"] - args: - - "--bind" - - "0.0.0.0" - {{- if .Values.redis.persistence.enabled }} - - "--appendonly" - - "yes" - - "--save" - - "900 1" - - "--save" - - "300 10" - - "--save" - - "60 10000" - {{- end }} - ports: - - containerPort: 6379 - name: redis - resources: - {{- toYaml .Values.redis.resources | nindent 12 }} - volumeMounts: - {{- if .Values.redis.persistence.enabled }} - - name: redis-data - mountPath: /data - {{- end }} - volumes: - {{- if .Values.redis.persistence.enabled }} - - name: redis-data - persistentVolumeClaim: - claimName: {{ include "firecrawl.fullname" . }}-redis-pvc - {{- end }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "firecrawl.fullname" . }}-redis - namespace: {{ .Values.namespace.name }} - labels: - {{- include "firecrawl.labels" . | nindent 4 }} - app.kubernetes.io/component: redis -spec: - type: {{ .Values.redis.service.type }} - ports: - - port: {{ .Values.redis.service.port }} - targetPort: redis - protocol: TCP - name: redis - selector: - {{- include "firecrawl.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: redis diff --git a/cluster/applications/firecrawl/chart/templates/worker.yaml b/cluster/applications/firecrawl/chart/templates/worker.yaml deleted file mode 100644 index 6adc5a8..0000000 --- a/cluster/applications/firecrawl/chart/templates/worker.yaml +++ /dev/null @@ -1,39 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "firecrawl.fullname" . }}-worker - namespace: {{ .Values.namespace.name }} - labels: - {{- include "firecrawl.labels" . | nindent 4 }} - app.kubernetes.io/component: worker -spec: - replicas: {{ .Values.worker.replicaCount }} - selector: - matchLabels: - {{- include "firecrawl.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: worker - template: - metadata: - labels: - {{- include "firecrawl.selectorLabels" . | nindent 8 }} - app.kubernetes.io/component: worker - spec: - terminationGracePeriodSeconds: 180 - containers: - - name: worker - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - command: ["node"] - args: ["--max-old-space-size=4096", "dist/src/services/queue-worker.js"] - envFrom: - - configMapRef: - name: {{ include "firecrawl.fullname" . }}-config - env: - - name: FLY_PROCESS_GROUP - value: "worker" - - name: NUQ_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - resources: - {{- toYaml .Values.worker.resources | nindent 12 }} diff --git a/cluster/applications/firecrawl/chart/values.yaml b/cluster/applications/firecrawl/chart/values.yaml deleted file mode 100644 index 8d82b90..0000000 --- a/cluster/applications/firecrawl/chart/values.yaml +++ /dev/null @@ -1,240 +0,0 @@ -# ============================================================================== -# Firecrawl Helm Chart Values -# Production-ready configuration for mk-labs homelab -# ============================================================================== - -# ------------------------------------------------------------------------------ -# Image Configuration -# ------------------------------------------------------------------------------ -image: - # Main Firecrawl API and Worker image - repository: ghcr.io/firecrawl/firecrawl - tag: "latest" # Using latest as official images don't use v1.0.0 tags - pullPolicy: IfNotPresent - -playwright: - repository: ghcr.io/firecrawl/playwright-service - tag: "latest" # Using latest as official images don't use v1.0.0 tags - pullPolicy: IfNotPresent - -# ------------------------------------------------------------------------------ -# API Service Configuration -# ------------------------------------------------------------------------------ -api: - replicaCount: 1 - - resources: - requests: - memory: "4Gi" - cpu: "2000m" - limits: - memory: "6Gi" - cpu: "2000m" - - service: - type: ClusterIP - port: 3002 - -# ------------------------------------------------------------------------------ -# Worker Service Configuration -# ------------------------------------------------------------------------------ -worker: - replicaCount: 1 - - resources: - requests: - memory: "3Gi" - cpu: "1000m" - limits: - memory: "4Gi" - cpu: "1000m" - -# ------------------------------------------------------------------------------ -# Playwright Service Configuration -# ------------------------------------------------------------------------------ -playwrightService: - replicaCount: 1 - - resources: - requests: - memory: "2Gi" - cpu: "1000m" - limits: - memory: "4Gi" - cpu: "2000m" - - service: - type: ClusterIP - port: 3000 - -# ------------------------------------------------------------------------------ -# PostgreSQL Configuration (nuq-postgres) -# ------------------------------------------------------------------------------ -postgres: - image: - repository: ghcr.io/firecrawl/nuq-postgres - tag: "latest" # Using latest as official images don't use v1.0.0 tags - pullPolicy: IfNotPresent - - replicaCount: 1 - - auth: - username: postgres - password: postgres - database: postgres - - resources: - requests: - memory: "512Mi" - cpu: "250m" - limits: - memory: "1Gi" - cpu: "500m" - - # Persistent storage configuration - persistence: - enabled: true - storageClass: "nfs-emporium" - accessMode: ReadWriteOnce - size: 20Gi - - service: - type: ClusterIP - port: 5432 - -# ------------------------------------------------------------------------------ -# Redis Configuration -# ------------------------------------------------------------------------------ -redis: - image: - repository: redis - tag: "7.4.1-alpine" # Pinned stable version - pullPolicy: IfNotPresent - - replicaCount: 1 - - resources: - requests: - memory: "256Mi" - cpu: "100m" - limits: - memory: "512Mi" - cpu: "500m" - - # Persistent storage configuration - persistence: - enabled: true - storageClass: "nfs-emporium" - accessMode: ReadWriteOnce - size: 10Gi - - service: - type: ClusterIP - port: 6379 - -# ------------------------------------------------------------------------------ -# RabbitMQ Configuration -# ------------------------------------------------------------------------------ -rabbitmq: - enabled: true - - image: - repository: rabbitmq - tag: "3.13.7-management-alpine" # Pinned stable version - pullPolicy: IfNotPresent - - replicaCount: 1 - - resources: - requests: - memory: "512Mi" - cpu: "250m" - limits: - memory: "1Gi" - cpu: "500m" - - # Persistent storage configuration - persistence: - enabled: true - storageClass: "nfs-emporium" - accessMode: ReadWriteOnce - size: 5Gi - - service: - type: ClusterIP - amqpPort: 5672 - managementPort: 15672 - -# ------------------------------------------------------------------------------ -# Application Configuration -# ------------------------------------------------------------------------------ -config: - # Environment - ENV: "production" - IS_KUBERNETES: "true" - LOGGING_LEVEL: "INFO" - - # Service configuration - PORT: "3002" - HOST: "0.0.0.0" - WORKER_PORT: "3005" - - # Performance tuning - NUM_WORKERS_PER_QUEUE: "8" - CRAWL_CONCURRENT_REQUESTS: "10" - MAX_CONCURRENT_JOBS: "5" - - # Authentication - USE_DB_AUTHENTICATION: "false" - - # URLs (auto-generated if empty) - REDIS_URL: "" - REDIS_RATE_LIMIT_URL: "" - PLAYWRIGHT_MICROSERVICE_URL: "" - - # Optional external services - OPENAI_BASE_URL: "" - MODEL_NAME: "" - MODEL_EMBEDDING_NAME: "" - OLLAMA_BASE_URL: "" - PROXY_SERVER: "" - PROXY_USERNAME: "" - SEARXNG_ENDPOINT: "" - SEARXNG_ENGINES: "" - SEARXNG_CATEGORIES: "" - SELF_HOSTED_WEBHOOK_URL: "" - -# ------------------------------------------------------------------------------ -# Playwright Service Configuration -# ------------------------------------------------------------------------------ -playwrightConfig: - PORT: "3000" - ALLOW_LOCAL_WEBHOOKS: "" - BLOCK_MEDIA: "" - MAX_CONCURRENT_PAGES: "10" - -# ------------------------------------------------------------------------------ -# Secrets (Optional - use ExternalSecrets in production) -# ------------------------------------------------------------------------------ -secrets: - OPENAI_API_KEY: "" - SLACK_WEBHOOK_URL: "" - LLAMAPARSE_API_KEY: "" - BULL_AUTH_KEY: "" - TEST_API_KEY: "" - FIRE_ENGINE_BETA_URL: "" - SUPABASE_ANON_TOKEN: "" - SUPABASE_URL: "" - SUPABASE_SERVICE_TOKEN: "" - PROXY_PASSWORD: "" - SELF_HOSTED_WEBHOOK_HMAC_SECRET: "" - -# ------------------------------------------------------------------------------ -# Namespace Configuration -# ------------------------------------------------------------------------------ -namespace: - name: firecrawl - labels: - app.kubernetes.io/name: firecrawl - app.kubernetes.io/part-of: mk-labs - epcot.theme/name: spaceship-earth diff --git a/cluster/applications/firecrawl/old-manifests/api-deployment.yaml b/cluster/applications/firecrawl/old-manifests/api-deployment.yaml deleted file mode 100644 index 3f65ee0..0000000 --- a/cluster/applications/firecrawl/old-manifests/api-deployment.yaml +++ /dev/null @@ -1,67 +0,0 @@ -# ------------------------------------------------------------------------------ -# Deployment: Firecrawl API -# Main API service for web scraping and crawling -# ------------------------------------------------------------------------------ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: firecrawl-api - namespace: firecrawl - labels: - app: firecrawl-api - epcot.theme/service: spaceship-earth -spec: - replicas: 1 - selector: - matchLabels: - app: firecrawl-api - template: - metadata: - labels: - app: firecrawl-api - epcot.theme/service: spaceship-earth - spec: - terminationGracePeriodSeconds: 180 - containers: - - name: api - image: ghcr.io/mendableai/firecrawl:latest - command: ["node"] - args: ["--max-old-space-size=6144", "dist/src/index.js"] - envFrom: - - configMapRef: - name: firecrawl-config - env: - - name: FLY_PROCESS_GROUP - value: "app" - - name: NUQ_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - ports: - - containerPort: 3002 - name: http - resources: - requests: - memory: "4Gi" - cpu: "2000m" - limits: - memory: "6Gi" - cpu: "2000m" - livenessProbe: - httpGet: - path: /v0/health/liveness - port: 3002 - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /v0/health/readiness - port: 3002 - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 3 diff --git a/cluster/applications/firecrawl/old-manifests/api-service.yaml b/cluster/applications/firecrawl/old-manifests/api-service.yaml deleted file mode 100644 index 8cb4a86..0000000 --- a/cluster/applications/firecrawl/old-manifests/api-service.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# ------------------------------------------------------------------------------ -# Service: Firecrawl API -# ------------------------------------------------------------------------------ -apiVersion: v1 -kind: Service -metadata: - name: firecrawl-api - namespace: firecrawl - labels: - epcot.theme/service: spaceship-earth -spec: - type: ClusterIP - selector: - app: firecrawl-api - ports: - - protocol: TCP - port: 3002 - targetPort: 3002 - name: http diff --git a/cluster/applications/firecrawl/old-manifests/configmap.yaml b/cluster/applications/firecrawl/old-manifests/configmap.yaml deleted file mode 100644 index a969d8d..0000000 --- a/cluster/applications/firecrawl/old-manifests/configmap.yaml +++ /dev/null @@ -1,60 +0,0 @@ -# ------------------------------------------------------------------------------ -# ConfigMap: firecrawl-config -# Environment configuration for Firecrawl services -# ------------------------------------------------------------------------------ -apiVersion: v1 -kind: ConfigMap -metadata: - name: firecrawl-config - namespace: firecrawl -data: - # Service configuration - PORT: "3002" - HOST: "0.0.0.0" - WORKER_PORT: "3005" - EXTRACT_WORKER_PORT: "3004" - NUQ_WORKER_PORT: "3006" - NUQ_PREFETCH_WORKER_PORT: "3011" - - # Redis URLs - REDIS_URL: "redis://firecrawl-redis:6379" - REDIS_RATE_LIMIT_URL: "redis://firecrawl-redis:6379" - - # PostgreSQL configuration - POSTGRES_HOST: "firecrawl-postgres" - POSTGRES_PORT: "5432" - POSTGRES_USER: "postgres" - POSTGRES_PASSWORD: "postgres" - POSTGRES_DB: "postgres" - - # Database URLs (constructed from above) - NUQ_DATABASE_URL: "postgresql://postgres:postgres@firecrawl-postgres:5432/postgres" - NUQ_DATABASE_URL_LISTEN: "postgresql://postgres:postgres@firecrawl-postgres:5432/postgres" - - # RabbitMQ - NUQ_RABBITMQ_URL: "amqp://firecrawl-rabbitmq:5672" - - # Playwright service - PLAYWRIGHT_MICROSERVICE_URL: "http://firecrawl-playwright:3000/scrape" - - # Worker configuration - NUM_WORKERS_PER_QUEUE: "8" - NUQ_WORKER_COUNT: "5" - - # Performance tuning - CRAWL_CONCURRENT_REQUESTS: "10" - MAX_CONCURRENT_JOBS: "5" - BROWSER_POOL_SIZE: "5" - - # Authentication (disabled for internal use) - USE_DB_AUTHENTICATION: "false" - - # Environment - IS_KUBERNETES: "true" - ENV: "production" - LOGGING_LEVEL: "INFO" - - # Application URLs (internal service mesh) - FIRECRAWL_APP_SCHEME: "http" - FIRECRAWL_APP_HOST: "" - FIRECRAWL_APP_PORT: "" diff --git a/cluster/applications/firecrawl/old-manifests/httproute.yaml b/cluster/applications/firecrawl/old-manifests/httproute.yaml deleted file mode 100644 index a0e80fc..0000000 --- a/cluster/applications/firecrawl/old-manifests/httproute.yaml +++ /dev/null @@ -1,87 +0,0 @@ -# ------------------------------------------------------------------------------ -# HTTPRoute — Firecrawl via Cilium Gateway -# Both primary (spaceship-earth) and secondary (firecrawl) DNS names -# ------------------------------------------------------------------------------ -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: spaceship-earth-tls - namespace: firecrawl -spec: - secretName: spaceship-earth-tls - issuerRef: - name: letsencrypt-prod - kind: ClusterIssuer - dnsNames: - - spaceship-earth.local.mk-labs.cloud ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: firecrawl-tls - namespace: firecrawl -spec: - secretName: firecrawl-tls - issuerRef: - name: letsencrypt-prod - kind: ClusterIssuer - dnsNames: - - firecrawl.local.mk-labs.cloud ---- -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: spaceship-earth - namespace: firecrawl - annotations: - external-dns.alpha.kubernetes.io/hostname: spaceship-earth.local.mk-labs.cloud - external-dns.alpha.kubernetes.io/target: "10.1.71.90" -spec: - parentRefs: - - group: gateway.networking.k8s.io - kind: Gateway - name: fastpass-gateway - namespace: gateway - sectionName: https - hostnames: - - spaceship-earth.local.mk-labs.cloud - rules: - - matches: - - path: - type: PathPrefix - value: / - backendRefs: - - group: "" - kind: Service - name: firecrawl-api - port: 3002 - weight: 1 ---- -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: firecrawl - namespace: firecrawl - annotations: - external-dns.alpha.kubernetes.io/hostname: firecrawl.local.mk-labs.cloud - external-dns.alpha.kubernetes.io/target: "10.1.71.90" -spec: - parentRefs: - - group: gateway.networking.k8s.io - kind: Gateway - name: fastpass-gateway - namespace: gateway - sectionName: https - hostnames: - - firecrawl.local.mk-labs.cloud - rules: - - matches: - - path: - type: PathPrefix - value: / - backendRefs: - - group: "" - kind: Service - name: firecrawl-api - port: 3002 - weight: 1 diff --git a/cluster/applications/firecrawl/old-manifests/namespace.yaml b/cluster/applications/firecrawl/old-manifests/namespace.yaml deleted file mode 100644 index 1fd3b73..0000000 --- a/cluster/applications/firecrawl/old-manifests/namespace.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# ------------------------------------------------------------------------------ -# Namespace: firecrawl -# Web scraping and search service for JARVIS -# ------------------------------------------------------------------------------ -apiVersion: v1 -kind: Namespace -metadata: - name: firecrawl - labels: - app.kubernetes.io/name: firecrawl - app.kubernetes.io/part-of: mk-labs - epcot.theme/name: spaceship-earth diff --git a/cluster/applications/firecrawl/old-manifests/playwright-configmap.yaml b/cluster/applications/firecrawl/old-manifests/playwright-configmap.yaml deleted file mode 100644 index 6419455..0000000 --- a/cluster/applications/firecrawl/old-manifests/playwright-configmap.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# ------------------------------------------------------------------------------ -# ConfigMap: Playwright configuration -# ------------------------------------------------------------------------------ -apiVersion: v1 -kind: ConfigMap -metadata: - name: firecrawl-playwright-config - namespace: firecrawl -data: - PORT: "3000" - MAX_CONCURRENT_PAGES: "10" - ALLOW_LOCAL_WEBHOOKS: "" - BLOCK_MEDIA: "" diff --git a/cluster/applications/firecrawl/old-manifests/playwright-deployment.yaml b/cluster/applications/firecrawl/old-manifests/playwright-deployment.yaml deleted file mode 100644 index 5552d8c..0000000 --- a/cluster/applications/firecrawl/old-manifests/playwright-deployment.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# ------------------------------------------------------------------------------ -# Deployment: Playwright Service -# Browser automation service for web scraping -# ------------------------------------------------------------------------------ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: firecrawl-playwright - namespace: firecrawl - labels: - app: firecrawl-playwright -spec: - replicas: 1 - selector: - matchLabels: - app: firecrawl-playwright - template: - metadata: - labels: - app: firecrawl-playwright - spec: - containers: - - name: playwright - image: ghcr.io/mendableai/firecrawl/playwright-service:latest - envFrom: - - configMapRef: - name: firecrawl-playwright-config - ports: - - containerPort: 3000 - name: http - resources: - requests: - memory: "2Gi" - cpu: "1000m" - limits: - memory: "4Gi" - cpu: "2000m" - volumeMounts: - - name: cache - mountPath: /tmp/.cache - volumes: - - name: cache - emptyDir: - sizeLimit: 1Gi diff --git a/cluster/applications/firecrawl/old-manifests/playwright-service.yaml b/cluster/applications/firecrawl/old-manifests/playwright-service.yaml deleted file mode 100644 index 7eec515..0000000 --- a/cluster/applications/firecrawl/old-manifests/playwright-service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# ------------------------------------------------------------------------------ -# Service: Playwright -# ------------------------------------------------------------------------------ -apiVersion: v1 -kind: Service -metadata: - name: firecrawl-playwright - namespace: firecrawl -spec: - type: ClusterIP - selector: - app: firecrawl-playwright - ports: - - protocol: TCP - port: 3000 - targetPort: 3000 - name: http diff --git a/cluster/applications/firecrawl/old-manifests/postgres-deployment.yaml b/cluster/applications/firecrawl/old-manifests/postgres-deployment.yaml deleted file mode 100644 index b589960..0000000 --- a/cluster/applications/firecrawl/old-manifests/postgres-deployment.yaml +++ /dev/null @@ -1,47 +0,0 @@ -# ------------------------------------------------------------------------------ -# Deployment: PostgreSQL (nuq-postgres) -# Database for Firecrawl job queue and state management -# ------------------------------------------------------------------------------ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: firecrawl-postgres - namespace: firecrawl - labels: - app: firecrawl-postgres -spec: - replicas: 1 - selector: - matchLabels: - app: firecrawl-postgres - template: - metadata: - labels: - app: firecrawl-postgres - spec: - containers: - - name: postgres - image: ghcr.io/mendableai/firecrawl/nuq-postgres:latest - env: - - name: POSTGRES_USER - value: "postgres" - - name: POSTGRES_PASSWORD - value: "postgres" - - name: POSTGRES_DB - value: "postgres" - ports: - - containerPort: 5432 - name: postgres - resources: - requests: - memory: "512Mi" - cpu: "250m" - limits: - memory: "1Gi" - cpu: "500m" - volumeMounts: - - name: postgres-data - mountPath: /var/lib/postgresql/data - volumes: - - name: postgres-data - emptyDir: {} # For now, ephemeral storage. TODO: Add PVC for persistence diff --git a/cluster/applications/firecrawl/old-manifests/postgres-service.yaml b/cluster/applications/firecrawl/old-manifests/postgres-service.yaml deleted file mode 100644 index 4020782..0000000 --- a/cluster/applications/firecrawl/old-manifests/postgres-service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# ------------------------------------------------------------------------------ -# Service: PostgreSQL -# ------------------------------------------------------------------------------ -apiVersion: v1 -kind: Service -metadata: - name: firecrawl-postgres - namespace: firecrawl -spec: - type: ClusterIP - selector: - app: firecrawl-postgres - ports: - - protocol: TCP - port: 5432 - targetPort: 5432 - name: postgres diff --git a/cluster/applications/firecrawl/old-manifests/rabbitmq-deployment.yaml b/cluster/applications/firecrawl/old-manifests/rabbitmq-deployment.yaml deleted file mode 100644 index aa260b4..0000000 --- a/cluster/applications/firecrawl/old-manifests/rabbitmq-deployment.yaml +++ /dev/null @@ -1,51 +0,0 @@ -# ------------------------------------------------------------------------------ -# Deployment: RabbitMQ -# Message queue for Firecrawl distributed task processing -# ------------------------------------------------------------------------------ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: firecrawl-rabbitmq - namespace: firecrawl - labels: - app: firecrawl-rabbitmq -spec: - replicas: 1 - selector: - matchLabels: - app: firecrawl-rabbitmq - template: - metadata: - labels: - app: firecrawl-rabbitmq - spec: - containers: - - name: rabbitmq - image: rabbitmq:3-management - command: ["rabbitmq-server"] - ports: - - containerPort: 5672 - name: amqp - - containerPort: 15672 - name: management - resources: - requests: - memory: "512Mi" - cpu: "250m" - limits: - memory: "1Gi" - cpu: "500m" - livenessProbe: - exec: - command: ["rabbitmq-diagnostics", "-q", "check_running"] - initialDelaySeconds: 5 - periodSeconds: 5 - timeoutSeconds: 5 - failureThreshold: 3 - readinessProbe: - exec: - command: ["rabbitmq-diagnostics", "-q", "check_running"] - initialDelaySeconds: 5 - periodSeconds: 5 - timeoutSeconds: 5 - failureThreshold: 3 diff --git a/cluster/applications/firecrawl/old-manifests/rabbitmq-service.yaml b/cluster/applications/firecrawl/old-manifests/rabbitmq-service.yaml deleted file mode 100644 index 4f637a7..0000000 --- a/cluster/applications/firecrawl/old-manifests/rabbitmq-service.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# ------------------------------------------------------------------------------ -# Service: RabbitMQ -# ------------------------------------------------------------------------------ -apiVersion: v1 -kind: Service -metadata: - name: firecrawl-rabbitmq - namespace: firecrawl -spec: - type: ClusterIP - selector: - app: firecrawl-rabbitmq - ports: - - protocol: TCP - port: 5672 - targetPort: 5672 - name: amqp - - protocol: TCP - port: 15672 - targetPort: 15672 - name: management diff --git a/cluster/applications/firecrawl/old-manifests/redis-deployment.yaml b/cluster/applications/firecrawl/old-manifests/redis-deployment.yaml deleted file mode 100644 index b5d84a3..0000000 --- a/cluster/applications/firecrawl/old-manifests/redis-deployment.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# ------------------------------------------------------------------------------ -# Deployment: Redis -# In-memory cache and job queue for Firecrawl -# ------------------------------------------------------------------------------ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: firecrawl-redis - namespace: firecrawl - labels: - app: firecrawl-redis -spec: - replicas: 1 - selector: - matchLabels: - app: firecrawl-redis - template: - metadata: - labels: - app: firecrawl-redis - spec: - containers: - - name: redis - image: redis:alpine - command: ["redis-server", "--bind", "0.0.0.0"] - ports: - - containerPort: 6379 - name: redis - resources: - requests: - memory: "256Mi" - cpu: "100m" - limits: - memory: "512Mi" - cpu: "500m" diff --git a/cluster/applications/firecrawl/old-manifests/redis-service.yaml b/cluster/applications/firecrawl/old-manifests/redis-service.yaml deleted file mode 100644 index 74c7cdc..0000000 --- a/cluster/applications/firecrawl/old-manifests/redis-service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# ------------------------------------------------------------------------------ -# Service: Redis -# ------------------------------------------------------------------------------ -apiVersion: v1 -kind: Service -metadata: - name: firecrawl-redis - namespace: firecrawl -spec: - type: ClusterIP - selector: - app: firecrawl-redis - ports: - - protocol: TCP - port: 6379 - targetPort: 6379 - name: redis diff --git a/cluster/applications/firecrawl/old-manifests/worker-deployment.yaml b/cluster/applications/firecrawl/old-manifests/worker-deployment.yaml deleted file mode 100644 index 470ad6b..0000000 --- a/cluster/applications/firecrawl/old-manifests/worker-deployment.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# ------------------------------------------------------------------------------ -# Deployment: Firecrawl Worker -# Background worker for processing crawl jobs -# ------------------------------------------------------------------------------ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: firecrawl-worker - namespace: firecrawl - labels: - app: firecrawl-worker -spec: - replicas: 1 - selector: - matchLabels: - app: firecrawl-worker - template: - metadata: - labels: - app: firecrawl-worker - spec: - terminationGracePeriodSeconds: 180 - containers: - - name: worker - image: ghcr.io/mendableai/firecrawl:latest - command: ["node"] - args: ["--max-old-space-size=4096", "dist/src/services/queue-worker.js"] - envFrom: - - configMapRef: - name: firecrawl-config - env: - - name: FLY_PROCESS_GROUP - value: "worker" - - name: NUQ_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - resources: - requests: - memory: "3Gi" - cpu: "1000m" - limits: - memory: "4Gi" - cpu: "1000m"