diff --git a/cluster/applications/firecrawl/configmap-playwright.yaml b/cluster/applications/firecrawl/configmap-playwright.yaml new file mode 100644 index 0000000..80203af --- /dev/null +++ b/cluster/applications/firecrawl/configmap-playwright.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: playwright-service-config + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: browser + app.kubernetes.io/part-of: firecrawl +data: + PORT: "3000" + ALLOW_LOCAL_WEBHOOKS: "false" + MAX_CONCURRENT_PAGES: "10" diff --git a/cluster/applications/firecrawl/configmap.yaml b/cluster/applications/firecrawl/configmap.yaml new file mode 100644 index 0000000..43253a8 --- /dev/null +++ b/cluster/applications/firecrawl/configmap.yaml @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: firecrawl-config + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: configuration + app.kubernetes.io/part-of: firecrawl +data: + # Service URLs (Kubernetes internal DNS) + REDIS_URL: "redis://redis:6379" + REDIS_RATE_LIMIT_URL: "redis://redis:6379" + POSTGRES_HOST: "nuq-postgres" + POSTGRES_PORT: "5432" + POSTGRES_USER: "postgres" + POSTGRES_DB: "postgres" + NUQ_RABBITMQ_URL: "amqp://rabbitmq:5672" + PLAYWRIGHT_MICROSERVICE_URL: "http://playwright-service:3000/scrape" + + # Server Configuration + HOST: "0.0.0.0" + PORT: "3002" + WORKER_PORT: "3005" + EXTRACT_WORKER_PORT: "3004" + USE_DB_AUTHENTICATION: "false" + ENV: "production" + + # Performance Tuning + NUM_WORKERS_PER_QUEUE: "8" + CRAWL_CONCURRENT_REQUESTS: "10" + MAX_CONCURRENT_JOBS: "5" + BROWSER_POOL_SIZE: "5" + MAX_CONCURRENT_PAGES: "10" + HARNESS_STARTUP_TIMEOUT_MS: "60000" + + # Resource Limits (Self-Protection) + MAX_CPU: "0.8" + MAX_RAM: "0.8" + + # Logging + LOGGING_LEVEL: "info" + + # Security + ALLOW_LOCAL_WEBHOOKS: "false" diff --git a/cluster/applications/firecrawl/deployment-api.yaml b/cluster/applications/firecrawl/deployment-api.yaml new file mode 100644 index 0000000..92c69f8 --- /dev/null +++ b/cluster/applications/firecrawl/deployment-api.yaml @@ -0,0 +1,60 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: firecrawl-api + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: api + app.kubernetes.io/part-of: firecrawl +spec: + replicas: 1 + selector: + matchLabels: + app: firecrawl-api + template: + metadata: + labels: + app: firecrawl-api + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: api + spec: + containers: + - name: api + image: the-seas.local.mk-labs.cloud/library/firecrawl-api:latest + imagePullPolicy: Always + ports: + - name: http + containerPort: 3002 + protocol: TCP + envFrom: + - configMapRef: + name: firecrawl-config + - secretRef: + name: firecrawl-secrets + env: + - name: FLY_PROCESS_GROUP + value: "app" + resources: + requests: + cpu: 1000m + memory: 4Gi + limits: + cpu: 2000m + memory: 6Gi + livenessProbe: + httpGet: + path: /v0/health/liveness + port: 3002 + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /v0/health/readiness + port: 3002 + initialDelaySeconds: 30 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 diff --git a/cluster/applications/firecrawl/deployment-nuq-worker.yaml b/cluster/applications/firecrawl/deployment-nuq-worker.yaml new file mode 100644 index 0000000..ab66529 --- /dev/null +++ b/cluster/applications/firecrawl/deployment-nuq-worker.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: firecrawl-api-nuq-worker + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: nuq-worker + app.kubernetes.io/part-of: firecrawl +spec: + replicas: 1 + selector: + matchLabels: + app: firecrawl-api-nuq-worker + template: + metadata: + labels: + app: firecrawl-api-nuq-worker + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: nuq-worker + spec: + containers: + - name: nuq-worker + image: the-seas.local.mk-labs.cloud/library/firecrawl-api:latest + imagePullPolicy: Always + command: ["node"] + args: ["dist/src/services/worker/nuq-worker.js"] + envFrom: + - configMapRef: + name: firecrawl-config + - secretRef: + name: firecrawl-secrets + env: + - name: FLY_PROCESS_GROUP + value: "nuq-worker" + resources: + requests: + cpu: 500m + memory: 3Gi + limits: + cpu: 1000m + memory: 4Gi + livenessProbe: + exec: + command: + - pgrep + - -f + - nuq-worker + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + exec: + command: + - pgrep + - -f + - nuq-worker + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 diff --git a/cluster/applications/firecrawl/deployment-playwright.yaml b/cluster/applications/firecrawl/deployment-playwright.yaml new file mode 100644 index 0000000..24212c5 --- /dev/null +++ b/cluster/applications/firecrawl/deployment-playwright.yaml @@ -0,0 +1,63 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: playwright-service + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: browser + app.kubernetes.io/part-of: firecrawl +spec: + replicas: 1 + selector: + matchLabels: + app: playwright-service + template: + metadata: + labels: + app: playwright-service + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: browser + spec: + containers: + - name: playwright + image: the-seas.local.mk-labs.cloud/library/firecrawl-playwright:latest + imagePullPolicy: Always + ports: + - name: http + containerPort: 3000 + protocol: TCP + envFrom: + - configMapRef: + name: playwright-service-config + resources: + requests: + cpu: 1000m + memory: 2Gi + limits: + cpu: 2000m + memory: 4Gi + volumeMounts: + - name: tmpfs + mountPath: /tmp + livenessProbe: + httpGet: + path: / + port: 3000 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: / + port: 3000 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + volumes: + - name: tmpfs + emptyDir: + medium: Memory + sizeLimit: 1Gi diff --git a/cluster/applications/firecrawl/deployment-postgres.yaml b/cluster/applications/firecrawl/deployment-postgres.yaml new file mode 100644 index 0000000..0e38b96 --- /dev/null +++ b/cluster/applications/firecrawl/deployment-postgres.yaml @@ -0,0 +1,83 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nuq-postgres + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: database + app.kubernetes.io/part-of: firecrawl +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: nuq-postgres + template: + metadata: + labels: + app: nuq-postgres + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: database + spec: + containers: + - name: postgres + image: the-seas.local.mk-labs.cloud/library/firecrawl-postgres:latest + imagePullPolicy: Always + ports: + - name: postgres + containerPort: 5432 + protocol: TCP + env: + - name: POSTGRES_USER + valueFrom: + configMapKeyRef: + name: firecrawl-config + key: POSTGRES_USER + - name: POSTGRES_DB + valueFrom: + configMapKeyRef: + name: firecrawl-config + key: POSTGRES_DB + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: firecrawl-secrets + key: POSTGRES_PASSWORD + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + volumeMounts: + - name: postgres-data + mountPath: /var/lib/postgresql/data + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi + livenessProbe: + exec: + command: + - pg_isready + - -U + - postgres + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + exec: + command: + - pg_isready + - -U + - postgres + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + volumes: + - name: postgres-data + persistentVolumeClaim: + claimName: postgres-data diff --git a/cluster/applications/firecrawl/deployment-rabbitmq.yaml b/cluster/applications/firecrawl/deployment-rabbitmq.yaml new file mode 100644 index 0000000..1f633b4 --- /dev/null +++ b/cluster/applications/firecrawl/deployment-rabbitmq.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rabbitmq + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: messaging + app.kubernetes.io/part-of: firecrawl +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: rabbitmq + template: + metadata: + labels: + app: rabbitmq + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: messaging + spec: + containers: + - name: rabbitmq + image: rabbitmq:3-management + imagePullPolicy: IfNotPresent + ports: + - name: amqp + containerPort: 5672 + protocol: TCP + - name: management + containerPort: 15672 + protocol: TCP + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 500m + memory: 1Gi + livenessProbe: + exec: + command: + - rabbitmq-diagnostics + - -q + - ping + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 3 + readinessProbe: + exec: + command: + - rabbitmq-diagnostics + - -q + - check_running + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 diff --git a/cluster/applications/firecrawl/deployment-redis.yaml b/cluster/applications/firecrawl/deployment-redis.yaml new file mode 100644 index 0000000..da7247b --- /dev/null +++ b/cluster/applications/firecrawl/deployment-redis.yaml @@ -0,0 +1,56 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: cache + app.kubernetes.io/part-of: firecrawl +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: cache + spec: + containers: + - name: redis + image: redis:alpine + imagePullPolicy: IfNotPresent + ports: + - name: redis + containerPort: 6379 + protocol: TCP + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 500m + memory: 1Gi + livenessProbe: + exec: + command: + - redis-cli + - ping + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + exec: + command: + - redis-cli + - ping + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 diff --git a/cluster/applications/firecrawl/deployment-worker.yaml b/cluster/applications/firecrawl/deployment-worker.yaml new file mode 100644 index 0000000..b409a74 --- /dev/null +++ b/cluster/applications/firecrawl/deployment-worker.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: firecrawl-api-worker + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: worker + app.kubernetes.io/part-of: firecrawl +spec: + replicas: 1 + selector: + matchLabels: + app: firecrawl-api-worker + template: + metadata: + labels: + app: firecrawl-api-worker + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: worker + spec: + containers: + - name: worker + image: the-seas.local.mk-labs.cloud/library/firecrawl-api:latest + imagePullPolicy: Always + command: ["node"] + args: ["dist/src/services/queue-worker.js"] + ports: + - name: http + containerPort: 3005 + protocol: TCP + envFrom: + - configMapRef: + name: firecrawl-config + - secretRef: + name: firecrawl-secrets + env: + - name: FLY_PROCESS_GROUP + value: "worker" + resources: + requests: + cpu: 500m + memory: 3Gi + limits: + cpu: 1000m + memory: 4Gi + livenessProbe: + httpGet: + path: / + port: 3005 + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: / + port: 3005 + initialDelaySeconds: 30 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 diff --git a/cluster/applications/firecrawl/httproute.yaml b/cluster/applications/firecrawl/httproute.yaml new file mode 100644 index 0000000..22c7cf0 --- /dev/null +++ b/cluster/applications/firecrawl/httproute.yaml @@ -0,0 +1,26 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: firecrawl + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: ingress + app.kubernetes.io/part-of: firecrawl + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod +spec: + parentRefs: + - name: gateway + namespace: default + hostnames: + - spaceship-earth.local.mk-labs.cloud + - firecrawl.local.mk-labs.cloud + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: firecrawl-api + port: 3002 diff --git a/cluster/applications/firecrawl/namespace.yaml b/cluster/applications/firecrawl/namespace.yaml new file mode 100644 index 0000000..e5fea4f --- /dev/null +++ b/cluster/applications/firecrawl/namespace.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: firecrawl + labels: + name: firecrawl + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: application + app.kubernetes.io/part-of: platform-buildout + epcot-theme: spaceship-earth + annotations: + description: "Firecrawl web scraping and crawling service - enables JARVIS web search capability" diff --git a/cluster/applications/firecrawl/postgres-pvc.yaml b/cluster/applications/firecrawl/postgres-pvc.yaml new file mode 100644 index 0000000..0230e1c --- /dev/null +++ b/cluster/applications/firecrawl/postgres-pvc.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgres-data + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: database + app.kubernetes.io/part-of: firecrawl +spec: + accessModes: + - ReadWriteOnce + storageClassName: nfs-emporium + resources: + requests: + storage: 10Gi diff --git a/cluster/applications/firecrawl/service-api.yaml b/cluster/applications/firecrawl/service-api.yaml new file mode 100644 index 0000000..c3d12bf --- /dev/null +++ b/cluster/applications/firecrawl/service-api.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: firecrawl-api + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: api + app.kubernetes.io/part-of: firecrawl +spec: + type: ClusterIP + selector: + app: firecrawl-api + ports: + - name: http + port: 3002 + targetPort: 3002 + protocol: TCP diff --git a/cluster/applications/firecrawl/service-playwright.yaml b/cluster/applications/firecrawl/service-playwright.yaml new file mode 100644 index 0000000..c539bb8 --- /dev/null +++ b/cluster/applications/firecrawl/service-playwright.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: playwright-service + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: browser + app.kubernetes.io/part-of: firecrawl +spec: + type: ClusterIP + selector: + app: playwright-service + ports: + - name: http + port: 3000 + targetPort: 3000 + protocol: TCP diff --git a/cluster/applications/firecrawl/service-postgres.yaml b/cluster/applications/firecrawl/service-postgres.yaml new file mode 100644 index 0000000..e21715c --- /dev/null +++ b/cluster/applications/firecrawl/service-postgres.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: nuq-postgres + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: database + app.kubernetes.io/part-of: firecrawl +spec: + type: ClusterIP + selector: + app: nuq-postgres + ports: + - name: postgres + port: 5432 + targetPort: 5432 + protocol: TCP diff --git a/cluster/applications/firecrawl/service-rabbitmq.yaml b/cluster/applications/firecrawl/service-rabbitmq.yaml new file mode 100644 index 0000000..1dd5e6c --- /dev/null +++ b/cluster/applications/firecrawl/service-rabbitmq.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + name: rabbitmq + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: messaging + app.kubernetes.io/part-of: firecrawl +spec: + type: ClusterIP + selector: + app: rabbitmq + ports: + - name: amqp + port: 5672 + targetPort: 5672 + protocol: TCP + - name: management + port: 15672 + targetPort: 15672 + protocol: TCP diff --git a/cluster/applications/firecrawl/service-redis.yaml b/cluster/applications/firecrawl/service-redis.yaml new file mode 100644 index 0000000..86d23b9 --- /dev/null +++ b/cluster/applications/firecrawl/service-redis.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: firecrawl + labels: + app.kubernetes.io/name: firecrawl + app.kubernetes.io/component: cache + app.kubernetes.io/part-of: firecrawl +spec: + type: ClusterIP + selector: + app: redis + ports: + - name: redis + port: 6379 + targetPort: 6379 + protocol: TCP