refactor(firecrawl): Convert to production-ready Helm chart with persistent storage

CRITICAL FIXES:
✅ Add PersistentVolumeClaims for all stateful services
   - PostgreSQL: 20Gi PVC on nfs-emporium (data persistence)
   - Redis: 10Gi PVC on nfs-emporium (cache and queue)
   - RabbitMQ: 5Gi PVC on nfs-emporium (message queue)

✅ Pin all image versions (no more 'latest' tags)
   - Firecrawl API/Worker: v1.0.0
   - Playwright Service: v1.0.0
   - PostgreSQL (nuq-postgres): v1.0.0
   - Redis: 7.4.1-alpine
   - RabbitMQ: 3.13.7-management-alpine

✅ Convert raw manifests to proper Helm chart
   - Template-based configuration
   - Centralized values.yaml
   - Proper Helm helpers and labels
   - Easy configuration management

WHAT CHANGED:
- Created chart/ directory with full Helm chart structure
- Moved old manifests to old-manifests/ for reference
- Updated ArgoCD Application to use Helm chart source
- Added comprehensive README and MIGRATION docs
- All services now use nfs-emporium storage class
- Redis configured with AOF persistence
- Proper resource limits and health checks
- Gateway/HTTPRoute configs integrated into chart

DEPLOYMENT:
ArgoCD will automatically sync and apply changes.
Old ephemeral data will be lost (fresh start with persistence).

Resolves data loss issues and brings deployment to production standards.

cluster/applications/firecrawl/chart/templates/_helpers.tpl

@@ -0,0 +1,49 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "firecrawl.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+*/}}
+{{- define "firecrawl.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "firecrawl.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "firecrawl.labels" -}}
+helm.sh/chart: {{ include "firecrawl.chart" . }}
+{{ include "firecrawl.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "firecrawl.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "firecrawl.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}

cluster/applications/firecrawl/chart/templates/api.yaml

@@ -0,0 +1,81 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "firecrawl.fullname" . }}-api
+  namespace: {{ .Values.namespace.name }}
+  labels:
+    {{- include "firecrawl.labels" . | nindent 4 }}
+    app.kubernetes.io/component: api
+    epcot.theme/service: spaceship-earth
+spec:
+  replicas: {{ .Values.api.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "firecrawl.selectorLabels" . | nindent 6 }}
+      app.kubernetes.io/component: api
+  template:
+    metadata:
+      labels:
+        {{- include "firecrawl.selectorLabels" . | nindent 8 }}
+        app.kubernetes.io/component: api
+        epcot.theme/service: spaceship-earth
+    spec:
+      terminationGracePeriodSeconds: 180
+      containers:
+        - name: api
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          command: ["node"]
+          args: ["--max-old-space-size=6144", "dist/src/index.js"]
+          envFrom:
+            - configMapRef:
+                name: {{ include "firecrawl.fullname" . }}-config
+          env:
+            - name: FLY_PROCESS_GROUP
+              value: "app"
+            - name: NUQ_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+          ports:
+            - containerPort: 3002
+              name: http
+          resources:
+            {{- toYaml .Values.api.resources | nindent 12 }}
+          livenessProbe:
+            httpGet:
+              path: /v0/health/liveness
+              port: 3002
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 5
+            successThreshold: 1
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /v0/health/readiness
+              port: 3002
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 5
+            successThreshold: 1
+            failureThreshold: 3
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "firecrawl.fullname" . }}-api
+  namespace: {{ .Values.namespace.name }}
+  labels:
+    {{- include "firecrawl.labels" . | nindent 4 }}
+    app.kubernetes.io/component: api
+spec:
+  type: {{ .Values.api.service.type }}
+  ports:
+    - port: {{ .Values.api.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "firecrawl.selectorLabels" . | nindent 4 }}
+    app.kubernetes.io/component: api

cluster/applications/firecrawl/chart/templates/configmap.yaml

@@ -0,0 +1,89 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "firecrawl.fullname" . }}-config
+  namespace: {{ .Values.namespace.name }}
+  labels:
+    {{- include "firecrawl.labels" . | nindent 4 }}
+data:
+  # Environment
+  ENV: {{ .Values.config.ENV | quote }}
+  IS_KUBERNETES: {{ .Values.config.IS_KUBERNETES | quote }}
+  LOGGING_LEVEL: {{ .Values.config.LOGGING_LEVEL | quote }}
+  
+  # Service configuration
+  PORT: {{ .Values.config.PORT | quote }}
+  HOST: {{ .Values.config.HOST | quote }}
+  WORKER_PORT: {{ .Values.config.WORKER_PORT | quote }}
+  
+  # Performance tuning
+  NUM_WORKERS_PER_QUEUE: {{ .Values.config.NUM_WORKERS_PER_QUEUE | quote }}
+  CRAWL_CONCURRENT_REQUESTS: {{ .Values.config.CRAWL_CONCURRENT_REQUESTS | quote }}
+  MAX_CONCURRENT_JOBS: {{ .Values.config.MAX_CONCURRENT_JOBS | quote }}
+  
+  # Authentication
+  USE_DB_AUTHENTICATION: {{ .Values.config.USE_DB_AUTHENTICATION | quote }}
+  
+  # Service URLs
+  REDIS_URL: {{ .Values.config.REDIS_URL | default (printf "redis://%s-redis:%d" (include "firecrawl.fullname" .) (.Values.redis.service.port | int)) | quote }}
+  REDIS_RATE_LIMIT_URL: {{ .Values.config.REDIS_RATE_LIMIT_URL | default (printf "redis://%s-redis:%d" (include "firecrawl.fullname" .) (.Values.redis.service.port | int)) | quote }}
+  PLAYWRIGHT_MICROSERVICE_URL: {{ .Values.config.PLAYWRIGHT_MICROSERVICE_URL | default (printf "http://%s-playwright:%d/scrape" (include "firecrawl.fullname" .) (.Values.playwrightService.service.port | int)) | quote }}
+  
+  # Database configuration
+  POSTGRES_HOST: {{ printf "%s-postgres" (include "firecrawl.fullname" .) | quote }}
+  POSTGRES_PORT: {{ .Values.postgres.service.port | quote }}
+  POSTGRES_USER: {{ .Values.postgres.auth.username | quote }}
+  POSTGRES_PASSWORD: {{ .Values.postgres.auth.password | quote }}
+  POSTGRES_DB: {{ .Values.postgres.auth.database | quote }}
+  
+  # RabbitMQ configuration
+  NUQ_RABBITMQ_URL: {{ printf "amqp://%s-rabbitmq:%d" (include "firecrawl.fullname" .) (.Values.rabbitmq.service.amqpPort | int) | quote }}
+  
+  # Optional external services
+  {{- if .Values.config.OPENAI_BASE_URL }}
+  OPENAI_BASE_URL: {{ .Values.config.OPENAI_BASE_URL | quote }}
+  {{- end }}
+  {{- if .Values.config.MODEL_NAME }}
+  MODEL_NAME: {{ .Values.config.MODEL_NAME | quote }}
+  {{- end }}
+  {{- if .Values.config.MODEL_EMBEDDING_NAME }}
+  MODEL_EMBEDDING_NAME: {{ .Values.config.MODEL_EMBEDDING_NAME | quote }}
+  {{- end }}
+  {{- if .Values.config.OLLAMA_BASE_URL }}
+  OLLAMA_BASE_URL: {{ .Values.config.OLLAMA_BASE_URL | quote }}
+  {{- end }}
+  {{- if .Values.config.PROXY_SERVER }}
+  PROXY_SERVER: {{ .Values.config.PROXY_SERVER | quote }}
+  {{- end }}
+  {{- if .Values.config.PROXY_USERNAME }}
+  PROXY_USERNAME: {{ .Values.config.PROXY_USERNAME | quote }}
+  {{- end }}
+  {{- if .Values.config.SEARXNG_ENDPOINT }}
+  SEARXNG_ENDPOINT: {{ .Values.config.SEARXNG_ENDPOINT | quote }}
+  {{- end }}
+  {{- if .Values.config.SEARXNG_ENGINES }}
+  SEARXNG_ENGINES: {{ .Values.config.SEARXNG_ENGINES | quote }}
+  {{- end }}
+  {{- if .Values.config.SEARXNG_CATEGORIES }}
+  SEARXNG_CATEGORIES: {{ .Values.config.SEARXNG_CATEGORIES | quote }}
+  {{- end }}
+  {{- if .Values.config.SELF_HOSTED_WEBHOOK_URL }}
+  SELF_HOSTED_WEBHOOK_URL: {{ .Values.config.SELF_HOSTED_WEBHOOK_URL | quote }}
+  {{- end }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "firecrawl.fullname" . }}-playwright-config
+  namespace: {{ .Values.namespace.name }}
+  labels:
+    {{- include "firecrawl.labels" . | nindent 4 }}
+data:
+  PORT: {{ .Values.playwrightConfig.PORT | quote }}
+  MAX_CONCURRENT_PAGES: {{ .Values.playwrightConfig.MAX_CONCURRENT_PAGES | quote }}
+  {{- if .Values.playwrightConfig.ALLOW_LOCAL_WEBHOOKS }}
+  ALLOW_LOCAL_WEBHOOKS: {{ .Values.playwrightConfig.ALLOW_LOCAL_WEBHOOKS | quote }}
+  {{- end }}
+  {{- if .Values.playwrightConfig.BLOCK_MEDIA }}
+  BLOCK_MEDIA: {{ .Values.playwrightConfig.BLOCK_MEDIA | quote }}
+  {{- end }}

cluster/applications/firecrawl/chart/templates/httproute.yaml

@@ -0,0 +1,87 @@
+# ------------------------------------------------------------------------------
+# HTTPRoute — Firecrawl via Cilium Gateway
+# Both primary (spaceship-earth) and secondary (firecrawl) DNS names
+# ------------------------------------------------------------------------------
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: spaceship-earth-tls
+  namespace: {{ .Values.namespace.name }}
+spec:
+  secretName: spaceship-earth-tls
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+  dnsNames:
+    - spaceship-earth.local.mk-labs.cloud
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: firecrawl-tls
+  namespace: {{ .Values.namespace.name }}
+spec:
+  secretName: firecrawl-tls
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+  dnsNames:
+    - firecrawl.local.mk-labs.cloud
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: spaceship-earth
+  namespace: {{ .Values.namespace.name }}
+  annotations:
+    external-dns.alpha.kubernetes.io/hostname: spaceship-earth.local.mk-labs.cloud
+    external-dns.alpha.kubernetes.io/target: "10.1.71.90"
+spec:
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: fastpass-gateway
+      namespace: gateway
+      sectionName: https
+  hostnames:
+    - spaceship-earth.local.mk-labs.cloud
+  rules:
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /
+      backendRefs:
+        - group: ""
+          kind: Service
+          name: {{ include "firecrawl.fullname" . }}-api
+          port: {{ .Values.api.service.port }}
+          weight: 1
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: firecrawl
+  namespace: {{ .Values.namespace.name }}
+  annotations:
+    external-dns.alpha.kubernetes.io/hostname: firecrawl.local.mk-labs.cloud
+    external-dns.alpha.kubernetes.io/target: "10.1.71.90"
+spec:
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: fastpass-gateway
+      namespace: gateway
+      sectionName: https
+  hostnames:
+    - firecrawl.local.mk-labs.cloud
+  rules:
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /
+      backendRefs:
+        - group: ""
+          kind: Service
+          name: {{ include "firecrawl.fullname" . }}-api
+          port: {{ .Values.api.service.port }}
+          weight: 1

cluster/applications/firecrawl/chart/templates/namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: {{ .Values.namespace.name }}
+  labels:
+    {{- toYaml .Values.namespace.labels | nindent 4 }}

cluster/applications/firecrawl/chart/templates/playwright.yaml

@@ -0,0 +1,58 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "firecrawl.fullname" . }}-playwright
+  namespace: {{ .Values.namespace.name }}
+  labels:
+    {{- include "firecrawl.labels" . | nindent 4 }}
+    app.kubernetes.io/component: playwright
+spec:
+  replicas: {{ .Values.playwrightService.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "firecrawl.selectorLabels" . | nindent 6 }}
+      app.kubernetes.io/component: playwright
+  template:
+    metadata:
+      labels:
+        {{- include "firecrawl.selectorLabels" . | nindent 8 }}
+        app.kubernetes.io/component: playwright
+    spec:
+      containers:
+        - name: playwright
+          image: "{{ .Values.playwright.repository }}:{{ .Values.playwright.tag }}"
+          imagePullPolicy: {{ .Values.playwright.pullPolicy }}
+          envFrom:
+            - configMapRef:
+                name: {{ include "firecrawl.fullname" . }}-playwright-config
+          ports:
+            - containerPort: 3000
+              name: http
+          resources:
+            {{- toYaml .Values.playwrightService.resources | nindent 12 }}
+          volumeMounts:
+            - name: cache
+              mountPath: /tmp/.cache
+      volumes:
+        - name: cache
+          emptyDir:
+            sizeLimit: 1Gi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "firecrawl.fullname" . }}-playwright
+  namespace: {{ .Values.namespace.name }}
+  labels:
+    {{- include "firecrawl.labels" . | nindent 4 }}
+    app.kubernetes.io/component: playwright
+spec:
+  type: {{ .Values.playwrightService.service.type }}
+  ports:
+    - port: {{ .Values.playwrightService.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "firecrawl.selectorLabels" . | nindent 4 }}
+    app.kubernetes.io/component: playwright

cluster/applications/firecrawl/chart/templates/postgres.yaml

@@ -0,0 +1,67 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "firecrawl.fullname" . }}-postgres
+  namespace: {{ .Values.namespace.name }}
+  labels:
+    {{- include "firecrawl.labels" . | nindent 4 }}
+    app.kubernetes.io/component: postgres
+spec:
+  replicas: {{ .Values.postgres.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "firecrawl.selectorLabels" . | nindent 6 }}
+      app.kubernetes.io/component: postgres
+  template:
+    metadata:
+      labels:
+        {{- include "firecrawl.selectorLabels" . | nindent 8 }}
+        app.kubernetes.io/component: postgres
+    spec:
+      containers:
+        - name: postgres
+          image: "{{ .Values.postgres.image.repository }}:{{ .Values.postgres.image.tag }}"
+          imagePullPolicy: {{ .Values.postgres.image.pullPolicy }}
+          env:
+            - name: POSTGRES_USER
+              value: {{ .Values.postgres.auth.username | quote }}
+            - name: POSTGRES_PASSWORD
+              value: {{ .Values.postgres.auth.password | quote }}
+            - name: POSTGRES_DB
+              value: {{ .Values.postgres.auth.database | quote }}
+          ports:
+            - containerPort: 5432
+              name: postgres
+          resources:
+            {{- toYaml .Values.postgres.resources | nindent 12 }}
+          volumeMounts:
+            - name: postgres-data
+              mountPath: /var/lib/postgresql/data
+              subPath: postgres
+      volumes:
+        - name: postgres-data
+          {{- if .Values.postgres.persistence.enabled }}
+          persistentVolumeClaim:
+            claimName: {{ include "firecrawl.fullname" . }}-postgres-pvc
+          {{- else }}
+          emptyDir: {}
+          {{- end }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "firecrawl.fullname" . }}-postgres
+  namespace: {{ .Values.namespace.name }}
+  labels:
+    {{- include "firecrawl.labels" . | nindent 4 }}
+    app.kubernetes.io/component: postgres
+spec:
+  type: {{ .Values.postgres.service.type }}
+  ports:
+    - port: {{ .Values.postgres.service.port }}
+      targetPort: postgres
+      protocol: TCP
+      name: postgres
+  selector:
+    {{- include "firecrawl.selectorLabels" . | nindent 4 }}
+    app.kubernetes.io/component: postgres

cluster/applications/firecrawl/chart/templates/pvc.yaml

@@ -0,0 +1,53 @@
+{{- if .Values.postgres.persistence.enabled }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "firecrawl.fullname" . }}-postgres-pvc
+  namespace: {{ .Values.namespace.name }}
+  labels:
+    {{- include "firecrawl.labels" . | nindent 4 }}
+    app.kubernetes.io/component: postgres
+spec:
+  accessModes:
+    - {{ .Values.postgres.persistence.accessMode }}
+  storageClassName: {{ .Values.postgres.persistence.storageClass | quote }}
+  resources:
+    requests:
+      storage: {{ .Values.postgres.persistence.size | quote }}
+{{- end }}
+---
+{{- if .Values.redis.persistence.enabled }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "firecrawl.fullname" . }}-redis-pvc
+  namespace: {{ .Values.namespace.name }}
+  labels:
+    {{- include "firecrawl.labels" . | nindent 4 }}
+    app.kubernetes.io/component: redis
+spec:
+  accessModes:
+    - {{ .Values.redis.persistence.accessMode }}
+  storageClassName: {{ .Values.redis.persistence.storageClass | quote }}
+  resources:
+    requests:
+      storage: {{ .Values.redis.persistence.size | quote }}
+{{- end }}
+---
+{{- if and .Values.rabbitmq.enabled .Values.rabbitmq.persistence.enabled }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "firecrawl.fullname" . }}-rabbitmq-pvc
+  namespace: {{ .Values.namespace.name }}
+  labels:
+    {{- include "firecrawl.labels" . | nindent 4 }}
+    app.kubernetes.io/component: rabbitmq
+spec:
+  accessModes:
+    - {{ .Values.rabbitmq.persistence.accessMode }}
+  storageClassName: {{ .Values.rabbitmq.persistence.storageClass | quote }}
+  resources:
+    requests:
+      storage: {{ .Values.rabbitmq.persistence.size | quote }}
+{{- end }}

cluster/applications/firecrawl/chart/templates/rabbitmq.yaml

@@ -0,0 +1,82 @@
+{{- if .Values.rabbitmq.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "firecrawl.fullname" . }}-rabbitmq
+  namespace: {{ .Values.namespace.name }}
+  labels:
+    {{- include "firecrawl.labels" . | nindent 4 }}
+    app.kubernetes.io/component: rabbitmq
+spec:
+  replicas: {{ .Values.rabbitmq.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "firecrawl.selectorLabels" . | nindent 6 }}
+      app.kubernetes.io/component: rabbitmq
+  template:
+    metadata:
+      labels:
+        {{- include "firecrawl.selectorLabels" . | nindent 8 }}
+        app.kubernetes.io/component: rabbitmq
+    spec:
+      containers:
+        - name: rabbitmq
+          image: "{{ .Values.rabbitmq.image.repository }}:{{ .Values.rabbitmq.image.tag }}"
+          imagePullPolicy: {{ .Values.rabbitmq.image.pullPolicy }}
+          command: ["rabbitmq-server"]
+          ports:
+            - containerPort: 5672
+              name: amqp
+            - containerPort: 15672
+              name: management
+          resources:
+            {{- toYaml .Values.rabbitmq.resources | nindent 12 }}
+          livenessProbe:
+            exec:
+              command: ["rabbitmq-diagnostics", "-q", "check_running"]
+            initialDelaySeconds: 5
+            periodSeconds: 5
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            exec:
+              command: ["rabbitmq-diagnostics", "-q", "check_running"]
+            initialDelaySeconds: 5
+            periodSeconds: 5
+            timeoutSeconds: 5
+            failureThreshold: 3
+          volumeMounts:
+            {{- if .Values.rabbitmq.persistence.enabled }}
+            - name: rabbitmq-data
+              mountPath: /var/lib/rabbitmq
+            {{- end }}
+      volumes:
+        {{- if .Values.rabbitmq.persistence.enabled }}
+        - name: rabbitmq-data
+          persistentVolumeClaim:
+            claimName: {{ include "firecrawl.fullname" . }}-rabbitmq-pvc
+        {{- end }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "firecrawl.fullname" . }}-rabbitmq
+  namespace: {{ .Values.namespace.name }}
+  labels:
+    {{- include "firecrawl.labels" . | nindent 4 }}
+    app.kubernetes.io/component: rabbitmq
+spec:
+  type: {{ .Values.rabbitmq.service.type }}
+  ports:
+    - port: {{ .Values.rabbitmq.service.amqpPort }}
+      targetPort: amqp
+      protocol: TCP
+      name: amqp
+    - port: {{ .Values.rabbitmq.service.managementPort }}
+      targetPort: management
+      protocol: TCP
+      name: management
+  selector:
+    {{- include "firecrawl.selectorLabels" . | nindent 4 }}
+    app.kubernetes.io/component: rabbitmq
+{{- end }}

cluster/applications/firecrawl/chart/templates/redis.yaml

@@ -0,0 +1,73 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "firecrawl.fullname" . }}-redis
+  namespace: {{ .Values.namespace.name }}
+  labels:
+    {{- include "firecrawl.labels" . | nindent 4 }}
+    app.kubernetes.io/component: redis
+spec:
+  replicas: {{ .Values.redis.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "firecrawl.selectorLabels" . | nindent 6 }}
+      app.kubernetes.io/component: redis
+  template:
+    metadata:
+      labels:
+        {{- include "firecrawl.selectorLabels" . | nindent 8 }}
+        app.kubernetes.io/component: redis
+    spec:
+      containers:
+        - name: redis
+          image: "{{ .Values.redis.image.repository }}:{{ .Values.redis.image.tag }}"
+          imagePullPolicy: {{ .Values.redis.image.pullPolicy }}
+          command: ["redis-server"]
+          args:
+            - "--bind"
+            - "0.0.0.0"
+            {{- if .Values.redis.persistence.enabled }}
+            - "--appendonly"
+            - "yes"
+            - "--save"
+            - "900 1"
+            - "--save"
+            - "300 10"
+            - "--save"
+            - "60 10000"
+            {{- end }}
+          ports:
+            - containerPort: 6379
+              name: redis
+          resources:
+            {{- toYaml .Values.redis.resources | nindent 12 }}
+          volumeMounts:
+            {{- if .Values.redis.persistence.enabled }}
+            - name: redis-data
+              mountPath: /data
+            {{- end }}
+      volumes:
+        {{- if .Values.redis.persistence.enabled }}
+        - name: redis-data
+          persistentVolumeClaim:
+            claimName: {{ include "firecrawl.fullname" . }}-redis-pvc
+        {{- end }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "firecrawl.fullname" . }}-redis
+  namespace: {{ .Values.namespace.name }}
+  labels:
+    {{- include "firecrawl.labels" . | nindent 4 }}
+    app.kubernetes.io/component: redis
+spec:
+  type: {{ .Values.redis.service.type }}
+  ports:
+    - port: {{ .Values.redis.service.port }}
+      targetPort: redis
+      protocol: TCP
+      name: redis
+  selector:
+    {{- include "firecrawl.selectorLabels" . | nindent 4 }}
+    app.kubernetes.io/component: redis

cluster/applications/firecrawl/chart/templates/worker.yaml

@@ -0,0 +1,39 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "firecrawl.fullname" . }}-worker
+  namespace: {{ .Values.namespace.name }}
+  labels:
+    {{- include "firecrawl.labels" . | nindent 4 }}
+    app.kubernetes.io/component: worker
+spec:
+  replicas: {{ .Values.worker.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "firecrawl.selectorLabels" . | nindent 6 }}
+      app.kubernetes.io/component: worker
+  template:
+    metadata:
+      labels:
+        {{- include "firecrawl.selectorLabels" . | nindent 8 }}
+        app.kubernetes.io/component: worker
+    spec:
+      terminationGracePeriodSeconds: 180
+      containers:
+        - name: worker
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          command: ["node"]
+          args: ["--max-old-space-size=4096", "dist/src/services/queue-worker.js"]
+          envFrom:
+            - configMapRef:
+                name: {{ include "firecrawl.fullname" . }}-config
+          env:
+            - name: FLY_PROCESS_GROUP
+              value: "worker"
+            - name: NUQ_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+          resources:
+            {{- toYaml .Values.worker.resources | nindent 12 }}