Compare commits
5 Commits
f8e137b67b
...
818b6505dd
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
818b6505dd | ||
|
|
4a1958876f | ||
|
|
6bdb536848 | ||
|
|
6023ee25e1 | ||
|
|
e5d24f557a |
59
cluster/applications/firecrawl/SECRETS_SETUP.md
Normal file
59
cluster/applications/firecrawl/SECRETS_SETUP.md
Normal file
@@ -0,0 +1,59 @@
|
|||||||
|
# Firecrawl Secrets for 1Password
|
||||||
|
|
||||||
|
## Action Required: Manual Secret Creation in 1Password
|
||||||
|
|
||||||
|
The 1Password Connect token is READ-ONLY for security. These secrets must be manually added to 1Password.
|
||||||
|
|
||||||
|
### Vault Information
|
||||||
|
- **Vault Name:** mk-labs
|
||||||
|
- **Vault ID:** mnefk7klz35qqvv2eucygh4pzm
|
||||||
|
|
||||||
|
### Item to Create
|
||||||
|
- **Item Name:** firecrawl
|
||||||
|
- **Item Type:** Server / API Credential
|
||||||
|
|
||||||
|
### Fields to Add
|
||||||
|
|
||||||
|
#### 1. postgres-password
|
||||||
|
- **Label:** postgres-password
|
||||||
|
- **Type:** Password (concealed)
|
||||||
|
- **Value:** `ll1fTJSer8h0Bwq7PBmkTcRbAlFKGzbV`
|
||||||
|
- **Purpose:** PostgreSQL database authentication
|
||||||
|
|
||||||
|
#### 2. bull-auth-key
|
||||||
|
- **Label:** bull-auth-key
|
||||||
|
- **Type:** Password (concealed)
|
||||||
|
- **Value:** `fCHhkTNeplAwI2scqGo0C4cwJGA33FBpmyIVxlbzkodRbFKy5WlroxJeezjYg1UQ`
|
||||||
|
- **Purpose:** Redis Bull queue admin UI authentication
|
||||||
|
|
||||||
|
## Instructions
|
||||||
|
|
||||||
|
1. Open 1Password desktop app or web interface
|
||||||
|
2. Navigate to the "mk-labs" vault
|
||||||
|
3. Create a new item:
|
||||||
|
- Name: `firecrawl`
|
||||||
|
- Type: `Server` or `Password`
|
||||||
|
4. Add custom fields:
|
||||||
|
- Field 1: Label = `postgres-password`, Value = (32-char password above)
|
||||||
|
- Field 2: Label = `bull-auth-key`, Value = (64-char key above)
|
||||||
|
5. Save the item
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
After creating the item in 1Password, verify it's accessible:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Port-forward to 1Password Connect (if not already running)
|
||||||
|
kubectl port-forward -n onepassword-connect svc/onepassword-connect 8080:8080 &
|
||||||
|
|
||||||
|
# Test retrieval (read-only access should work)
|
||||||
|
curl -s -H "Authorization: Bearer *** kubectl get secret -n onepassword-connect connect-token -o jsonpath='{.data.token}' | base64 -d)" \
|
||||||
|
"http://localhost:8080/v1/vaults/mnefk7klz35qqvv2eucygh4pzm/items" | \
|
||||||
|
jq -r '.[] | select(.title == "firecrawl") | .title'
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected output: `firecrawl`
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
|
||||||
|
Once the item is created in 1Password, the ExternalSecret manifest will automatically sync these values to Kubernetes.
|
||||||
34
cluster/applications/firecrawl/application.yaml
Normal file
34
cluster/applications/firecrawl/application.yaml
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Application: firecrawl
|
||||||
|
# Wave 20 — applications tier
|
||||||
|
# AI-powered web scraping and crawling service with Redis, PostgreSQL, Playwright
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Application
|
||||||
|
metadata:
|
||||||
|
name: firecrawl
|
||||||
|
namespace: argocd
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: firecrawl
|
||||||
|
app.kubernetes.io/part-of: mk-labs
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-wave: "20" # Wave 20 — applications tier
|
||||||
|
spec:
|
||||||
|
project: default
|
||||||
|
source:
|
||||||
|
repoURL: https://gitea.mk-labs.cloud/rblundon/homelab.git
|
||||||
|
targetRevision: main
|
||||||
|
path: cluster/applications/firecrawl
|
||||||
|
directory:
|
||||||
|
recurse: false
|
||||||
|
exclude: application.yaml # prevent self-reference loop
|
||||||
|
destination:
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
namespace: firecrawl
|
||||||
|
syncPolicy:
|
||||||
|
automated:
|
||||||
|
prune: true
|
||||||
|
selfHeal: true
|
||||||
|
syncOptions:
|
||||||
|
- CreateNamespace=true
|
||||||
|
- ApplyOutOfSyncOnly=true
|
||||||
13
cluster/applications/firecrawl/configmap-playwright.yaml
Normal file
13
cluster/applications/firecrawl/configmap-playwright.yaml
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: playwright-service-config
|
||||||
|
namespace: firecrawl
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: firecrawl
|
||||||
|
app.kubernetes.io/component: browser
|
||||||
|
app.kubernetes.io/part-of: firecrawl
|
||||||
|
data:
|
||||||
|
PORT: "3000"
|
||||||
|
ALLOW_LOCAL_WEBHOOKS: "false"
|
||||||
|
MAX_CONCURRENT_PAGES: "10"
|
||||||
45
cluster/applications/firecrawl/configmap.yaml
Normal file
45
cluster/applications/firecrawl/configmap.yaml
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: firecrawl-config
|
||||||
|
namespace: firecrawl
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: firecrawl
|
||||||
|
app.kubernetes.io/component: configuration
|
||||||
|
app.kubernetes.io/part-of: firecrawl
|
||||||
|
data:
|
||||||
|
# Service URLs (Kubernetes internal DNS)
|
||||||
|
REDIS_URL: "redis://redis:6379"
|
||||||
|
REDIS_RATE_LIMIT_URL: "redis://redis:6379"
|
||||||
|
POSTGRES_HOST: "nuq-postgres"
|
||||||
|
POSTGRES_PORT: "5432"
|
||||||
|
POSTGRES_USER: "postgres"
|
||||||
|
POSTGRES_DB: "postgres"
|
||||||
|
NUQ_RABBITMQ_URL: "amqp://rabbitmq:5672"
|
||||||
|
PLAYWRIGHT_MICROSERVICE_URL: "http://playwright-service:3000/scrape"
|
||||||
|
|
||||||
|
# Server Configuration
|
||||||
|
HOST: "0.0.0.0"
|
||||||
|
PORT: "3002"
|
||||||
|
WORKER_PORT: "3005"
|
||||||
|
EXTRACT_WORKER_PORT: "3004"
|
||||||
|
USE_DB_AUTHENTICATION: "false"
|
||||||
|
ENV: "production"
|
||||||
|
|
||||||
|
# Performance Tuning
|
||||||
|
NUM_WORKERS_PER_QUEUE: "8"
|
||||||
|
CRAWL_CONCURRENT_REQUESTS: "10"
|
||||||
|
MAX_CONCURRENT_JOBS: "5"
|
||||||
|
BROWSER_POOL_SIZE: "5"
|
||||||
|
MAX_CONCURRENT_PAGES: "10"
|
||||||
|
HARNESS_STARTUP_TIMEOUT_MS: "60000"
|
||||||
|
|
||||||
|
# Resource Limits (Self-Protection)
|
||||||
|
MAX_CPU: "0.8"
|
||||||
|
MAX_RAM: "0.8"
|
||||||
|
|
||||||
|
# Logging
|
||||||
|
LOGGING_LEVEL: "info"
|
||||||
|
|
||||||
|
# Security
|
||||||
|
ALLOW_LOCAL_WEBHOOKS: "false"
|
||||||
60
cluster/applications/firecrawl/deployment-api.yaml
Normal file
60
cluster/applications/firecrawl/deployment-api.yaml
Normal file
@@ -0,0 +1,60 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: firecrawl-api
|
||||||
|
namespace: firecrawl
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: firecrawl
|
||||||
|
app.kubernetes.io/component: api
|
||||||
|
app.kubernetes.io/part-of: firecrawl
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: firecrawl-api
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: firecrawl-api
|
||||||
|
app.kubernetes.io/name: firecrawl
|
||||||
|
app.kubernetes.io/component: api
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: api
|
||||||
|
image: the-seas.local.mk-labs.cloud/library/firecrawl-api:latest
|
||||||
|
imagePullPolicy: Always
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: 3002
|
||||||
|
protocol: TCP
|
||||||
|
envFrom:
|
||||||
|
- configMapRef:
|
||||||
|
name: firecrawl-config
|
||||||
|
- secretRef:
|
||||||
|
name: firecrawl-secrets
|
||||||
|
env:
|
||||||
|
- name: FLY_PROCESS_GROUP
|
||||||
|
value: "app"
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 1000m
|
||||||
|
memory: 4Gi
|
||||||
|
limits:
|
||||||
|
cpu: 2000m
|
||||||
|
memory: 6Gi
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /v0/health/liveness
|
||||||
|
port: 3002
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
failureThreshold: 3
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /v0/health/readiness
|
||||||
|
port: 3002
|
||||||
|
initialDelaySeconds: 30
|
||||||
|
periodSeconds: 5
|
||||||
|
timeoutSeconds: 3
|
||||||
|
failureThreshold: 3
|
||||||
62
cluster/applications/firecrawl/deployment-nuq-worker.yaml
Normal file
62
cluster/applications/firecrawl/deployment-nuq-worker.yaml
Normal file
@@ -0,0 +1,62 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: firecrawl-api-nuq-worker
|
||||||
|
namespace: firecrawl
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: firecrawl
|
||||||
|
app.kubernetes.io/component: nuq-worker
|
||||||
|
app.kubernetes.io/part-of: firecrawl
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: firecrawl-api-nuq-worker
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: firecrawl-api-nuq-worker
|
||||||
|
app.kubernetes.io/name: firecrawl
|
||||||
|
app.kubernetes.io/component: nuq-worker
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: nuq-worker
|
||||||
|
image: the-seas.local.mk-labs.cloud/library/firecrawl-api:latest
|
||||||
|
imagePullPolicy: Always
|
||||||
|
command: ["node"]
|
||||||
|
args: ["dist/src/services/worker/nuq-worker.js"]
|
||||||
|
envFrom:
|
||||||
|
- configMapRef:
|
||||||
|
name: firecrawl-config
|
||||||
|
- secretRef:
|
||||||
|
name: firecrawl-secrets
|
||||||
|
env:
|
||||||
|
- name: FLY_PROCESS_GROUP
|
||||||
|
value: "nuq-worker"
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 3Gi
|
||||||
|
limits:
|
||||||
|
cpu: 1000m
|
||||||
|
memory: 4Gi
|
||||||
|
livenessProbe:
|
||||||
|
exec:
|
||||||
|
command:
|
||||||
|
- pgrep
|
||||||
|
- -f
|
||||||
|
- nuq-worker
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 30
|
||||||
|
timeoutSeconds: 5
|
||||||
|
failureThreshold: 3
|
||||||
|
readinessProbe:
|
||||||
|
exec:
|
||||||
|
command:
|
||||||
|
- pgrep
|
||||||
|
- -f
|
||||||
|
- nuq-worker
|
||||||
|
initialDelaySeconds: 30
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 3
|
||||||
|
failureThreshold: 3
|
||||||
63
cluster/applications/firecrawl/deployment-playwright.yaml
Normal file
63
cluster/applications/firecrawl/deployment-playwright.yaml
Normal file
@@ -0,0 +1,63 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: playwright-service
|
||||||
|
namespace: firecrawl
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: firecrawl
|
||||||
|
app.kubernetes.io/component: browser
|
||||||
|
app.kubernetes.io/part-of: firecrawl
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: playwright-service
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: playwright-service
|
||||||
|
app.kubernetes.io/name: firecrawl
|
||||||
|
app.kubernetes.io/component: browser
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: playwright
|
||||||
|
image: the-seas.local.mk-labs.cloud/library/firecrawl-playwright:latest
|
||||||
|
imagePullPolicy: Always
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: 3000
|
||||||
|
protocol: TCP
|
||||||
|
envFrom:
|
||||||
|
- configMapRef:
|
||||||
|
name: playwright-service-config
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 1000m
|
||||||
|
memory: 2Gi
|
||||||
|
limits:
|
||||||
|
cpu: 2000m
|
||||||
|
memory: 4Gi
|
||||||
|
volumeMounts:
|
||||||
|
- name: tmpfs
|
||||||
|
mountPath: /tmp
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /
|
||||||
|
port: 3000
|
||||||
|
initialDelaySeconds: 30
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
failureThreshold: 3
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /
|
||||||
|
port: 3000
|
||||||
|
initialDelaySeconds: 5
|
||||||
|
periodSeconds: 5
|
||||||
|
timeoutSeconds: 3
|
||||||
|
failureThreshold: 3
|
||||||
|
volumes:
|
||||||
|
- name: tmpfs
|
||||||
|
emptyDir:
|
||||||
|
medium: Memory
|
||||||
|
sizeLimit: 1Gi
|
||||||
83
cluster/applications/firecrawl/deployment-postgres.yaml
Normal file
83
cluster/applications/firecrawl/deployment-postgres.yaml
Normal file
@@ -0,0 +1,83 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: nuq-postgres
|
||||||
|
namespace: firecrawl
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: firecrawl
|
||||||
|
app.kubernetes.io/component: database
|
||||||
|
app.kubernetes.io/part-of: firecrawl
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
strategy:
|
||||||
|
type: Recreate
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: nuq-postgres
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: nuq-postgres
|
||||||
|
app.kubernetes.io/name: firecrawl
|
||||||
|
app.kubernetes.io/component: database
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: postgres
|
||||||
|
image: the-seas.local.mk-labs.cloud/library/firecrawl-postgres:latest
|
||||||
|
imagePullPolicy: Always
|
||||||
|
ports:
|
||||||
|
- name: postgres
|
||||||
|
containerPort: 5432
|
||||||
|
protocol: TCP
|
||||||
|
env:
|
||||||
|
- name: POSTGRES_USER
|
||||||
|
valueFrom:
|
||||||
|
configMapKeyRef:
|
||||||
|
name: firecrawl-config
|
||||||
|
key: POSTGRES_USER
|
||||||
|
- name: POSTGRES_DB
|
||||||
|
valueFrom:
|
||||||
|
configMapKeyRef:
|
||||||
|
name: firecrawl-config
|
||||||
|
key: POSTGRES_DB
|
||||||
|
- name: POSTGRES_PASSWORD
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: firecrawl-secrets
|
||||||
|
key: POSTGRES_PASSWORD
|
||||||
|
- name: PGDATA
|
||||||
|
value: /var/lib/postgresql/data/pgdata
|
||||||
|
volumeMounts:
|
||||||
|
- name: postgres-data
|
||||||
|
mountPath: /var/lib/postgresql/data
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 1Gi
|
||||||
|
limits:
|
||||||
|
cpu: 1000m
|
||||||
|
memory: 2Gi
|
||||||
|
livenessProbe:
|
||||||
|
exec:
|
||||||
|
command:
|
||||||
|
- pg_isready
|
||||||
|
- -U
|
||||||
|
- postgres
|
||||||
|
initialDelaySeconds: 30
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
failureThreshold: 3
|
||||||
|
readinessProbe:
|
||||||
|
exec:
|
||||||
|
command:
|
||||||
|
- pg_isready
|
||||||
|
- -U
|
||||||
|
- postgres
|
||||||
|
initialDelaySeconds: 5
|
||||||
|
periodSeconds: 5
|
||||||
|
timeoutSeconds: 3
|
||||||
|
failureThreshold: 3
|
||||||
|
volumes:
|
||||||
|
- name: postgres-data
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: postgres-data
|
||||||
61
cluster/applications/firecrawl/deployment-rabbitmq.yaml
Normal file
61
cluster/applications/firecrawl/deployment-rabbitmq.yaml
Normal file
@@ -0,0 +1,61 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: rabbitmq
|
||||||
|
namespace: firecrawl
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: firecrawl
|
||||||
|
app.kubernetes.io/component: messaging
|
||||||
|
app.kubernetes.io/part-of: firecrawl
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
strategy:
|
||||||
|
type: Recreate
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: rabbitmq
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: rabbitmq
|
||||||
|
app.kubernetes.io/name: firecrawl
|
||||||
|
app.kubernetes.io/component: messaging
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: rabbitmq
|
||||||
|
image: rabbitmq:3-management
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
ports:
|
||||||
|
- name: amqp
|
||||||
|
containerPort: 5672
|
||||||
|
protocol: TCP
|
||||||
|
- name: management
|
||||||
|
containerPort: 15672
|
||||||
|
protocol: TCP
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 250m
|
||||||
|
memory: 512Mi
|
||||||
|
limits:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 1Gi
|
||||||
|
livenessProbe:
|
||||||
|
exec:
|
||||||
|
command:
|
||||||
|
- rabbitmq-diagnostics
|
||||||
|
- -q
|
||||||
|
- ping
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 30
|
||||||
|
timeoutSeconds: 10
|
||||||
|
failureThreshold: 3
|
||||||
|
readinessProbe:
|
||||||
|
exec:
|
||||||
|
command:
|
||||||
|
- rabbitmq-diagnostics
|
||||||
|
- -q
|
||||||
|
- check_running
|
||||||
|
initialDelaySeconds: 10
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
failureThreshold: 3
|
||||||
56
cluster/applications/firecrawl/deployment-redis.yaml
Normal file
56
cluster/applications/firecrawl/deployment-redis.yaml
Normal file
@@ -0,0 +1,56 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: redis
|
||||||
|
namespace: firecrawl
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: firecrawl
|
||||||
|
app.kubernetes.io/component: cache
|
||||||
|
app.kubernetes.io/part-of: firecrawl
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
strategy:
|
||||||
|
type: Recreate
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: redis
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: redis
|
||||||
|
app.kubernetes.io/name: firecrawl
|
||||||
|
app.kubernetes.io/component: cache
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: redis
|
||||||
|
image: redis:alpine
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
ports:
|
||||||
|
- name: redis
|
||||||
|
containerPort: 6379
|
||||||
|
protocol: TCP
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 250m
|
||||||
|
memory: 512Mi
|
||||||
|
limits:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 1Gi
|
||||||
|
livenessProbe:
|
||||||
|
exec:
|
||||||
|
command:
|
||||||
|
- redis-cli
|
||||||
|
- ping
|
||||||
|
initialDelaySeconds: 30
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
failureThreshold: 3
|
||||||
|
readinessProbe:
|
||||||
|
exec:
|
||||||
|
command:
|
||||||
|
- redis-cli
|
||||||
|
- ping
|
||||||
|
initialDelaySeconds: 5
|
||||||
|
periodSeconds: 5
|
||||||
|
timeoutSeconds: 3
|
||||||
|
failureThreshold: 3
|
||||||
60
cluster/applications/firecrawl/deployment-worker.yaml
Normal file
60
cluster/applications/firecrawl/deployment-worker.yaml
Normal file
@@ -0,0 +1,60 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: firecrawl-api-worker
|
||||||
|
namespace: firecrawl
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: firecrawl
|
||||||
|
app.kubernetes.io/component: worker
|
||||||
|
app.kubernetes.io/part-of: firecrawl
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: firecrawl-api-worker
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: firecrawl-api-worker
|
||||||
|
app.kubernetes.io/name: firecrawl
|
||||||
|
app.kubernetes.io/component: worker
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: worker
|
||||||
|
image: the-seas.local.mk-labs.cloud/library/firecrawl-api:latest
|
||||||
|
imagePullPolicy: Always
|
||||||
|
command: ["node"]
|
||||||
|
args: ["dist/src/services/queue-worker.js"]
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: 3005
|
||||||
|
protocol: TCP
|
||||||
|
envFrom:
|
||||||
|
- configMapRef:
|
||||||
|
name: firecrawl-config
|
||||||
|
- secretRef:
|
||||||
|
name: firecrawl-secrets
|
||||||
|
env:
|
||||||
|
- name: FLY_PROCESS_GROUP
|
||||||
|
value: "worker"
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 3Gi
|
||||||
|
limits:
|
||||||
|
cpu: 1000m
|
||||||
|
memory: 4Gi
|
||||||
|
livenessProbe:
|
||||||
|
tcpSocket:
|
||||||
|
port: 3005
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
failureThreshold: 3
|
||||||
|
readinessProbe:
|
||||||
|
tcpSocket:
|
||||||
|
port: 3005
|
||||||
|
initialDelaySeconds: 30
|
||||||
|
periodSeconds: 5
|
||||||
|
timeoutSeconds: 3
|
||||||
|
failureThreshold: 3
|
||||||
34
cluster/applications/firecrawl/externalsecret.yaml
Normal file
34
cluster/applications/firecrawl/externalsecret.yaml
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
apiVersion: external-secrets.io/v1beta1
|
||||||
|
kind: ExternalSecret
|
||||||
|
metadata:
|
||||||
|
name: firecrawl-secrets
|
||||||
|
namespace: firecrawl
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: firecrawl
|
||||||
|
app.kubernetes.io/component: secrets
|
||||||
|
app.kubernetes.io/part-of: firecrawl
|
||||||
|
spec:
|
||||||
|
refreshInterval: 1h
|
||||||
|
secretStoreRef:
|
||||||
|
kind: ClusterSecretStore
|
||||||
|
name: onepassword-connect
|
||||||
|
target:
|
||||||
|
name: firecrawl-secrets
|
||||||
|
creationPolicy: Owner
|
||||||
|
template:
|
||||||
|
engineVersion: v2
|
||||||
|
type: Opaque
|
||||||
|
data:
|
||||||
|
# PostgreSQL database password
|
||||||
|
POSTGRES_PASSWORD: "{{ .postgresPassword }}"
|
||||||
|
# Redis Bull queue admin UI authentication key
|
||||||
|
BULL_AUTH_KEY: "{{ .bullAuthKey }}"
|
||||||
|
data:
|
||||||
|
- secretKey: postgresPassword
|
||||||
|
remoteRef:
|
||||||
|
key: firecrawl
|
||||||
|
property: postgres-password
|
||||||
|
- secretKey: bullAuthKey
|
||||||
|
remoteRef:
|
||||||
|
key: firecrawl
|
||||||
|
property: bull-auth-key
|
||||||
26
cluster/applications/firecrawl/httproute.yaml
Normal file
26
cluster/applications/firecrawl/httproute.yaml
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
apiVersion: gateway.networking.k8s.io/v1
|
||||||
|
kind: HTTPRoute
|
||||||
|
metadata:
|
||||||
|
name: firecrawl
|
||||||
|
namespace: firecrawl
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: firecrawl
|
||||||
|
app.kubernetes.io/component: ingress
|
||||||
|
app.kubernetes.io/part-of: firecrawl
|
||||||
|
annotations:
|
||||||
|
cert-manager.io/cluster-issuer: letsencrypt-prod
|
||||||
|
spec:
|
||||||
|
parentRefs:
|
||||||
|
- name: fastpass-gateway
|
||||||
|
namespace: gateway
|
||||||
|
hostnames:
|
||||||
|
- spaceship-earth.local.mk-labs.cloud
|
||||||
|
- firecrawl.local.mk-labs.cloud
|
||||||
|
rules:
|
||||||
|
- matches:
|
||||||
|
- path:
|
||||||
|
type: PathPrefix
|
||||||
|
value: /
|
||||||
|
backendRefs:
|
||||||
|
- name: firecrawl-api
|
||||||
|
port: 3002
|
||||||
13
cluster/applications/firecrawl/namespace.yaml
Normal file
13
cluster/applications/firecrawl/namespace.yaml
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: firecrawl
|
||||||
|
labels:
|
||||||
|
name: firecrawl
|
||||||
|
app.kubernetes.io/name: firecrawl
|
||||||
|
app.kubernetes.io/component: application
|
||||||
|
app.kubernetes.io/part-of: platform-buildout
|
||||||
|
epcot-theme: spaceship-earth
|
||||||
|
annotations:
|
||||||
|
description: "Firecrawl web scraping and crawling service - enables JARVIS web search capability"
|
||||||
17
cluster/applications/firecrawl/postgres-pvc.yaml
Normal file
17
cluster/applications/firecrawl/postgres-pvc.yaml
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: PersistentVolumeClaim
|
||||||
|
metadata:
|
||||||
|
name: postgres-data
|
||||||
|
namespace: firecrawl
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: firecrawl
|
||||||
|
app.kubernetes.io/component: database
|
||||||
|
app.kubernetes.io/part-of: firecrawl
|
||||||
|
spec:
|
||||||
|
accessModes:
|
||||||
|
- ReadWriteOnce
|
||||||
|
storageClassName: nfs-emporium
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 10Gi
|
||||||
18
cluster/applications/firecrawl/service-api.yaml
Normal file
18
cluster/applications/firecrawl/service-api.yaml
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: firecrawl-api
|
||||||
|
namespace: firecrawl
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: firecrawl
|
||||||
|
app.kubernetes.io/component: api
|
||||||
|
app.kubernetes.io/part-of: firecrawl
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
selector:
|
||||||
|
app: firecrawl-api
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
port: 3002
|
||||||
|
targetPort: 3002
|
||||||
|
protocol: TCP
|
||||||
18
cluster/applications/firecrawl/service-playwright.yaml
Normal file
18
cluster/applications/firecrawl/service-playwright.yaml
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: playwright-service
|
||||||
|
namespace: firecrawl
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: firecrawl
|
||||||
|
app.kubernetes.io/component: browser
|
||||||
|
app.kubernetes.io/part-of: firecrawl
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
selector:
|
||||||
|
app: playwright-service
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
port: 3000
|
||||||
|
targetPort: 3000
|
||||||
|
protocol: TCP
|
||||||
18
cluster/applications/firecrawl/service-postgres.yaml
Normal file
18
cluster/applications/firecrawl/service-postgres.yaml
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: nuq-postgres
|
||||||
|
namespace: firecrawl
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: firecrawl
|
||||||
|
app.kubernetes.io/component: database
|
||||||
|
app.kubernetes.io/part-of: firecrawl
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
selector:
|
||||||
|
app: nuq-postgres
|
||||||
|
ports:
|
||||||
|
- name: postgres
|
||||||
|
port: 5432
|
||||||
|
targetPort: 5432
|
||||||
|
protocol: TCP
|
||||||
22
cluster/applications/firecrawl/service-rabbitmq.yaml
Normal file
22
cluster/applications/firecrawl/service-rabbitmq.yaml
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: rabbitmq
|
||||||
|
namespace: firecrawl
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: firecrawl
|
||||||
|
app.kubernetes.io/component: messaging
|
||||||
|
app.kubernetes.io/part-of: firecrawl
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
selector:
|
||||||
|
app: rabbitmq
|
||||||
|
ports:
|
||||||
|
- name: amqp
|
||||||
|
port: 5672
|
||||||
|
targetPort: 5672
|
||||||
|
protocol: TCP
|
||||||
|
- name: management
|
||||||
|
port: 15672
|
||||||
|
targetPort: 15672
|
||||||
|
protocol: TCP
|
||||||
18
cluster/applications/firecrawl/service-redis.yaml
Normal file
18
cluster/applications/firecrawl/service-redis.yaml
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: redis
|
||||||
|
namespace: firecrawl
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: firecrawl
|
||||||
|
app.kubernetes.io/component: cache
|
||||||
|
app.kubernetes.io/part-of: firecrawl
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
selector:
|
||||||
|
app: redis
|
||||||
|
ports:
|
||||||
|
- name: redis
|
||||||
|
port: 6379
|
||||||
|
targetPort: 6379
|
||||||
|
protocol: TCP
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
---
|
||||||
|
apiVersion: tekton.dev/v1
|
||||||
|
kind: Pipeline
|
||||||
|
metadata:
|
||||||
|
name: firecrawl-api-pipeline
|
||||||
|
namespace: innoventions
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: firecrawl-api
|
||||||
|
app.kubernetes.io/component: pipeline
|
||||||
|
app.kubernetes.io/part-of: firecrawl
|
||||||
|
spec:
|
||||||
|
description: |
|
||||||
|
Pipeline to build the Firecrawl API container image.
|
||||||
|
Multi-stage build with Go, Node.js, and Rust dependencies.
|
||||||
|
params:
|
||||||
|
- name: git-url
|
||||||
|
type: string
|
||||||
|
description: Git repository URL
|
||||||
|
default: https://github.com/mendableai/firecrawl
|
||||||
|
- name: git-revision
|
||||||
|
type: string
|
||||||
|
description: Git revision to checkout
|
||||||
|
default: main
|
||||||
|
- name: image-name
|
||||||
|
type: string
|
||||||
|
description: Target image name
|
||||||
|
default: the-seas.local.mk-labs.cloud/library/firecrawl-api
|
||||||
|
- name: image-tag
|
||||||
|
type: string
|
||||||
|
description: Image tag
|
||||||
|
default: latest
|
||||||
|
- name: dockerfile-path
|
||||||
|
type: string
|
||||||
|
description: Path to Dockerfile relative to repo root
|
||||||
|
default: ./Dockerfile
|
||||||
|
- name: build-context
|
||||||
|
type: string
|
||||||
|
description: Build context path
|
||||||
|
default: ./apps/api
|
||||||
|
workspaces:
|
||||||
|
- name: shared-data
|
||||||
|
description: Workspace for source code
|
||||||
|
- name: docker-credentials
|
||||||
|
description: Docker config for Harbor authentication
|
||||||
|
tasks:
|
||||||
|
- name: fetch-source
|
||||||
|
taskRef:
|
||||||
|
name: git-clone
|
||||||
|
workspaces:
|
||||||
|
- name: output
|
||||||
|
workspace: shared-data
|
||||||
|
params:
|
||||||
|
- name: url
|
||||||
|
value: $(params.git-url)
|
||||||
|
- name: revision
|
||||||
|
value: $(params.git-revision)
|
||||||
|
- name: depth
|
||||||
|
value: "1"
|
||||||
|
- name: sslVerify
|
||||||
|
value: "false"
|
||||||
|
- name: deleteExisting
|
||||||
|
value: "true"
|
||||||
|
- name: build-push
|
||||||
|
taskRef:
|
||||||
|
name: kaniko-build
|
||||||
|
runAfter:
|
||||||
|
- fetch-source
|
||||||
|
workspaces:
|
||||||
|
- name: source
|
||||||
|
workspace: shared-data
|
||||||
|
- name: dockerconfig
|
||||||
|
workspace: docker-credentials
|
||||||
|
params:
|
||||||
|
- name: IMAGE
|
||||||
|
value: $(params.image-name):$(params.image-tag)
|
||||||
|
- name: DOCKERFILE
|
||||||
|
value: $(params.dockerfile-path)
|
||||||
|
- name: CONTEXT
|
||||||
|
value: $(params.build-context)
|
||||||
|
- name: EXTRA_ARGS
|
||||||
|
value:
|
||||||
|
- --insecure
|
||||||
|
- --skip-tls-verify
|
||||||
|
- --cache=true
|
||||||
|
- --cache-ttl=24h
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
---
|
||||||
|
apiVersion: tekton.dev/v1
|
||||||
|
kind: Pipeline
|
||||||
|
metadata:
|
||||||
|
name: firecrawl-playwright-pipeline
|
||||||
|
namespace: innoventions
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: firecrawl-playwright
|
||||||
|
app.kubernetes.io/component: pipeline
|
||||||
|
app.kubernetes.io/part-of: firecrawl
|
||||||
|
spec:
|
||||||
|
description: |
|
||||||
|
Pipeline to build the Firecrawl Playwright service container image.
|
||||||
|
Node.js application with Chromium browser dependencies.
|
||||||
|
params:
|
||||||
|
- name: git-url
|
||||||
|
type: string
|
||||||
|
description: Git repository URL
|
||||||
|
default: https://github.com/mendableai/firecrawl
|
||||||
|
- name: git-revision
|
||||||
|
type: string
|
||||||
|
description: Git revision to checkout
|
||||||
|
default: main
|
||||||
|
- name: image-name
|
||||||
|
type: string
|
||||||
|
description: Target image name
|
||||||
|
default: the-seas.local.mk-labs.cloud/library/firecrawl-playwright
|
||||||
|
- name: image-tag
|
||||||
|
type: string
|
||||||
|
description: Image tag
|
||||||
|
default: latest
|
||||||
|
- name: dockerfile-path
|
||||||
|
type: string
|
||||||
|
description: Path to Dockerfile relative to repo root
|
||||||
|
default: ./Dockerfile
|
||||||
|
- name: build-context
|
||||||
|
type: string
|
||||||
|
description: Build context path
|
||||||
|
default: ./apps/playwright-service-ts
|
||||||
|
workspaces:
|
||||||
|
- name: shared-data
|
||||||
|
description: Workspace for source code
|
||||||
|
- name: docker-credentials
|
||||||
|
description: Docker config for Harbor authentication
|
||||||
|
tasks:
|
||||||
|
- name: fetch-source
|
||||||
|
taskRef:
|
||||||
|
name: git-clone
|
||||||
|
workspaces:
|
||||||
|
- name: output
|
||||||
|
workspace: shared-data
|
||||||
|
params:
|
||||||
|
- name: url
|
||||||
|
value: $(params.git-url)
|
||||||
|
- name: revision
|
||||||
|
value: $(params.git-revision)
|
||||||
|
- name: depth
|
||||||
|
value: "1"
|
||||||
|
- name: sslVerify
|
||||||
|
value: "false"
|
||||||
|
- name: deleteExisting
|
||||||
|
value: "true"
|
||||||
|
- name: build-push
|
||||||
|
taskRef:
|
||||||
|
name: kaniko-build
|
||||||
|
runAfter:
|
||||||
|
- fetch-source
|
||||||
|
workspaces:
|
||||||
|
- name: source
|
||||||
|
workspace: shared-data
|
||||||
|
- name: dockerconfig
|
||||||
|
workspace: docker-credentials
|
||||||
|
params:
|
||||||
|
- name: IMAGE
|
||||||
|
value: $(params.image-name):$(params.image-tag)
|
||||||
|
- name: DOCKERFILE
|
||||||
|
value: $(params.dockerfile-path)
|
||||||
|
- name: CONTEXT
|
||||||
|
value: $(params.build-context)
|
||||||
|
- name: EXTRA_ARGS
|
||||||
|
value:
|
||||||
|
- --insecure
|
||||||
|
- --skip-tls-verify
|
||||||
|
- --cache=true
|
||||||
|
- --cache-ttl=24h
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
---
|
||||||
|
apiVersion: tekton.dev/v1
|
||||||
|
kind: Pipeline
|
||||||
|
metadata:
|
||||||
|
name: firecrawl-postgres-pipeline
|
||||||
|
namespace: innoventions
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: firecrawl-postgres
|
||||||
|
app.kubernetes.io/component: pipeline
|
||||||
|
app.kubernetes.io/part-of: firecrawl
|
||||||
|
spec:
|
||||||
|
description: |
|
||||||
|
Pipeline to build the Firecrawl PostgreSQL container image.
|
||||||
|
PostgreSQL with pg_cron extension and custom configuration.
|
||||||
|
params:
|
||||||
|
- name: git-url
|
||||||
|
type: string
|
||||||
|
description: Git repository URL
|
||||||
|
default: https://github.com/mendableai/firecrawl
|
||||||
|
- name: git-revision
|
||||||
|
type: string
|
||||||
|
description: Git revision to checkout
|
||||||
|
default: main
|
||||||
|
- name: image-name
|
||||||
|
type: string
|
||||||
|
description: Target image name
|
||||||
|
default: the-seas.local.mk-labs.cloud/library/firecrawl-postgres
|
||||||
|
- name: image-tag
|
||||||
|
type: string
|
||||||
|
description: Image tag
|
||||||
|
default: latest
|
||||||
|
- name: dockerfile-path
|
||||||
|
type: string
|
||||||
|
description: Path to Dockerfile relative to repo root
|
||||||
|
default: ./apps/nuq-postgres/Dockerfile
|
||||||
|
- name: build-context
|
||||||
|
type: string
|
||||||
|
description: Build context path
|
||||||
|
default: ./apps/nuq-postgres
|
||||||
|
workspaces:
|
||||||
|
- name: shared-data
|
||||||
|
description: Workspace for source code
|
||||||
|
- name: docker-credentials
|
||||||
|
description: Docker config for Harbor authentication
|
||||||
|
tasks:
|
||||||
|
- name: fetch-source
|
||||||
|
taskRef:
|
||||||
|
name: git-clone
|
||||||
|
workspaces:
|
||||||
|
- name: output
|
||||||
|
workspace: shared-data
|
||||||
|
params:
|
||||||
|
- name: url
|
||||||
|
value: $(params.git-url)
|
||||||
|
- name: revision
|
||||||
|
value: $(params.git-revision)
|
||||||
|
- name: depth
|
||||||
|
value: "1"
|
||||||
|
- name: sslVerify
|
||||||
|
value: "false"
|
||||||
|
- name: deleteExisting
|
||||||
|
value: "true"
|
||||||
|
- name: build-push
|
||||||
|
taskRef:
|
||||||
|
name: kaniko-build
|
||||||
|
runAfter:
|
||||||
|
- fetch-source
|
||||||
|
workspaces:
|
||||||
|
- name: source
|
||||||
|
workspace: shared-data
|
||||||
|
- name: dockerconfig
|
||||||
|
workspace: docker-credentials
|
||||||
|
params:
|
||||||
|
- name: IMAGE
|
||||||
|
value: $(params.image-name):$(params.image-tag)
|
||||||
|
- name: DOCKERFILE
|
||||||
|
value: $(params.dockerfile-path)
|
||||||
|
- name: CONTEXT
|
||||||
|
value: $(params.build-context)
|
||||||
|
- name: EXTRA_ARGS
|
||||||
|
value:
|
||||||
|
- --insecure
|
||||||
|
- --skip-tls-verify
|
||||||
|
- --cache=true
|
||||||
|
- --cache-ttl=24h
|
||||||
Reference in New Issue
Block a user