fix(honcho): cover all LLM subsystems, enable flush, disable embeddings

Smoke-testing the first deploy uncovered three default-config issues
that no amount of Quadlet tuning would have caught:

  1. DIALECTIC subsystem ignored DERIVER_MODEL_CONFIG__*. Honcho splits
     dialectic into five reasoning levels (minimal/low/medium/high/max)
     each with its own MODEL_CONFIG that defaults to OpenAI. Without
     overrides, every /chat call fails with: ValidationException:
     Missing API key for openai model config. Now setting all five
     DIALECTIC_LEVELS__<level>__MODEL_CONFIG__* env vars to anthropic.

  2. DERIVER batches representation tasks until a token threshold is
     reached. For low-volume homelab use (one chatty operator), tasks
     can sit unprocessed forever. Add DERIVER_FLUSH_ENABLED knob,
     default true.

  3. Embeddings default to OpenAI text-embedding-3-small. Anthropic
     has no embedding API, so without an OpenAI key the embed step
     fails the entire derivation. Default EMBED_MESSAGES=false until
     a separate embedding provider is wired up (OpenAI for embeds-only
     or a local BGE endpoint on astro-orbiter).

defaults/main.yml documents all three issues and the migration path
back to embeddings when ready.

ansible/roles/honcho/defaults/main.yml

@@ -68,17 +68,50 @@ honcho_postgres_initdb_args: "--encoding=UTF8 --locale=C"
 # ---------------------------------------------------------------------------
 # Supported transports: openai, anthropic, gemini. We start with anthropic
 # and can flip to a local OpenAI-compatible endpoint later by changing
-# these two values plus the API-key env-var name.
+# these knobs plus the API-key env-var name.
+#
+# Honcho has THREE distinct LLM subsystems, each with its own settings
+# class and env-var prefix:
+#
+#   DERIVER  — generates observations from messages (background worker)
+#   SUMMARY  — periodic session summaries
+#   DIALECTIC— answers theory-of-mind queries (the `peers/{peer}/chat` API)
+#
+# Honcho's dialectic subsystem uses PER-LEVEL config (minimal/low/medium/
+# high/max) under DIALECTIC_LEVELS__<level>__MODEL_CONFIG__*. Defaults
+# point at OpenAI, so any honcho.dev install without an OpenAI key will
+# fail dialectic queries with "Missing API key for openai model config".
+# We override all five levels to Anthropic below in the API template.
 honcho_llm_transport: "anthropic"
 honcho_deriver_model: "claude-sonnet-4-5"
 honcho_summary_model: "claude-sonnet-4-5"
 honcho_dialectic_model: "claude-sonnet-4-5"
 
-# Deriver tuning — number of workers, polling cadence, etc. Conservative
-# defaults appropriate for a small homelab deployment.
+# Deriver tuning. Honcho batches representation tasks until the per-batch
+# token threshold is reached. For homelab use with one chatty operator,
+# that means short bursts of conversation can sit unprocessed forever.
+# DERIVER_FLUSH_ENABLED=true bypasses the batching threshold so each
+# message is processed promptly.
 honcho_deriver_enabled: true
 honcho_deriver_workers: 1
 honcho_deriver_polling_seconds: "1.0"
+honcho_deriver_flush_enabled: true
+
+# ---------------------------------------------------------------------------
+# Embeddings
+# ---------------------------------------------------------------------------
+# Anthropic does not provide an embedding API. Honcho defaults to
+# OpenAI text-embedding-3-small, which fails without an OpenAI key.
+# Three options going forward:
+#   1. Set EMBED_MESSAGES=false — disables semantic search/vector recall
+#      but theory-of-mind derivation still works. Default for now.
+#   2. Provide an OpenAI API key purely for embeddings (cheap; embeddings
+#      are ~$0.02 per million tokens). Set honcho_embed_messages=true and
+#      add vault_honcho_openai_api_key.
+#   3. Stand up a local BGE/nomic embedding endpoint (e.g. on astro-orbiter
+#      once GPU is ready) and point Honcho at it via
+#      EMBEDDING_MODEL_CONFIG__OVERRIDES__BASE_URL.
+honcho_embed_messages: false
 
 # ---------------------------------------------------------------------------
 # Auth

ansible/roles/honcho/templates/honcho-api.container.j2

@@ -1,4 +1,3 @@
-# {{ ansible_managed }}
 # mk-labs Honcho API service (FastAPI on :8000)
 # Managed by Ansible - honcho role
 
@@ -21,26 +20,48 @@ Exec=/app/docker/entrypoint.sh
 Environment=DB_CONNECTION_URI=postgresql+psycopg://{{ honcho_db_user }}:{{ honcho_db_password }}@{{ honcho_postgres_container_name }}:5432/{{ honcho_db_name }}
 
 # -- Auth -------------------------------------------------------------------
-Environment=AUTH_USE_AUTH={{ honcho_auth_enabled | string | lower }}
-Environment=AUTH_JWT_SECRET={{ honcho_jwt_secret }}
+Environment=AUTH_USE_AUTH=*** honcho_auth_enabled | string | lower }}
+Environment=AUTH_JWT_SECRET=*** honcho_jwt_secret }}
 
-# -- LLM provider (deriver / summary / dialectic) ---------------------------
+# -- LLM provider keys ------------------------------------------------------
 {% if honcho_llm_transport == 'anthropic' %}
-Environment=LLM_ANTHROPIC_API_KEY={{ honcho_anthropic_api_key }}
+Environment=LLM_ANTHROPIC_API_KEY=*** honcho_anthropic_api_key }}
 {% endif %}
+
+# -- Deriver subsystem (background worker calls these to derive observations)
+Environment=DERIVER_FLUSH_ENABLED={{ honcho_deriver_flush_enabled | string | lower }}
 Environment=DERIVER_MODEL_CONFIG__TRANSPORT={{ honcho_llm_transport }}
 Environment=DERIVER_MODEL_CONFIG__MODEL={{ honcho_deriver_model }}
+
+# -- Summary subsystem (periodic session summaries) -------------------------
 Environment=SUMMARY_MODEL_CONFIG__TRANSPORT={{ honcho_llm_transport }}
 Environment=SUMMARY_MODEL_CONFIG__MODEL={{ honcho_summary_model }}
-Environment=DIALECTIC_MODEL_CONFIG__TRANSPORT={{ honcho_llm_transport }}
-Environment=DIALECTIC_MODEL_CONFIG__MODEL={{ honcho_dialectic_model }}
+
+# -- Dialectic subsystem (answers theory-of-mind queries via peers/.../chat)
+# Honcho has PER-LEVEL config for dialectic (minimal/low/medium/high/max).
+# Each level defaults to OpenAI, so we override all five to anthropic.
+Environment=DIALECTIC_LEVELS__minimal__MODEL_CONFIG__TRANSPORT={{ honcho_llm_transport }}
+Environment=DIALECTIC_LEVELS__minimal__MODEL_CONFIG__MODEL={{ honcho_dialectic_model }}
+Environment=DIALECTIC_LEVELS__low__MODEL_CONFIG__TRANSPORT={{ honcho_llm_transport }}
+Environment=DIALECTIC_LEVELS__low__MODEL_CONFIG__MODEL={{ honcho_dialectic_model }}
+Environment=DIALECTIC_LEVELS__medium__MODEL_CONFIG__TRANSPORT={{ honcho_llm_transport }}
+Environment=DIALECTIC_LEVELS__medium__MODEL_CONFIG__MODEL={{ honcho_dialectic_model }}
+Environment=DIALECTIC_LEVELS__high__MODEL_CONFIG__TRANSPORT={{ honcho_llm_transport }}
+Environment=DIALECTIC_LEVELS__high__MODEL_CONFIG__MODEL={{ honcho_dialectic_model }}
+Environment=DIALECTIC_LEVELS__max__MODEL_CONFIG__TRANSPORT={{ honcho_llm_transport }}
+Environment=DIALECTIC_LEVELS__max__MODEL_CONFIG__MODEL={{ honcho_dialectic_model }}
+
+# -- Embeddings -------------------------------------------------------------
+# Anthropic has no embedding API. Disable embeddings unless an OpenAI key
+# is provided. See defaults/main.yml for the three follow-up options.
+Environment=EMBED_MESSAGES={{ honcho_embed_messages | string | lower }}
 
 # -- Vector store -----------------------------------------------------------
 Environment=VECTOR_STORE_TYPE=pgvector
 
 # -- Public-facing URL (empty -> relative URLs, host-agnostic) --------------
 {% if honcho_web_url | length > 0 %}
-Environment=HONCHO_PUBLIC_URL={{ honcho_web_url }}
+Environment=HONCHO_PUBLIC_URL=*** honcho_web_url }}
 {% endif %}
 
 # Timezone matches host baseline

ansible/roles/honcho/templates/honcho-deriver.container.j2

@@ -1,4 +1,3 @@
-# {{ ansible_managed }}
 # mk-labs Honcho deriver worker (background queue consumer)
 # Managed by Ansible - honcho role
 #
@@ -17,9 +16,10 @@ Image={{ honcho_image }}
 ContainerName={{ honcho_deriver_container_name }}
 Network={{ honcho_network_name }}
 
-# Run the deriver script instead of the FastAPI server. The deriver
-# polls the queue and processes work in a loop. Number of concurrent
-# workers controlled via DERIVER_WORKERS.
+# Run the deriver via the package's __main__ (not the helpers module).
+# src/deriver/deriver.py has no __main__ guard — loading it exits 0 in
+# ~3s and systemd treats that as a crash loop. The actual entry point
+# lives in src/deriver/__main__.py and is invoked via "python -m src.deriver".
 WorkingDir=/app
 Exec=/app/.venv/bin/python -m src.deriver
 
@@ -29,21 +29,29 @@ Environment=DB_CONNECTION_URI=postgresql+psycopg://{{ honcho_db_user }}:{{ honch
 # -- Auth -------------------------------------------------------------------
 # Deriver does not serve HTTP, but it reads the same config object — keep
 # secrets aligned with the API container.
-Environment=AUTH_USE_AUTH={{ honcho_auth_enabled | string | lower }}
-Environment=AUTH_JWT_SECRET={{ honcho_jwt_secret }}
+Environment=AUTH_USE_AUTH=*** honcho_auth_enabled | string | lower }}
+Environment=AUTH_JWT_SECRET=*** honcho_jwt_secret }}
 
-# -- LLM provider -----------------------------------------------------------
+# -- LLM provider keys ------------------------------------------------------
 {% if honcho_llm_transport == 'anthropic' %}
-Environment=LLM_ANTHROPIC_API_KEY={{ honcho_anthropic_api_key }}
+Environment=LLM_ANTHROPIC_API_KEY=*** honcho_anthropic_api_key }}
 {% endif %}
+
+# -- Deriver runtime --------------------------------------------------------
 Environment=DERIVER_ENABLED={{ honcho_deriver_enabled | string | lower }}
 Environment=DERIVER_WORKERS={{ honcho_deriver_workers }}
 Environment=DERIVER_POLLING_SLEEP_INTERVAL_SECONDS={{ honcho_deriver_polling_seconds }}
+Environment=DERIVER_FLUSH_ENABLED={{ honcho_deriver_flush_enabled | string | lower }}
 Environment=DERIVER_MODEL_CONFIG__TRANSPORT={{ honcho_llm_transport }}
 Environment=DERIVER_MODEL_CONFIG__MODEL={{ honcho_deriver_model }}
+
+# -- Summary subsystem (the deriver triggers summaries periodically) --------
 Environment=SUMMARY_MODEL_CONFIG__TRANSPORT={{ honcho_llm_transport }}
 Environment=SUMMARY_MODEL_CONFIG__MODEL={{ honcho_summary_model }}
 
+# -- Embeddings (off by default — see defaults/main.yml for rationale) ------
+Environment=EMBED_MESSAGES={{ honcho_embed_messages | string | lower }}
+
 # -- Vector store -----------------------------------------------------------
 Environment=VECTOR_STORE_TYPE=pgvector