diff --git a/ansible/host_vars/astro-orbiter/vars.yml b/ansible/host_vars/astro-orbiter/vars.yml index f2c6499..7157fd7 100644 --- a/ansible/host_vars/astro-orbiter/vars.yml +++ b/ansible/host_vars/astro-orbiter/vars.yml @@ -34,21 +34,23 @@ common_root_lv: ubuntu-lv # (t_33acbb2e) so the router can keep more than one GGUF resident on-demand # and LRU-evict when needed. # -# VRAM NOTE (t_33acbb2e, updated t_55c164f5): With models-max=4 and all 4 GGUFs -# registered, worst case is all 4 loaded simultaneously: +# VRAM NOTE (t_33acbb2e, updated t_55c164f5, updated t_34b96e83): With models-max=4 and all 5 GGUFs +# registered, worst case is all 5 loaded simultaneously: # Qwen3.6-35B-A3B Q4_K_S: ~21.5GB (weights ~19.5GB + KV ~2GB @ 64K ctx, q4_0) # Phi-3.5-mini-instruct Q8_0: ~4.3GB (weights ~3.8GB + KV ~0.5GB @ 32K ctx) # Meta-Llama-3.1-8B Q4_K_M: ~5.6GB (weights ~4.6GB + KV ~0.2GB @ 8K ctx) # Qwen2.5-Coder-14B Q4_K_M: ~9.0GB (weights ~8.4GB + KV ~0.6GB @ 16K ctx) -# Total worst-case: ~40.4GB >> 24GB RTX 3090 +# nomic-embed-text-v1.5 Q4_K_M: ~0.09GB (~84MB, embedding only — no KV cache) +# Total worst-case: ~40.5GB >> 24GB RTX 3090 # # OOM RISK: Full co-residency is impossible on 24GB. LRU eviction prevents this -# in practice: models-max=4 means the router can REGISTER 4 models but only keeps +# in practice: models-max=4 means the router can REGISTER 5 models but only keeps # up to 4 LOADED simultaneously — the router will evict the LRU model when a new -# one is needed. In single-user homelab operation, only one model is active at a -# time. The realistic maximum co-residency is 2 models (whichever was last used). +# one is needed. nomic-embed-text-v1.5 is pinned via sleep-idle-seconds=-1 and +# load-on-startup=true but it uses only ~84MB, so it never meaningfully changes +# the budget. In single-user homelab operation, only one generative model is active +# at a time alongside the always-resident embedding model. # Qwen3.6-35B alone uses ~21.5GB; co-residency with Coder (~9GB) = ~30.5GB > 24GB. -# So effectively: Qwen3.6-35B + any second model will OOM IF both are held concurrently. # LRU eviction handles this automatically — the router evicts the idle model before # loading the new one. Ryan should be aware this means model-switching always incurs # a ~30-60s cold-load latency when switching between Qwen3.6-35B and any other model. @@ -68,4 +70,8 @@ llm_staged_models: url: "https://huggingface.co/bartowski/Qwen2.5-Coder-14B-Instruct-GGUF/resolve/main/Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf" size_bytes: 8988111072 source_repo: "bartowski/Qwen2.5-Coder-14B-Instruct-GGUF" + - filename: "nomic-embed-text-v1.5-Q4_K_M.gguf" + url: "https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.Q4_K_M.gguf" + size_bytes: 84106624 + source_repo: "nomic-ai/nomic-embed-text-v1.5-GGUF" diff --git a/ansible/playbooks/day2_add_nomic_embed.yml b/ansible/playbooks/day2_add_nomic_embed.yml new file mode 100644 index 0000000..1ef8b3d --- /dev/null +++ b/ansible/playbooks/day2_add_nomic_embed.yml @@ -0,0 +1,290 @@ +--- +# ------------------------------------------------------------------------------ +# FILE: playbooks/day2_add_nomic_embed.yml +# DESCRIPTION: Add nomic-embed-text-v1.5-Q4_K_M to the llama-server-router +# on astro-orbiter (10.1.71.130:8002). +# +# Context (t_34b96e83, 2026-08-13, OpenViking Phase 1b): +# Ryan approved adding nomic-embed-text-v1.5-Q4_K_M as an embedding model +# after Phase 0 follow-up confirmed embedding models fold cleanly into the +# existing router preset via embedding=true. Model ID is "nomic-embed-text-v1.5". +# No alias needed — peter-parker and Honcho consumers will call it by the section +# name directly. +# +# What this playbook does: +# 1. Downloads nomic-embed-text-v1.5-Q4_K_M.gguf into /opt/models if not +# already present (idempotent: exact size-check guard, no re-pull on match). +# 2. Redeploys the preset INI (adding the [nomic-embed-text-v1.5] section with +# embedding=true, n-gpu-layers=99, ctx-size=8192, load-on-startup=true, +# sleep-idle-seconds=-1). +# 3. Restarts llama-server-router to pick up the new model entry. +# 4. Verifies /v1/models returns all 5 models including the new nomic entry. +# 5. Runs a /v1/embeddings smoke test to confirm the model actually embeds. +# +# VRAM context note (t_34b96e83): +# nomic-embed-text-v1.5 Q4_K_M: ~84MB weights, embedding model (no KV cache). +# VRAM impact is negligible — always pinned via sleep-idle-seconds=-1. +# The 4 generative models remain unchanged (OOM analysis unchanged from t_55c164f5). +# +# Usage (from ~/git/homelab/ansible): +# env -u ANSIBLE_VAULT_PASSWORD_FILE ansible-playbook -i inventory.yml \ +# playbooks/day2_add_nomic_embed.yml +# +# Semaphore note: Semaphore SSH key for jarvis user is not loaded in the +# container (known pitfall, homelab-llm-serving skill). Run via CLI with +# id_jarvis key; document as exception per Ryan's standing CLI fallback directive. +# +# Author: War Machine (2026-08-13, t_34b96e83) +# ------------------------------------------------------------------------------ + +- name: "Add nomic-embed-text-v1.5 embedding model to astro-orbiter router" + hosts: astro_orbiter + gather_facts: false + become: true + + vars: + # Activate preset mode + llm_router_preset_enabled: true + llm_router_preset_path: /opt/llama-server-router-preset.ini + + # Production port (router is on 8002 since t_cd0d5388) + llm_router_port: 8002 + + # Per-model ctx-size settings (carried from t_55c164f5; nomic new) + llm_router_llama_ctx_size: 8192 + llm_router_llama_flash_attn: "true" + llm_router_phi_ctx_size: 32768 + llm_router_phi_flash_attn: "true" + llm_router_coder_ctx_size: 16384 + llm_router_coder_flash_attn: "true" + llm_router_nomic_ctx_size: 8192 + + # All other vars inherit from host_vars + defaults/main.yml. + llm_router_enabled: true + llm_service_user: jarvis + llm_binary_path: /opt/llama.cpp/build/bin/llama-server + llm_models_dir: /opt/models + llm_bind_address: "10.1.71.130" + llm_allowed_source_cidr: "10.1.70.0/24" + llm_router_service_name: llama-server-router + llm_router_bind_address: "10.1.71.130" + llm_router_allowed_source_cidr: "10.1.70.0/24" + llm_router_models_dir: /opt/models + llm_router_models_max: 4 # from host_vars; bumped by t_33acbb2e + llm_router_ctx_size: 65536 # Qwen3.6-35B default; per-model overrides above + llm_router_parallel: 1 + llm_router_gpu_layers: 99 + llm_router_batch_size: 2048 + llm_router_ubatch_size: 512 + llm_router_cache_type_k: q4_0 + llm_router_cache_type_v: q4_0 + llm_router_flash_attn: "auto" + llm_router_expected_model_id: "Qwen3.6-35B-A3B-UD-Q4_K_S" + llm_router_vram_max_mib: 23000 + + # nomic model staging + nomic_filename: "nomic-embed-text-v1.5-Q4_K_M.gguf" + nomic_url: "https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.Q4_K_M.gguf" + nomic_size_bytes: 84106624 + + handlers: + - name: reload systemd + ansible.builtin.systemd: + daemon_reload: true + become: true + listen: "reload systemd" + + - name: restart router + ansible.builtin.systemd: + name: llama-server-router + state: restarted + become: true + listen: "restart router" + + tasks: + + # ========================================================================== + # PHASE 1: Download nomic GGUF if not present / size mismatch + # ========================================================================== + + - name: "[nomic] Stat existing GGUF" + ansible.builtin.stat: + path: "{{ llm_models_dir }}/{{ nomic_filename }}" + get_checksum: false + register: nomic_stat + + - name: "[nomic] Download GGUF (skip if present and size matches)" + ansible.builtin.get_url: + url: "{{ nomic_url }}" + dest: "{{ llm_models_dir }}/{{ nomic_filename }}" + owner: "{{ llm_service_user }}" + group: "{{ llm_service_user }}" + mode: "0644" + timeout: 300 + when: > + not nomic_stat.stat.exists or + nomic_stat.stat.size != nomic_size_bytes + register: nomic_download + notify: restart router + + - name: "[nomic] Confirm GGUF size post-download" + ansible.builtin.stat: + path: "{{ llm_models_dir }}/{{ nomic_filename }}" + get_checksum: false + register: nomic_stat_post + + - name: "[nomic] FAIL if GGUF size mismatch after download" + ansible.builtin.fail: + msg: >- + GGUF size mismatch: expected {{ nomic_size_bytes }} bytes, + got {{ nomic_stat_post.stat.size }} bytes. + Re-download may be needed. + when: nomic_stat_post.stat.size != nomic_size_bytes + + # ========================================================================== + # PHASE 2: Deploy updated preset INI (adds nomic-embed-text-v1.5 section) + # ========================================================================== + + - name: "[nomic] Deploy preset INI to {{ llm_router_preset_path }}" + ansible.builtin.template: + src: "../roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2" + dest: "{{ llm_router_preset_path }}" + owner: root + group: root + mode: "0644" + register: nomic_preset_deployed + notify: restart router + + # ========================================================================== + # PHASE 3: Redeploy systemd unit (ensures unit is fresh; no flag changes) + # ========================================================================== + + - name: "[nomic] Deploy llama-server-router unit" + ansible.builtin.template: + src: "../roles/llm-inference-multimodel/templates/llama-server-router.service.j2" + dest: /etc/systemd/system/llama-server-router.service + owner: root + group: root + mode: "0644" + register: nomic_unit_deployed + notify: + - reload systemd + - restart router + + - name: "[nomic] Flush handlers (daemon-reload + router restart)" + ansible.builtin.meta: flush_handlers + + # ========================================================================== + # PHASE 4: Verify router is up and nomic model appears in /v1/models + # ========================================================================== + + - name: "[nomic] Wait for /health (router supervisor)" + ansible.builtin.uri: + url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/health" + status_code: 200 + timeout: 30 + retries: 12 + delay: 5 + register: nomic_health + until: nomic_health.status == 200 + + - name: "[nomic] Query /v1/models" + ansible.builtin.uri: + url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/models" + status_code: 200 + return_content: true + timeout: 30 + register: nomic_models + + - name: "[nomic] Extract model IDs and aliases" + ansible.builtin.set_fact: + nomic_model_ids: "{{ nomic_models.json.data | map(attribute='id') | list }}" + nomic_all_aliases: "{{ nomic_models.json.data | map(attribute='aliases') | flatten | list }}" + + - name: "[nomic] FAIL if nomic primary ID missing" + ansible.builtin.fail: + msg: >- + 'nomic-embed-text-v1.5' not in /v1/models. + IDs: {{ nomic_model_ids }} + when: "'nomic-embed-text-v1.5' not in nomic_model_ids" + + - name: "[nomic] FAIL if Qwen3.6-35B missing" + ansible.builtin.fail: + msg: "'Qwen3.6-35B-A3B-UD-Q4_K_S' not in /v1/models. IDs: {{ nomic_model_ids }}" + when: "'Qwen3.6-35B-A3B-UD-Q4_K_S' not in nomic_model_ids" + + - name: "[nomic] FAIL if Phi missing" + ansible.builtin.fail: + msg: "'Phi-3.5-mini-instruct-Q8_0' not in /v1/models. IDs: {{ nomic_model_ids }}" + when: "'Phi-3.5-mini-instruct-Q8_0' not in nomic_model_ids" + + - name: "[nomic] FAIL if Llama missing" + ansible.builtin.fail: + msg: "'Meta-Llama-3.1-8B-Instruct-Q4_K_M' not in /v1/models. IDs: {{ nomic_model_ids }}" + when: "'Meta-Llama-3.1-8B-Instruct-Q4_K_M' not in nomic_model_ids" + + - name: "[nomic] FAIL if Coder missing" + ansible.builtin.fail: + msg: "'Qwen2.5-Coder-14B-Instruct-Q4_K_M' not in /v1/models. IDs: {{ nomic_model_ids }}" + when: "'Qwen2.5-Coder-14B-Instruct-Q4_K_M' not in nomic_model_ids" + + # ========================================================================== + # PHASE 5: /v1/embeddings smoke test — confirm model actually embeds + # ========================================================================== + + - name: "[nomic] POST /v1/embeddings smoke test" + ansible.builtin.uri: + url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/embeddings" + method: POST + body_format: json + body: + model: "nomic-embed-text-v1.5" + input: "The dog ran across the park." + status_code: 200 + return_content: true + timeout: 120 + register: nomic_embed_result + + - name: "[nomic] Extract embedding vector length" + ansible.builtin.set_fact: + nomic_embed_dims: >- + {{ (nomic_embed_result.json.data | first).embedding | length }} + when: + - nomic_embed_result.status == 200 + - nomic_embed_result.json.data is defined + - nomic_embed_result.json.data | length > 0 + + - name: "[nomic] FAIL if embedding vector is empty or missing" + ansible.builtin.fail: + msg: >- + Embedding smoke test returned no vector. + Response: {{ nomic_embed_result.json }} + when: >- + nomic_embed_result.status != 200 or + nomic_embed_result.json.data is not defined or + nomic_embed_result.json.data | length == 0 or + (nomic_embed_result.json.data | first).embedding | length == 0 + + - name: "[nomic] PASS — full summary" + ansible.builtin.debug: + msg: + - "========================================================================" + - "NOMIC-EMBED-TEXT-V1.5 DEPLOYMENT — COMPLETE" + - "" + - " Mode: --models-preset ({{ llm_router_preset_path }})" + - " Service: llama-server-router.service (:{{ llm_router_port }})" + - "" + - " /v1/models IDs: {{ nomic_model_ids }}" + - "" + - " VERIFY:" + - " Qwen3.6-35B-A3B-UD-Q4_K_S: {{ 'PRESENT' if 'Qwen3.6-35B-A3B-UD-Q4_K_S' in nomic_model_ids else 'MISSING' }}" + - " Phi-3.5-mini-instruct-Q8_0: {{ 'PRESENT' if 'Phi-3.5-mini-instruct-Q8_0' in nomic_model_ids else 'MISSING' }}" + - " Meta-Llama-3.1-8B-Instruct-Q4_K_M: {{ 'PRESENT' if 'Meta-Llama-3.1-8B-Instruct-Q4_K_M' in nomic_model_ids else 'MISSING' }}" + - " Qwen2.5-Coder-14B-Instruct-Q4_K_M: {{ 'PRESENT' if 'Qwen2.5-Coder-14B-Instruct-Q4_K_M' in nomic_model_ids else 'MISSING' }}" + - " nomic-embed-text-v1.5: {{ 'PRESENT' if 'nomic-embed-text-v1.5' in nomic_model_ids else 'MISSING' }}" + - "" + - " Embedding smoke test: PASS" + - " Vector dimensions: {{ nomic_embed_dims | default('unknown') }}" + - "" + - " GGUF download: {{ 'NEW DOWNLOAD' if (nomic_download is defined and nomic_download.changed) else 'ALREADY PRESENT (skipped)' }}" + - "========================================================================" diff --git a/ansible/roles/llm-inference-multimodel/defaults/main.yml b/ansible/roles/llm-inference-multimodel/defaults/main.yml index 7748515..8588050 100644 --- a/ansible/roles/llm-inference-multimodel/defaults/main.yml +++ b/ansible/roles/llm-inference-multimodel/defaults/main.yml @@ -160,3 +160,8 @@ llm_router_phi_flash_attn: "{{ llm_router_flash_attn }}" llm_router_coder_ctx_size: 16384 llm_router_coder_flash_attn: "true" llm_router_preset_path: /opt/llama-server-router-preset.ini +# nomic-embed-text-v1.5: embedding model, ctx-size=8192 per task t_34b96e83 +# No flash_attn or KV cache params — embedding models use bidirectional forward pass, +# not autoregressive KV cache. load-on-startup=true / sleep-idle-seconds=-1 keep it +# always warm at negligible VRAM cost (~84MB). +llm_router_nomic_ctx_size: 8192 diff --git a/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 b/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 index 9c904d9..a305946 100644 --- a/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 +++ b/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 @@ -40,6 +40,14 @@ ; Hermes custom_providers routing — see role README / deployment report for ; the alias-naming ambiguity flag (Ryan's pasted TOML used different alias ; strings: "llama-3.1-8b" / "phi-3.5-mini"). +; +; UPDATED (t_34b96e83, 2026-08-13, per Ryan approval): Added nomic-embed-text-v1.5 +; embedding model. Embedding models fold cleanly into the router preset via +; embedding=true. No alias needed — clients call it by section name. +; VRAM estimate ~90MB (negligible). sleep-idle-seconds=-1 keeps it always loaded +; since embedding calls are latency-sensitive and it costs near-nothing to hold. +; load-on-startup=true ensures the embedding endpoint is warm at boot without +; waiting for the first request. — War Machine. ; ------------------------------------------------------------------------------ ; --- Production model: Qwen3.6-35B-A3B-UD-Q4_K_S ---------------------------- @@ -124,3 +132,25 @@ cache-type-v = {{ llm_router_cache_type_v }} batch-size = {{ llm_router_batch_size }} ubatch-size = {{ llm_router_ubatch_size }} parallel = {{ llm_router_parallel }} + +; --- Embedding model: nomic-embed-text-v1.5 ---------------------------------- +; Primary model ID: nomic-embed-text-v1.5 (section name / client-visible ID) +; ~84MB GGUF — negligible VRAM, always-loaded. Embedding endpoint: /v1/embeddings. +; embedding=true: required to expose /v1/embeddings and embed the model (not chat). +; n-gpu-layers=99: GPU offload all layers (tiny model, no reason to leave on CPU). +; ctx-size=8192: per task spec (OpenViking Phase 1b, t_34b96e83). +; load-on-startup=true: warm at boot — embedding callers (peter-parker) are +; latency-sensitive; no cold-load wait on first request. +; sleep-idle-seconds=-1: never evict — ~84MB is negligible, always keep hot. +; NO flash-attn, NO KV cache params: embedding models use a different forward +; pass (bidirectional, no autoregressive KV cache). These keys are irrelevant +; for embedding inference and may be silently ignored or cause warnings; omit. +; Source: nomic-ai/nomic-embed-text-v1.5-GGUF (public, no auth needed) +; Added 2026-08-13 (t_34b96e83) — War Machine. +[nomic-embed-text-v1.5] +model = {{ llm_models_dir }}/nomic-embed-text-v1.5-Q4_K_M.gguf +embedding = true +n-gpu-layers = {{ llm_router_gpu_layers }} +ctx-size = {{ llm_router_nomic_ctx_size }} +load-on-startup = true +sleep-idle-seconds = -1