--- # ------------------------------------------------------------------------------ # FILE: playbooks/day2_add_nomic_embed.yml # DESCRIPTION: Add nomic-embed-text-v1.5-Q4_K_M to the llama-server-router # on astro-orbiter (10.1.71.130:8002). # # Context (t_34b96e83, 2026-08-13, OpenViking Phase 1b): # Ryan approved adding nomic-embed-text-v1.5-Q4_K_M as an embedding model # after Phase 0 follow-up confirmed embedding models fold cleanly into the # existing router preset via embedding=true. Model ID is "nomic-embed-text-v1.5". # No alias needed — peter-parker and Honcho consumers will call it by the section # name directly. # # What this playbook does: # 1. Downloads nomic-embed-text-v1.5-Q4_K_M.gguf into /opt/models if not # already present (idempotent: exact size-check guard, no re-pull on match). # 2. Redeploys the preset INI (adding the [nomic-embed-text-v1.5] section with # embedding=true, n-gpu-layers=99, ctx-size=8192, load-on-startup=true, # sleep-idle-seconds=-1). # 3. Restarts llama-server-router to pick up the new model entry. # 4. Verifies /v1/models returns all 5 models including the new nomic entry. # 5. Runs a /v1/embeddings smoke test to confirm the model actually embeds. # # VRAM context note (t_34b96e83): # nomic-embed-text-v1.5 Q4_K_M: ~84MB weights, embedding model (no KV cache). # VRAM impact is negligible — always pinned via sleep-idle-seconds=-1. # The 4 generative models remain unchanged (OOM analysis unchanged from t_55c164f5). # # Usage (from ~/git/homelab/ansible): # env -u ANSIBLE_VAULT_PASSWORD_FILE ansible-playbook -i inventory.yml \ # playbooks/day2_add_nomic_embed.yml # # Semaphore note: Semaphore SSH key for jarvis user is not loaded in the # container (known pitfall, homelab-llm-serving skill). Run via CLI with # id_jarvis key; document as exception per Ryan's standing CLI fallback directive. # # Author: War Machine (2026-08-13, t_34b96e83) # ------------------------------------------------------------------------------ - name: "Add nomic-embed-text-v1.5 embedding model to astro-orbiter router" hosts: astro_orbiter gather_facts: false become: true vars: # Activate preset mode llm_router_preset_enabled: true llm_router_preset_path: /opt/llama-server-router-preset.ini # Production port (router is on 8002 since t_cd0d5388) llm_router_port: 8002 # Per-model ctx-size settings (carried from t_55c164f5; nomic new) llm_router_llama_ctx_size: 8192 llm_router_llama_flash_attn: "true" llm_router_phi_ctx_size: 32768 llm_router_phi_flash_attn: "true" llm_router_coder_ctx_size: 16384 llm_router_coder_flash_attn: "true" llm_router_nomic_ctx_size: 8192 # NOTE (2026-08-14, t_openviking_embed_batch): per-model batch-size/ # ubatch-size lines in the preset INI are NOT honored by llama-server's # router — only ctx-size is applied per-model; batch-size/ubatch-size for # every spawned child come from the router's own global CLI flags # (confirmed via `ps aux` on astro-orbiter: child process launched with # the router's --batch-size/--ubatch-size regardless of the INI values). # Kept below for documentation/future-proofing but the REAL fix is the # global llm_router_batch_size / llm_router_ubatch_size override further # down, which raises the physical batch for ALL models on this router # (Qwen3.6-35B, Phi, Llama, Coder, nomic). llm_router_nomic_batch_size: 4096 llm_router_nomic_ubatch_size: 4096 # All other vars inherit from host_vars + defaults/main.yml. llm_router_enabled: true llm_service_user: jarvis llm_binary_path: /opt/llama.cpp/build/bin/llama-server llm_models_dir: /opt/models llm_bind_address: "10.1.71.130" llm_allowed_source_cidr: "10.1.70.0/24" llm_router_service_name: llama-server-router llm_router_bind_address: "10.1.71.130" llm_router_allowed_source_cidr: "10.1.70.0/24" llm_router_models_dir: /opt/models llm_router_models_max: 4 # from host_vars; bumped by t_33acbb2e llm_router_ctx_size: 65536 # Qwen3.6-35B default; per-model overrides above llm_router_parallel: 1 llm_router_gpu_layers: 99 # FIX (2026-08-14, t_openviking_embed_batch): raised from 512 to 4096. # This is a GLOBAL router flag applied to every spawned model process # (per-model INI batch-size/ubatch-size overrides are not honored by # llama-server's router — see note above nomic vars). 512 tokens was too # small for OpenViking's chunked-document embedding inputs (observed # 2000-3400 tokens/chunk), causing hard 500 errors ("input (N tokens) is # too large to process") that tripped OpenViking's circuit breaker into a # permanent fail/re-enqueue loop. 4096 comfortably covers observed chunk # sizes and stays under nomic's ctx-size=8192. VRAM impact of raising # ubatch-size is in compute-buffer scratch space, not KV cache; monitored # post-deploy against the 23000 MiB budget (host_vars/astro-orbiter). llm_router_batch_size: 4096 llm_router_ubatch_size: 4096 llm_router_cache_type_k: q4_0 llm_router_cache_type_v: q4_0 llm_router_flash_attn: "auto" llm_router_expected_model_id: "Qwen3.6-35B-A3B-UD-Q4_K_S" llm_router_vram_max_mib: 23000 # nomic model staging nomic_filename: "nomic-embed-text-v1.5-Q4_K_M.gguf" nomic_url: "https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.Q4_K_M.gguf" nomic_size_bytes: 84106624 handlers: - name: reload systemd ansible.builtin.systemd: daemon_reload: true become: true listen: "reload systemd" - name: restart router ansible.builtin.systemd: name: llama-server-router state: restarted become: true listen: "restart router" tasks: # ========================================================================== # PHASE 1: Download nomic GGUF if not present / size mismatch # ========================================================================== - name: "[nomic] Stat existing GGUF" ansible.builtin.stat: path: "{{ llm_models_dir }}/{{ nomic_filename }}" get_checksum: false register: nomic_stat - name: "[nomic] Download GGUF (skip if present and size matches)" ansible.builtin.get_url: url: "{{ nomic_url }}" dest: "{{ llm_models_dir }}/{{ nomic_filename }}" owner: "{{ llm_service_user }}" group: "{{ llm_service_user }}" mode: "0644" timeout: 300 when: > not nomic_stat.stat.exists or nomic_stat.stat.size != nomic_size_bytes register: nomic_download notify: restart router - name: "[nomic] Confirm GGUF size post-download" ansible.builtin.stat: path: "{{ llm_models_dir }}/{{ nomic_filename }}" get_checksum: false register: nomic_stat_post - name: "[nomic] FAIL if GGUF size mismatch after download" ansible.builtin.fail: msg: >- GGUF size mismatch: expected {{ nomic_size_bytes }} bytes, got {{ nomic_stat_post.stat.size }} bytes. Re-download may be needed. when: nomic_stat_post.stat.size != nomic_size_bytes # ========================================================================== # PHASE 2: Deploy updated preset INI (adds nomic-embed-text-v1.5 section) # ========================================================================== - name: "[nomic] Deploy preset INI to {{ llm_router_preset_path }}" ansible.builtin.template: src: "../roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2" dest: "{{ llm_router_preset_path }}" owner: root group: root mode: "0644" register: nomic_preset_deployed notify: restart router # ========================================================================== # PHASE 3: Redeploy systemd unit (ensures unit is fresh; no flag changes) # ========================================================================== - name: "[nomic] Deploy llama-server-router unit" ansible.builtin.template: src: "../roles/llm-inference-multimodel/templates/llama-server-router.service.j2" dest: /etc/systemd/system/llama-server-router.service owner: root group: root mode: "0644" register: nomic_unit_deployed notify: - reload systemd - restart router - name: "[nomic] Flush handlers (daemon-reload + router restart)" ansible.builtin.meta: flush_handlers # ========================================================================== # PHASE 4: Verify router is up and nomic model appears in /v1/models # ========================================================================== - name: "[nomic] Wait for /health (router supervisor)" ansible.builtin.uri: url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/health" status_code: 200 timeout: 30 retries: 12 delay: 5 register: nomic_health until: nomic_health.status == 200 - name: "[nomic] Query /v1/models" ansible.builtin.uri: url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/models" status_code: 200 return_content: true timeout: 30 register: nomic_models - name: "[nomic] Extract model IDs and aliases" ansible.builtin.set_fact: nomic_model_ids: "{{ nomic_models.json.data | map(attribute='id') | list }}" nomic_all_aliases: "{{ nomic_models.json.data | map(attribute='aliases') | flatten | list }}" - name: "[nomic] FAIL if nomic primary ID missing" ansible.builtin.fail: msg: >- 'nomic-embed-text-v1.5' not in /v1/models. IDs: {{ nomic_model_ids }} when: "'nomic-embed-text-v1.5' not in nomic_model_ids" - name: "[nomic] FAIL if Qwen3.6-35B missing" ansible.builtin.fail: msg: "'Qwen3.6-35B-A3B-UD-Q4_K_S' not in /v1/models. IDs: {{ nomic_model_ids }}" when: "'Qwen3.6-35B-A3B-UD-Q4_K_S' not in nomic_model_ids" - name: "[nomic] FAIL if Phi missing" ansible.builtin.fail: msg: "'Phi-3.5-mini-instruct-Q8_0' not in /v1/models. IDs: {{ nomic_model_ids }}" when: "'Phi-3.5-mini-instruct-Q8_0' not in nomic_model_ids" - name: "[nomic] FAIL if Llama missing" ansible.builtin.fail: msg: "'Meta-Llama-3.1-8B-Instruct-Q4_K_M' not in /v1/models. IDs: {{ nomic_model_ids }}" when: "'Meta-Llama-3.1-8B-Instruct-Q4_K_M' not in nomic_model_ids" - name: "[nomic] FAIL if Coder missing" ansible.builtin.fail: msg: "'Qwen2.5-Coder-14B-Instruct-Q4_K_M' not in /v1/models. IDs: {{ nomic_model_ids }}" when: "'Qwen2.5-Coder-14B-Instruct-Q4_K_M' not in nomic_model_ids" # ========================================================================== # PHASE 5: /v1/embeddings smoke test — confirm model actually embeds # ========================================================================== - name: "[nomic] POST /v1/embeddings smoke test" ansible.builtin.uri: url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/embeddings" method: POST body_format: json body: model: "nomic-embed-text-v1.5" input: "The dog ran across the park." status_code: 200 return_content: true timeout: 120 register: nomic_embed_result - name: "[nomic] Extract embedding vector length" ansible.builtin.set_fact: nomic_embed_dims: >- {{ (nomic_embed_result.json.data | first).embedding | length }} when: - nomic_embed_result.status == 200 - nomic_embed_result.json.data is defined - nomic_embed_result.json.data | length > 0 - name: "[nomic] FAIL if embedding vector is empty or missing" ansible.builtin.fail: msg: >- Embedding smoke test returned no vector. Response: {{ nomic_embed_result.json }} when: >- nomic_embed_result.status != 200 or nomic_embed_result.json.data is not defined or nomic_embed_result.json.data | length == 0 or (nomic_embed_result.json.data | first).embedding | length == 0 - name: "[nomic] PASS — full summary" ansible.builtin.debug: msg: - "========================================================================" - "NOMIC-EMBED-TEXT-V1.5 DEPLOYMENT — COMPLETE" - "" - " Mode: --models-preset ({{ llm_router_preset_path }})" - " Service: llama-server-router.service (:{{ llm_router_port }})" - "" - " /v1/models IDs: {{ nomic_model_ids }}" - "" - " VERIFY:" - " Qwen3.6-35B-A3B-UD-Q4_K_S: {{ 'PRESENT' if 'Qwen3.6-35B-A3B-UD-Q4_K_S' in nomic_model_ids else 'MISSING' }}" - " Phi-3.5-mini-instruct-Q8_0: {{ 'PRESENT' if 'Phi-3.5-mini-instruct-Q8_0' in nomic_model_ids else 'MISSING' }}" - " Meta-Llama-3.1-8B-Instruct-Q4_K_M: {{ 'PRESENT' if 'Meta-Llama-3.1-8B-Instruct-Q4_K_M' in nomic_model_ids else 'MISSING' }}" - " Qwen2.5-Coder-14B-Instruct-Q4_K_M: {{ 'PRESENT' if 'Qwen2.5-Coder-14B-Instruct-Q4_K_M' in nomic_model_ids else 'MISSING' }}" - " nomic-embed-text-v1.5: {{ 'PRESENT' if 'nomic-embed-text-v1.5' in nomic_model_ids else 'MISSING' }}" - "" - " Embedding smoke test: PASS" - " Vector dimensions: {{ nomic_embed_dims | default('unknown') }}" - "" - " GGUF download: {{ 'NEW DOWNLOAD' if (nomic_download is defined and nomic_download.changed) else 'ALREADY PRESENT (skipped)' }}" - "========================================================================"