feat(llm): add nomic-embed-text-v1.5-Q4_K_M to astro-orbiter router
OpenViking Phase 1b (t_34b96e83) — Ryan-approved implementation. Changes: - roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2: Add [nomic-embed-text-v1.5] section with embedding=true, n-gpu-layers=99, ctx-size=8192, load-on-startup=true, sleep-idle-seconds=-1. No flash-attn or KV cache params (embedding models use bidirectional forward pass, not autoregressive KV cache). Var: llm_router_nomic_ctx_size. - roles/llm-inference-multimodel/defaults/main.yml: Add llm_router_nomic_ctx_size: 8192. - host_vars/astro-orbiter/vars.yml: Add nomic-embed-text-v1.5-Q4_K_M.gguf to llm_staged_models list (size_bytes: 84106624, source: nomic-ai/nomic-embed-text-v1.5-GGUF). Update VRAM note to reflect 5 registered models (nomic adds ~84MB, negligible given sleep-idle-seconds=-1 / load-on-startup=true pinning). - playbooks/day2_add_nomic_embed.yml: New day2 playbook following the coder-alias pattern: Phase 1: idempotent GGUF download (exact size check) Phase 2: redeploy preset INI Phase 3: redeploy + restart systemd unit Phase 4: /v1/models gate (all 5 models present) Phase 5: /v1/embeddings smoke test (vector returned, not empty) VRAM: ~84MB, always pinned. No impact on generative model LRU behavior. peter-parker Helm values already point at :8002 for the embedding endpoint.
This commit is contained in:
Hermes Agent service account
290
ansible/playbooks/day2_add_nomic_embed.yml
Normal file
290
ansible/playbooks/day2_add_nomic_embed.yml
Normal file
@@ -0,0 +1,290 @@
|
||||
---
|
||||
# ------------------------------------------------------------------------------
|
||||
# FILE: playbooks/day2_add_nomic_embed.yml
|
||||
# DESCRIPTION: Add nomic-embed-text-v1.5-Q4_K_M to the llama-server-router
|
||||
# on astro-orbiter (10.1.71.130:8002).
|
||||
#
|
||||
# Context (t_34b96e83, 2026-08-13, OpenViking Phase 1b):
|
||||
# Ryan approved adding nomic-embed-text-v1.5-Q4_K_M as an embedding model
|
||||
# after Phase 0 follow-up confirmed embedding models fold cleanly into the
|
||||
# existing router preset via embedding=true. Model ID is "nomic-embed-text-v1.5".
|
||||
# No alias needed — peter-parker and Honcho consumers will call it by the section
|
||||
# name directly.
|
||||
#
|
||||
# What this playbook does:
|
||||
# 1. Downloads nomic-embed-text-v1.5-Q4_K_M.gguf into /opt/models if not
|
||||
# already present (idempotent: exact size-check guard, no re-pull on match).
|
||||
# 2. Redeploys the preset INI (adding the [nomic-embed-text-v1.5] section with
|
||||
# embedding=true, n-gpu-layers=99, ctx-size=8192, load-on-startup=true,
|
||||
# sleep-idle-seconds=-1).
|
||||
# 3. Restarts llama-server-router to pick up the new model entry.
|
||||
# 4. Verifies /v1/models returns all 5 models including the new nomic entry.
|
||||
# 5. Runs a /v1/embeddings smoke test to confirm the model actually embeds.
|
||||
#
|
||||
# VRAM context note (t_34b96e83):
|
||||
# nomic-embed-text-v1.5 Q4_K_M: ~84MB weights, embedding model (no KV cache).
|
||||
# VRAM impact is negligible — always pinned via sleep-idle-seconds=-1.
|
||||
# The 4 generative models remain unchanged (OOM analysis unchanged from t_55c164f5).
|
||||
#
|
||||
# Usage (from ~/git/homelab/ansible):
|
||||
# env -u ANSIBLE_VAULT_PASSWORD_FILE ansible-playbook -i inventory.yml \
|
||||
# playbooks/day2_add_nomic_embed.yml
|
||||
#
|
||||
# Semaphore note: Semaphore SSH key for jarvis user is not loaded in the
|
||||
# container (known pitfall, homelab-llm-serving skill). Run via CLI with
|
||||
# id_jarvis key; document as exception per Ryan's standing CLI fallback directive.
|
||||
#
|
||||
# Author: War Machine (2026-08-13, t_34b96e83)
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
- name: "Add nomic-embed-text-v1.5 embedding model to astro-orbiter router"
|
||||
hosts: astro_orbiter
|
||||
gather_facts: false
|
||||
become: true
|
||||
|
||||
vars:
|
||||
# Activate preset mode
|
||||
llm_router_preset_enabled: true
|
||||
llm_router_preset_path: /opt/llama-server-router-preset.ini
|
||||
|
||||
# Production port (router is on 8002 since t_cd0d5388)
|
||||
llm_router_port: 8002
|
||||
|
||||
# Per-model ctx-size settings (carried from t_55c164f5; nomic new)
|
||||
llm_router_llama_ctx_size: 8192
|
||||
llm_router_llama_flash_attn: "true"
|
||||
llm_router_phi_ctx_size: 32768
|
||||
llm_router_phi_flash_attn: "true"
|
||||
llm_router_coder_ctx_size: 16384
|
||||
llm_router_coder_flash_attn: "true"
|
||||
llm_router_nomic_ctx_size: 8192
|
||||
|
||||
# All other vars inherit from host_vars + defaults/main.yml.
|
||||
llm_router_enabled: true
|
||||
llm_service_user: jarvis
|
||||
llm_binary_path: /opt/llama.cpp/build/bin/llama-server
|
||||
llm_models_dir: /opt/models
|
||||
llm_bind_address: "10.1.71.130"
|
||||
llm_allowed_source_cidr: "10.1.70.0/24"
|
||||
llm_router_service_name: llama-server-router
|
||||
llm_router_bind_address: "10.1.71.130"
|
||||
llm_router_allowed_source_cidr: "10.1.70.0/24"
|
||||
llm_router_models_dir: /opt/models
|
||||
llm_router_models_max: 4 # from host_vars; bumped by t_33acbb2e
|
||||
llm_router_ctx_size: 65536 # Qwen3.6-35B default; per-model overrides above
|
||||
llm_router_parallel: 1
|
||||
llm_router_gpu_layers: 99
|
||||
llm_router_batch_size: 2048
|
||||
llm_router_ubatch_size: 512
|
||||
llm_router_cache_type_k: q4_0
|
||||
llm_router_cache_type_v: q4_0
|
||||
llm_router_flash_attn: "auto"
|
||||
llm_router_expected_model_id: "Qwen3.6-35B-A3B-UD-Q4_K_S"
|
||||
llm_router_vram_max_mib: 23000
|
||||
|
||||
# nomic model staging
|
||||
nomic_filename: "nomic-embed-text-v1.5-Q4_K_M.gguf"
|
||||
nomic_url: "https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.Q4_K_M.gguf"
|
||||
nomic_size_bytes: 84106624
|
||||
|
||||
handlers:
|
||||
- name: reload systemd
|
||||
ansible.builtin.systemd:
|
||||
daemon_reload: true
|
||||
become: true
|
||||
listen: "reload systemd"
|
||||
|
||||
- name: restart router
|
||||
ansible.builtin.systemd:
|
||||
name: llama-server-router
|
||||
state: restarted
|
||||
become: true
|
||||
listen: "restart router"
|
||||
|
||||
tasks:
|
||||
|
||||
# ==========================================================================
|
||||
# PHASE 1: Download nomic GGUF if not present / size mismatch
|
||||
# ==========================================================================
|
||||
|
||||
- name: "[nomic] Stat existing GGUF"
|
||||
ansible.builtin.stat:
|
||||
path: "{{ llm_models_dir }}/{{ nomic_filename }}"
|
||||
get_checksum: false
|
||||
register: nomic_stat
|
||||
|
||||
- name: "[nomic] Download GGUF (skip if present and size matches)"
|
||||
ansible.builtin.get_url:
|
||||
url: "{{ nomic_url }}"
|
||||
dest: "{{ llm_models_dir }}/{{ nomic_filename }}"
|
||||
owner: "{{ llm_service_user }}"
|
||||
group: "{{ llm_service_user }}"
|
||||
mode: "0644"
|
||||
timeout: 300
|
||||
when: >
|
||||
not nomic_stat.stat.exists or
|
||||
nomic_stat.stat.size != nomic_size_bytes
|
||||
register: nomic_download
|
||||
notify: restart router
|
||||
|
||||
- name: "[nomic] Confirm GGUF size post-download"
|
||||
ansible.builtin.stat:
|
||||
path: "{{ llm_models_dir }}/{{ nomic_filename }}"
|
||||
get_checksum: false
|
||||
register: nomic_stat_post
|
||||
|
||||
- name: "[nomic] FAIL if GGUF size mismatch after download"
|
||||
ansible.builtin.fail:
|
||||
msg: >-
|
||||
GGUF size mismatch: expected {{ nomic_size_bytes }} bytes,
|
||||
got {{ nomic_stat_post.stat.size }} bytes.
|
||||
Re-download may be needed.
|
||||
when: nomic_stat_post.stat.size != nomic_size_bytes
|
||||
|
||||
# ==========================================================================
|
||||
# PHASE 2: Deploy updated preset INI (adds nomic-embed-text-v1.5 section)
|
||||
# ==========================================================================
|
||||
|
||||
- name: "[nomic] Deploy preset INI to {{ llm_router_preset_path }}"
|
||||
ansible.builtin.template:
|
||||
src: "../roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2"
|
||||
dest: "{{ llm_router_preset_path }}"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
register: nomic_preset_deployed
|
||||
notify: restart router
|
||||
|
||||
# ==========================================================================
|
||||
# PHASE 3: Redeploy systemd unit (ensures unit is fresh; no flag changes)
|
||||
# ==========================================================================
|
||||
|
||||
- name: "[nomic] Deploy llama-server-router unit"
|
||||
ansible.builtin.template:
|
||||
src: "../roles/llm-inference-multimodel/templates/llama-server-router.service.j2"
|
||||
dest: /etc/systemd/system/llama-server-router.service
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
register: nomic_unit_deployed
|
||||
notify:
|
||||
- reload systemd
|
||||
- restart router
|
||||
|
||||
- name: "[nomic] Flush handlers (daemon-reload + router restart)"
|
||||
ansible.builtin.meta: flush_handlers
|
||||
|
||||
# ==========================================================================
|
||||
# PHASE 4: Verify router is up and nomic model appears in /v1/models
|
||||
# ==========================================================================
|
||||
|
||||
- name: "[nomic] Wait for /health (router supervisor)"
|
||||
ansible.builtin.uri:
|
||||
url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/health"
|
||||
status_code: 200
|
||||
timeout: 30
|
||||
retries: 12
|
||||
delay: 5
|
||||
register: nomic_health
|
||||
until: nomic_health.status == 200
|
||||
|
||||
- name: "[nomic] Query /v1/models"
|
||||
ansible.builtin.uri:
|
||||
url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/models"
|
||||
status_code: 200
|
||||
return_content: true
|
||||
timeout: 30
|
||||
register: nomic_models
|
||||
|
||||
- name: "[nomic] Extract model IDs and aliases"
|
||||
ansible.builtin.set_fact:
|
||||
nomic_model_ids: "{{ nomic_models.json.data | map(attribute='id') | list }}"
|
||||
nomic_all_aliases: "{{ nomic_models.json.data | map(attribute='aliases') | flatten | list }}"
|
||||
|
||||
- name: "[nomic] FAIL if nomic primary ID missing"
|
||||
ansible.builtin.fail:
|
||||
msg: >-
|
||||
'nomic-embed-text-v1.5' not in /v1/models.
|
||||
IDs: {{ nomic_model_ids }}
|
||||
when: "'nomic-embed-text-v1.5' not in nomic_model_ids"
|
||||
|
||||
- name: "[nomic] FAIL if Qwen3.6-35B missing"
|
||||
ansible.builtin.fail:
|
||||
msg: "'Qwen3.6-35B-A3B-UD-Q4_K_S' not in /v1/models. IDs: {{ nomic_model_ids }}"
|
||||
when: "'Qwen3.6-35B-A3B-UD-Q4_K_S' not in nomic_model_ids"
|
||||
|
||||
- name: "[nomic] FAIL if Phi missing"
|
||||
ansible.builtin.fail:
|
||||
msg: "'Phi-3.5-mini-instruct-Q8_0' not in /v1/models. IDs: {{ nomic_model_ids }}"
|
||||
when: "'Phi-3.5-mini-instruct-Q8_0' not in nomic_model_ids"
|
||||
|
||||
- name: "[nomic] FAIL if Llama missing"
|
||||
ansible.builtin.fail:
|
||||
msg: "'Meta-Llama-3.1-8B-Instruct-Q4_K_M' not in /v1/models. IDs: {{ nomic_model_ids }}"
|
||||
when: "'Meta-Llama-3.1-8B-Instruct-Q4_K_M' not in nomic_model_ids"
|
||||
|
||||
- name: "[nomic] FAIL if Coder missing"
|
||||
ansible.builtin.fail:
|
||||
msg: "'Qwen2.5-Coder-14B-Instruct-Q4_K_M' not in /v1/models. IDs: {{ nomic_model_ids }}"
|
||||
when: "'Qwen2.5-Coder-14B-Instruct-Q4_K_M' not in nomic_model_ids"
|
||||
|
||||
# ==========================================================================
|
||||
# PHASE 5: /v1/embeddings smoke test — confirm model actually embeds
|
||||
# ==========================================================================
|
||||
|
||||
- name: "[nomic] POST /v1/embeddings smoke test"
|
||||
ansible.builtin.uri:
|
||||
url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/embeddings"
|
||||
method: POST
|
||||
body_format: json
|
||||
body:
|
||||
model: "nomic-embed-text-v1.5"
|
||||
input: "The dog ran across the park."
|
||||
status_code: 200
|
||||
return_content: true
|
||||
timeout: 120
|
||||
register: nomic_embed_result
|
||||
|
||||
- name: "[nomic] Extract embedding vector length"
|
||||
ansible.builtin.set_fact:
|
||||
nomic_embed_dims: >-
|
||||
{{ (nomic_embed_result.json.data | first).embedding | length }}
|
||||
when:
|
||||
- nomic_embed_result.status == 200
|
||||
- nomic_embed_result.json.data is defined
|
||||
- nomic_embed_result.json.data | length > 0
|
||||
|
||||
- name: "[nomic] FAIL if embedding vector is empty or missing"
|
||||
ansible.builtin.fail:
|
||||
msg: >-
|
||||
Embedding smoke test returned no vector.
|
||||
Response: {{ nomic_embed_result.json }}
|
||||
when: >-
|
||||
nomic_embed_result.status != 200 or
|
||||
nomic_embed_result.json.data is not defined or
|
||||
nomic_embed_result.json.data | length == 0 or
|
||||
(nomic_embed_result.json.data | first).embedding | length == 0
|
||||
|
||||
- name: "[nomic] PASS — full summary"
|
||||
ansible.builtin.debug:
|
||||
msg:
|
||||
- "========================================================================"
|
||||
- "NOMIC-EMBED-TEXT-V1.5 DEPLOYMENT — COMPLETE"
|
||||
- ""
|
||||
- " Mode: --models-preset ({{ llm_router_preset_path }})"
|
||||
- " Service: llama-server-router.service (:{{ llm_router_port }})"
|
||||
- ""
|
||||
- " /v1/models IDs: {{ nomic_model_ids }}"
|
||||
- ""
|
||||
- " VERIFY:"
|
||||
- " Qwen3.6-35B-A3B-UD-Q4_K_S: {{ 'PRESENT' if 'Qwen3.6-35B-A3B-UD-Q4_K_S' in nomic_model_ids else 'MISSING' }}"
|
||||
- " Phi-3.5-mini-instruct-Q8_0: {{ 'PRESENT' if 'Phi-3.5-mini-instruct-Q8_0' in nomic_model_ids else 'MISSING' }}"
|
||||
- " Meta-Llama-3.1-8B-Instruct-Q4_K_M: {{ 'PRESENT' if 'Meta-Llama-3.1-8B-Instruct-Q4_K_M' in nomic_model_ids else 'MISSING' }}"
|
||||
- " Qwen2.5-Coder-14B-Instruct-Q4_K_M: {{ 'PRESENT' if 'Qwen2.5-Coder-14B-Instruct-Q4_K_M' in nomic_model_ids else 'MISSING' }}"
|
||||
- " nomic-embed-text-v1.5: {{ 'PRESENT' if 'nomic-embed-text-v1.5' in nomic_model_ids else 'MISSING' }}"
|
||||
- ""
|
||||
- " Embedding smoke test: PASS"
|
||||
- " Vector dimensions: {{ nomic_embed_dims | default('unknown') }}"
|
||||
- ""
|
||||
- " GGUF download: {{ 'NEW DOWNLOAD' if (nomic_download is defined and nomic_download.changed) else 'ALREADY PRESENT (skipped)' }}"
|
||||
- "========================================================================"
|
||||
Reference in New Issue
Block a user